From eb8b64af302e591cea95da9e6271f7653d6ca84d Mon Sep 17 00:00:00 2001 From: Ashmit JaiSarita Gupta Date: Fri, 5 Jun 2026 01:23:19 +0530 Subject: [PATCH 01/27] feat: add Qureg checkpointing via ADIOS2 (#747) Adds saveQuregToFile() and createQuregFromFile() to write a Qureg to disk and restore it later, behind the optional CMake flag ENABLE_CHECKPOINTING (which requires ADIOS2). The file records only the Qureg dimension (numQubits, isDensityMatrix) and its amplitudes - never the incidental deployment fields, nor derivable fields like numAmps - so a Qureg may be restored under a different deployment than it was saved with. Amplitudes are written as an ADIOS2 global array of interleaved (real, imag) reals, with each node contributing only its local slice, so the implementation streams without excessive memory and is distributed- and GPU-ready: GPU state is synced to host before writing and back after reading, and the global-array selection lets any node count read back its own portion. Also adds a validation error when the API is called in a build without checkpointing, reports isCheckpointingCompiled in the environment info (alongside isOmpCompiled, isGpuCompiled, etc), a guarded Catch2 test (tests/unit/checkpoint.cpp) exercising statevector and density-matrix round-trips, and documents the build flag in docs/compile.md. --- CMakeLists.txt | 11 ++++ docs/compile.md | 27 +++++++++ quest/include/qureg.h | 45 ++++++++++++++ quest/src/api/environment.cpp | 23 ++++++-- quest/src/api/qureg.cpp | 107 ++++++++++++++++++++++++++++++++++ quest/src/core/validation.cpp | 35 +++++++++++ quest/src/core/validation.hpp | 4 ++ tests/CMakeLists.txt | 4 ++ tests/unit/CMakeLists.txt | 1 + tests/unit/checkpoint.cpp | 88 ++++++++++++++++++++++++++++ 10 files changed, 339 insertions(+), 6 deletions(-) create mode 100644 tests/unit/checkpoint.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index b5a438713..72093fe49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -542,6 +542,17 @@ if (QUEST_ENABLE_CUQUANTUM) endif() +# Checkpointing (ADIOS2) +option(ENABLE_CHECKPOINTING "Enable Qureg checkpointing (saveQuregToFile / createQuregFromFile) via ADIOS2. Turned OFF by default." OFF) +if (ENABLE_CHECKPOINTING) + find_package(adios2 REQUIRED) + target_link_libraries(QuEST PRIVATE adios2::cxx) + target_compile_definitions(QuEST PRIVATE ENABLE_CHECKPOINTING=1) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) + message(STATUS "Qureg checkpointing is turned ON (via ADIOS2).") +endif() + + # =============================== # Set options to save in config.h diff --git a/docs/compile.md b/docs/compile.md index ba4306a85..56157ce72 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -689,3 +689,30 @@ Note that distributed executables are launched in a distinct way to the other de > - UCX > - launch flags > - checking via reportenv + + + + +------------------ + + +## Checkpointing + +QuEST can optionally _checkpoint_ a `Qureg` to disk; writing its state to a file with `saveQuregToFile()`, to later be restored into a new `Qureg` with `createQuregFromFile()`. This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. The file records only the `Qureg` dimension (the number of qubits, and whether it is a density matrix) and its amplitudes; never the incidental deployment configuration. A `Qureg` saved by one deployment (say, distributed over `8` nodes) can therefore be restored by any other (say, a single GPU-accelerated node). + +Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `ENABLE_CHECKPOINTING` at configuration: +```bash +# configure +cmake .. -D ENABLE_CHECKPOINTING=ON + +# build +cmake --build . --parallel +``` + +> [!IMPORTANT] +> ADIOS2 must be discoverable by CMake. If it was installed to a non-standard location (such as `~/.local`), pass its prefix via `CMAKE_PREFIX_PATH`: +> ```bash +> cmake .. -D ENABLE_CHECKPOINTING=ON -D CMAKE_PREFIX_PATH=$HOME/.local +> ``` + +Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled throws a validation error. diff --git a/quest/include/qureg.h b/quest/include/qureg.h index 4ff4c5627..042bf5676 100644 --- a/quest/include/qureg.h +++ b/quest/include/qureg.h @@ -488,6 +488,51 @@ void getDensityQuregAmps(qcomp** outAmps, Qureg qureg, qindex startRow, qindex s /** @} */ + +/** + * @defgroup qureg_checkpoint Checkpointing + * @brief Functions for saving a Qureg to file and restoring it later. + * @details These functions are only available when QuEST is compiled with + * checkpointing support (CMake variable @c ENABLE_CHECKPOINTING=ON), + * which additionally requires the ADIOS2 library. Calling them in a + * build without checkpointing support throws a validation error. + * @{ + */ + + +/** Writes the contents of @p qureg to the file @p fn, so that it may later be + * restored with createQuregFromFile(). The file records only the @p qureg + * dimension (number of qubits and whether it is a density matrix) and its full + * set of amplitudes; incidental deployment information (e.g. multithreading, + * GPU-acceleration, distribution) is not recorded. + * + * @param[in] qureg the Qureg to write to disk. + * @param[in] fn the output file path. + * @notyetdoced + * @notyettested + * @see + * - createQuregFromFile() to restore a Qureg saved by this function. + */ +void saveQuregToFile(Qureg qureg, const char* fn); + + +/** Creates a new Qureg from a file previously written by saveQuregToFile(), + * with automatically chosen deployments (independent of those used when the + * file was saved), and populates it with the stored amplitudes. + * + * @param[in] fn the input file path. + * @returns A new Qureg instance matching the saved dimension and amplitudes. + * @notyetdoced + * @notyettested + * @see + * - saveQuregToFile() to create a file readable by this function. + */ +Qureg createQuregFromFile(const char* fn); + + +/** @} */ + + // end de-mangler #ifdef __cplusplus } diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index c59334b55..10ffc44d6 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -204,16 +204,27 @@ void printPrecisionInfo() { } +// reports whether QuEST was compiled with Qureg checkpointing support (ADIOS2) +static bool isCheckpointingCompiled() { +#ifdef ENABLE_CHECKPOINTING + return true; +#else + return false; +#endif +} + + void printCompilationInfo() { print_table( "compilation", { - {"isOmpCompiled", cpu_isOpenmpCompiled()}, - {"isMpiCompiled", comm_isMpiCompiled()}, - {"isMpiSubCommCompiled", comm_isMpiSubCommCompiled()}, - {"isGpuCompiled", gpu_isGpuCompiled()}, - {"isHipCompiled", gpu_isHipCompiled()}, - {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, + {"isOmpCompiled", cpu_isOpenmpCompiled()}, + {"isMpiCompiled", comm_isMpiCompiled()}, + {"isMpiSubCommCompiled", comm_isMpiSubCommCompiled()}, + {"isGpuCompiled", gpu_isGpuCompiled()}, + {"isHipCompiled", gpu_isHipCompiled()}, + {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, + {"isCheckpointingCompiled", isCheckpointingCompiled()}, }); } diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index 84bcd2bd0..db5350a64 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -25,6 +25,10 @@ #include #include +#ifdef ENABLE_CHECKPOINTING +#include +#endif + using std::string; using std::vector; @@ -560,3 +564,106 @@ vector> getDensityQuregAmps(Qureg qureg, qindex startRow, qindex s getDensityQuregAmps(ptrs.data(), qureg, startRow, startCol, numRows, numCols); return out; } + + + +/* + * CHECKPOINTING + * + * which is compiled only when ENABLE_CHECKPOINTING=ON (requiring ADIOS2). + * The API functions are always defined so that the validation layer can throw + * a clear error in non-checkpointing builds, rather than failing to link. + */ + + +void saveQuregToFile(Qureg qureg, const char* fn) { + validate_quregCheckpointingIsCompiled(__func__); + +#ifdef ENABLE_CHECKPOINTING + validate_quregFields(qureg, __func__); + + // ensure the CPU amplitudes reflect any GPU-resident state before writing + syncQuregFromGpu(qureg); + + adios2::ADIOS adios; + adios2::IO io = adios.DeclareIO("QuESTQuregSave"); + adios2::Engine engine = io.Open(fn, adios2::Mode::Write); + + // global single-value metadata; we deliberately record only the dimension + // and precision, never incidental deployment fields (the loader chooses its + // own deployment) nor derivable fields (like numAmps) + adios2::Variable vNumQubits = io.DefineVariable("numQubits"); + adios2::Variable vIsDensMatr = io.DefineVariable("isDensityMatrix"); + adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); + + // amplitudes are stored as interleaved (real, imag) reals to stay agnostic + // to precision and to ADIOS2's complex-type support; each node writes only + // its local slice into the global array, avoiding excessive memory use + qindex globalReals = 2 * qureg.numAmps; + qindex localReals = 2 * qureg.numAmpsPerNode; + qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; + adios2::Variable vAmps = io.DefineVariable( + "amps", + { (size_t) globalReals }, + { (size_t) startReal }, + { (size_t) localReals }); + + int qrealBytes = (int) sizeof(qreal); + + engine.BeginStep(); + engine.Put(vNumQubits, qureg.numQubits); + engine.Put(vIsDensMatr, qureg.isDensityMatrix); + engine.Put(vQrealBytes, qrealBytes); + engine.Put(vAmps, reinterpret_cast(qureg.cpuAmps)); + engine.EndStep(); + engine.Close(); +#endif +} + + +Qureg createQuregFromFile(const char* fn) { + validate_quregCheckpointingIsCompiled(__func__); + +#ifdef ENABLE_CHECKPOINTING + adios2::ADIOS adios; + adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); + adios2::Engine engine = io.Open(fn, adios2::Mode::Read); + + engine.BeginStep(); + + // read dimension + precision metadata first, so we can size the new Qureg + int numQubits = 0; + int isDensMatr = 0; + int fileQrealBytes = 0; + engine.Get(io.InquireVariable("numQubits"), numQubits); + engine.Get(io.InquireVariable("isDensityMatrix"), isDensMatr); + engine.Get(io.InquireVariable("qrealBytes"), fileQrealBytes); + engine.PerformGets(); + + validate_quregFileMatchesPrecision(fileQrealBytes, __func__); + + // create a matching-dimension Qureg with automatically chosen deployments, + // independent of those used when the file was saved + Qureg qureg = (isDensMatr)? + createDensityQureg(numQubits) : + createQureg(numQubits); + + // read only this node's slice of the global amplitude array into its buffer + qindex localReals = 2 * qureg.numAmpsPerNode; + qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; + adios2::Variable vAmps = io.InquireVariable("amps"); + vAmps.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); + engine.Get(vAmps, reinterpret_cast(qureg.cpuAmps)); + + engine.EndStep(); + engine.Close(); + + // propagate the restored CPU amplitudes to the GPU, if deployed + syncQuregToGpu(qureg); + + return qureg; +#else + // unreachable: the validation above always throws in non-checkpointing builds + return Qureg{}; +#endif +} diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 62ff93166..fb7a6d583 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -277,6 +277,12 @@ namespace report { string QUREG_NOT_STATE_VECTOR = "Expected a statevector Qureg but received a density matrix."; + string QUREG_CHECKPOINTING_NOT_COMPILED = + "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with checkpointing support. Reconfigure with the CMake option -DENABLE_CHECKPOINTING=ON, which additionally requires the ADIOS2 library."; + + string QUREG_FILE_PRECISION_MISMATCH = + "The checkpoint file was written with a qreal precision of ${FILE_BYTES} bytes, but this QuEST build uses ${EXEC_BYTES} bytes. A Qureg can only be restored by a QuEST build using the same floating-point precision (QUEST_FLOAT_PRECISION) as the build which saved it."; + /* * MUTABLE OBJECT FLAGS @@ -1990,6 +1996,35 @@ void validate_quregIsDensityMatrix(Qureg qureg, const char* caller) { assertThat(qureg.isDensityMatrix, report::QUREG_NOT_DENSITY_MATRIX, caller); } +void validate_quregCheckpointingIsCompiled(const char* caller) { + + if (!global_isValidationEnabled) + return; + + // this validation must fire regardless of ENABLE_CHECKPOINTING, so the user + // receives a clear error (rather than a linker error) when calling the + // checkpointing API in a build which did not compile it + #ifdef ENABLE_CHECKPOINTING + bool isCompiled = true; + #else + bool isCompiled = false; + #endif + + assertThat(isCompiled, report::QUREG_CHECKPOINTING_NOT_COMPILED, caller); +} + +void validate_quregFileMatchesPrecision(int fileQrealBytes, const char* caller) { + + if (!global_isValidationEnabled) + return; + + tokenSubs vars = { + {"${FILE_BYTES}", fileQrealBytes}, + {"${EXEC_BYTES}", (int) sizeof(qreal)}}; + + assertThat(fileQrealBytes == (int) sizeof(qreal), report::QUREG_FILE_PRECISION_MISMATCH, vars, caller); +} + /* diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 87f81a0d6..e8eb7306d 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -137,6 +137,10 @@ void validate_quregIsStateVector(Qureg qureg, const char* caller); void validate_quregIsDensityMatrix(Qureg qureg, const char* caller); +void validate_quregCheckpointingIsCompiled(const char* caller); + +void validate_quregFileMatchesPrecision(int fileQrealBytes, const char* caller); + /* diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4d5050e51..7ddcafee8 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,6 +7,10 @@ add_executable(tests target_link_libraries(tests PRIVATE QuEST::QuEST Catch2::Catch2) target_compile_features(tests PUBLIC cxx_std_20) +if (ENABLE_CHECKPOINTING) + target_compile_definitions(tests PRIVATE ENABLE_CHECKPOINTING=1) +endif() + if (QUEST_ENABLE_MPI AND QUEST_ENABLE_SUBCOMM) target_link_libraries(tests PRIVATE MPI::MPI_CXX) endif() diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 59341759f..4e06fac9d 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -4,6 +4,7 @@ target_sources(tests PUBLIC calculations.cpp channels.cpp + checkpoint.cpp debug.cpp decoherence.cpp environment.cpp diff --git a/tests/unit/checkpoint.cpp b/tests/unit/checkpoint.cpp new file mode 100644 index 000000000..8326e62e1 --- /dev/null +++ b/tests/unit/checkpoint.cpp @@ -0,0 +1,88 @@ +/** @file + * Unit tests of Qureg checkpointing (saveQuregToFile / createQuregFromFile). + * + * These tests are only compiled when QuEST is built with the CMake option + * -DENABLE_CHECKPOINTING=ON (which additionally requires the ADIOS2 library). + * + * @author Ashmit JaiSarita Gupta + * + * @defgroup unitcheckpoint Checkpointing + * @ingroup unittests + */ + +#include "quest.h" + +#ifdef ENABLE_CHECKPOINTING + +#include + +#include +#include +#include +#include +#include + +namespace { + + const char* SV_FILE = "test_checkpoint_statevector.bp"; + const char* DM_FILE = "test_checkpoint_densitymatrix.bp"; + + qreal maxStatevectorAmpDiff(Qureg a, Qureg b) { + qreal m = 0; + for (qindex i = 0; i < a.numAmps; i++) + m = std::max(m, std::abs(getQuregAmp(a, i) - getQuregAmp(b, i))); + return m; + } + + qreal maxDensityMatrixAmpDiff(Qureg a, Qureg b) { + qreal m = 0; + qindex dim = (qindex) 1 << a.numQubits; + for (qindex r = 0; r < dim; r++) + for (qindex c = 0; c < dim; c++) + m = std::max(m, std::abs(getDensityQuregAmp(a, r, c) - getDensityQuregAmp(b, r, c))); + return m; + } +} + +TEST_CASE( "saveQuregToFile and createQuregFromFile", "[checkpoint]" ) { + + SECTION( "statevector round-trip preserves dimension and amplitudes" ) { + + Qureg q = createQureg(6); + initRandomPureState(q); + + saveQuregToFile(q, SV_FILE); + Qureg r = createQuregFromFile(SV_FILE); + + CHECK( r.numQubits == q.numQubits ); + CHECK( r.isDensityMatrix == q.isDensityMatrix ); + CHECK( maxStatevectorAmpDiff(q, r) < 1e-12 ); + + destroyQureg(q); + destroyQureg(r); + std::filesystem::remove_all(SV_FILE); + } + + SECTION( "density-matrix round-trip preserves dimension and amplitudes" ) { + + Qureg q = createDensityQureg(4); + initZeroState(q); + for (int t = 0; t < q.numQubits; t++) + applyHadamard(q, t); + applyT(q, 0); + applyControlledPauliX(q, 0, 1); + + saveQuregToFile(q, DM_FILE); + Qureg r = createQuregFromFile(DM_FILE); + + CHECK( r.numQubits == q.numQubits ); + CHECK( r.isDensityMatrix == q.isDensityMatrix ); + CHECK( maxDensityMatrixAmpDiff(q, r) < 1e-12 ); + + destroyQureg(q); + destroyQureg(r); + std::filesystem::remove_all(DM_FILE); + } +} + +#endif // ENABLE_CHECKPOINTING From fdec5e1cc5a51b524e87eca9d7eb439e645eb48e Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Thu, 4 Jun 2026 21:46:34 -0400 Subject: [PATCH 02/27] Adding adios2 download to CMake --- CMakeLists.txt | 51 +++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 72093fe49..606069a18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -542,17 +542,58 @@ if (QUEST_ENABLE_CUQUANTUM) endif() + +### DRAFT BELOW + + # Checkpointing (ADIOS2) option(ENABLE_CHECKPOINTING "Enable Qureg checkpointing (saveQuregToFile / createQuregFromFile) via ADIOS2. Turned OFF by default." OFF) if (ENABLE_CHECKPOINTING) - find_package(adios2 REQUIRED) - target_link_libraries(QuEST PRIVATE adios2::cxx) - target_compile_definitions(QuEST PRIVATE ENABLE_CHECKPOINTING=1) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) - message(STATUS "Qureg checkpointing is turned ON (via ADIOS2).") + + find_package(adios2 QUIET) + + if(NOT adios2_FOUND) + message(STATUS "adios2 not found: fetching ADIOS2 via FetchContent") + + include(FetchContent) + FetchContent_Declare( + adios2 + GIT_REPOSITORY https://github.com/ornladios/ADIOS2.git + GIT_TAG v2.12.1 + ) + + # Forego MPI and CUDA if QuEST won't use + set(ADIOS2_USE_MPI ${QUEST_ENABLE_MPI} CACHE BOOL "" FORCE) + set(ADIOS2_USE_CUDA ${QUEST_ENABLE_CUDA} CACHE BOOL "" FORCE) + + # Forego unused facilities + set(ADIOS2_BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(ADIOS2_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SODIUM OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Fortran OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_HDF5 OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_ZeroMQ OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SST OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_BZip2 OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Blosc OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SZ OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_ZFP OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_PNG OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Profiling OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Python OFF CACHE BOOL "" FORCE) + + FetchContent_MakeAvailable(adios2) + + else() + # force failure (see Oliver's Catch2 trick) + find_package(adios2 REQUIRED) + endif() endif() +### DRAFT ABOVE + + # =============================== # Set options to save in config.h From acfd1872a28cb134323f440bef0291d0b82bbec6 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Thu, 4 Jun 2026 22:00:47 -0400 Subject: [PATCH 03/27] Trigger checkpoint tests --- .github/workflows/compile.yml | 2 ++ .github/workflows/test_free.yml | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index c86de84f1..23087911d 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -67,6 +67,7 @@ jobs: cuda: [ON, OFF] hip: [ON, OFF] cuquantum: [ON, OFF] + adios2: [ON, OFF] mpilib: ['', 'mpich', 'ompi', 'impi', 'msmpi'] # disable deprecated API on MSVC, and assign unique compilers, @@ -249,6 +250,7 @@ jobs: -DQUEST_ENABLE_CUDA=${{ matrix.cuda }} -DQUEST_ENABLE_HIP=${{ matrix.hip }} -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }} + -DENABLE_CHECKPOINTING=${{ matrix.adios2 }} -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }} -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index 2d332e842..7d6ab8642 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -41,7 +41,7 @@ jobs: # we will compile QuEST with all precisions but no parallelisation matrix: os: [ubuntu-latest, macos-latest, windows-latest] - version: [3, 4] + version: [4] # [3, 4] precision: [1, 2, 4] # MSVC cannot compile deprecated v3 tests @@ -68,6 +68,7 @@ jobs: -DQUEST_ENABLE_DEPRECATED_API=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_FLOAT_PRECISION=${{ matrix.precision }} + -DENABLE_CHECKPOINTING=ON # force 'Release' build (needed by MSVC to enable optimisations) - name: Compile @@ -78,9 +79,13 @@ jobs: # TODO: # ctest currently doesn't know of our Catch2 tags, so we # are manually excluding each integration test by name + + # DEBUG: + # runining ONLY the checkpoint flags + - name: Run v4 tests if: ${{ matrix.version == 4 }} - run: ctest -j2 --output-on-failure --schedule-random -C Release -E "density evolution" + run: ctest -j2 --output-on-failure --schedule-random -C Release -E "density evolution" -R "saveQuregToFile" working-directory: ${{ env.build_dir }} # run v3 unit tests in random order From adf6427c92bd89d7aa46d201efb5cd05dc90ee31 Mon Sep 17 00:00:00 2001 From: Ashmit JaiSarita Gupta Date: Fri, 5 Jun 2026 23:00:15 +0000 Subject: [PATCH 04/27] fix: route checkpointing flag through config.h (QUEST_COMPILE_CHECKPOINTING) QuEST defines all compile-time feature macros centrally in config.h (generated from config.h.in). The checkpointing flag was instead passed as a raw target_compile_definitions, so validation.cpp (which doesn't include config.h) saw it undefined and always reported 'not compiled' under the project's normal build path. Add #cmakedefine01 QUEST_COMPILE_CHECKPOINTING to config.h.in, set it from the ENABLE_CHECKPOINTING option, link ADIOS2 to the QuEST target, and switch the sources/tests to #include config.h + #if QUEST_COMPILE_CHECKPOINTING. Remove the per-target compile-definition hacks. Verified: ON build -> config.h has =1 and tests/tests '[checkpoint]' passes (CPU, CPU+OMP); default OFF build has =0 and compiles without ADIOS2. --- CMakeLists.txt | 7 ++++--- quest/include/config.h.in | 4 ++++ quest/src/api/environment.cpp | 3 ++- quest/src/api/qureg.cpp | 7 ++++--- quest/src/core/validation.cpp | 7 ++++--- tests/CMakeLists.txt | 4 ---- tests/unit/checkpoint.cpp | 4 ++-- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 606069a18..2418ba1cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -588,10 +588,10 @@ if (ENABLE_CHECKPOINTING) # force failure (see Oliver's Catch2 trick) find_package(adios2 REQUIRED) endif() -endif() - -### DRAFT ABOVE + target_link_libraries(QuEST PRIVATE adios2::cxx) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) +endif() @@ -605,6 +605,7 @@ set(QUEST_COMPILE_OMP ${QUEST_ENABLE_OMP}) set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI}) set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM}) set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM}) +set(QUEST_COMPILE_CHECKPOINTING ${ENABLE_CHECKPOINTING}) set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API}) diff --git a/quest/include/config.h.in b/quest/include/config.h.in index 1bb8a0470..ef40e4e91 100644 --- a/quest/include/config.h.in +++ b/quest/include/config.h.in @@ -41,6 +41,7 @@ defined(QUEST_COMPILE_CUDA) || \ defined(QUEST_COMPILE_HIP) || \ defined(QUEST_COMPILE_CUQUANTUM) || \ + defined(QUEST_COMPILE_CHECKPOINTING) || \ defined(QUEST_ENABLE_NUMA) || \ defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -84,6 +85,7 @@ #cmakedefine01 QUEST_COMPILE_CUDA #cmakedefine01 QUEST_COMPILE_CUQUANTUM #cmakedefine01 QUEST_COMPILE_HIP +#cmakedefine01 QUEST_COMPILE_CHECKPOINTING // crucial to QuEST source (informs optional NUMA usage) @@ -125,6 +127,7 @@ ! defined(QUEST_COMPILE_CUDA) || \ ! defined(QUEST_COMPILE_HIP) || \ ! defined(QUEST_COMPILE_CUQUANTUM) || \ + ! defined(QUEST_COMPILE_CHECKPOINTING) || \ ! defined(QUEST_ENABLE_NUMA) || \ ! defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ ! defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -152,6 +155,7 @@ ! (QUEST_COMPILE_CUDA == 0 || QUEST_COMPILE_CUDA == 1) || \ ! (QUEST_COMPILE_HIP == 0 || QUEST_COMPILE_HIP == 1) || \ ! (QUEST_COMPILE_CUQUANTUM == 0 || QUEST_COMPILE_CUQUANTUM == 1) || \ + ! (QUEST_COMPILE_CHECKPOINTING == 0 || QUEST_COMPILE_CHECKPOINTING == 1) || \ ! (QUEST_ENABLE_NUMA == 0 || QUEST_ENABLE_NUMA == 1) || \ ! (QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 0 || QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 1) || \ ! (QUEST_DISABLE_DEPRECATION_WARNINGS == 0 || QUEST_DISABLE_DEPRECATION_WARNINGS == 1) diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 10ffc44d6..2685d1494 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -5,6 +5,7 @@ * @author Tyson Jones */ +#include "quest/include/config.h" #include "quest/include/environment.h" #include "quest/include/precision.h" #include "quest/include/modes.h" @@ -206,7 +207,7 @@ void printPrecisionInfo() { // reports whether QuEST was compiled with Qureg checkpointing support (ADIOS2) static bool isCheckpointingCompiled() { -#ifdef ENABLE_CHECKPOINTING +#if QUEST_COMPILE_CHECKPOINTING return true; #else return false; diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index db5350a64..7d1eec3aa 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -5,6 +5,7 @@ * @author Tyson Jones */ +#include "quest/include/config.h" #include "quest/include/qureg.h" #include "quest/include/modes.h" #include "quest/include/environment.h" @@ -25,7 +26,7 @@ #include #include -#ifdef ENABLE_CHECKPOINTING +#if QUEST_COMPILE_CHECKPOINTING #include #endif @@ -579,7 +580,7 @@ vector> getDensityQuregAmps(Qureg qureg, qindex startRow, qindex s void saveQuregToFile(Qureg qureg, const char* fn) { validate_quregCheckpointingIsCompiled(__func__); -#ifdef ENABLE_CHECKPOINTING +#if QUEST_COMPILE_CHECKPOINTING validate_quregFields(qureg, __func__); // ensure the CPU amplitudes reflect any GPU-resident state before writing @@ -624,7 +625,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { Qureg createQuregFromFile(const char* fn) { validate_quregCheckpointingIsCompiled(__func__); -#ifdef ENABLE_CHECKPOINTING +#if QUEST_COMPILE_CHECKPOINTING adios2::ADIOS adios; adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); adios2::Engine engine = io.Open(fn, adios2::Mode::Read); diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index fb7a6d583..c0e010bc0 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -7,6 +7,7 @@ * @author Kshitij Chhabra (patched v3 overflow bug) */ +#include "quest/include/config.h" #include "quest/include/modes.h" #include "quest/include/types.h" #include "quest/include/precision.h" @@ -2001,10 +2002,10 @@ void validate_quregCheckpointingIsCompiled(const char* caller) { if (!global_isValidationEnabled) return; - // this validation must fire regardless of ENABLE_CHECKPOINTING, so the user - // receives a clear error (rather than a linker error) when calling the + // this validation must fire regardless of QUEST_COMPILE_CHECKPOINTING, so the + // user receives a clear error (rather than a linker error) when calling the // checkpointing API in a build which did not compile it - #ifdef ENABLE_CHECKPOINTING + #if QUEST_COMPILE_CHECKPOINTING bool isCompiled = true; #else bool isCompiled = false; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7ddcafee8..4d5050e51 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -7,10 +7,6 @@ add_executable(tests target_link_libraries(tests PRIVATE QuEST::QuEST Catch2::Catch2) target_compile_features(tests PUBLIC cxx_std_20) -if (ENABLE_CHECKPOINTING) - target_compile_definitions(tests PRIVATE ENABLE_CHECKPOINTING=1) -endif() - if (QUEST_ENABLE_MPI AND QUEST_ENABLE_SUBCOMM) target_link_libraries(tests PRIVATE MPI::MPI_CXX) endif() diff --git a/tests/unit/checkpoint.cpp b/tests/unit/checkpoint.cpp index 8326e62e1..b11083e4c 100644 --- a/tests/unit/checkpoint.cpp +++ b/tests/unit/checkpoint.cpp @@ -12,7 +12,7 @@ #include "quest.h" -#ifdef ENABLE_CHECKPOINTING +#if QUEST_COMPILE_CHECKPOINTING #include @@ -85,4 +85,4 @@ TEST_CASE( "saveQuregToFile and createQuregFromFile", "[checkpoint]" ) { } } -#endif // ENABLE_CHECKPOINTING +#endif // QUEST_COMPILE_CHECKPOINTING From 4c4c3dddc9038e25a60259a7b96ddc5d5e096b20 Mon Sep 17 00:00:00 2001 From: Ashmit JaiSarita Gupta Date: Mon, 8 Jun 2026 07:38:23 +0530 Subject: [PATCH 05/27] fix: collective MPI checkpointing + CUDA-off build, rank-safe test cleanup --- CMakeLists.txt | 19 +++++++++++++++---- quest/src/api/qureg.cpp | 18 ++++++++++++++++-- tests/unit/checkpoint.cpp | 17 +++++++++++++++-- 3 files changed, 46 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2418ba1cb..0cf5b7bd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -562,9 +562,13 @@ if (ENABLE_CHECKPOINTING) GIT_TAG v2.12.1 ) - # Forego MPI and CUDA if QuEST won't use - set(ADIOS2_USE_MPI ${QUEST_ENABLE_MPI} CACHE BOOL "" FORCE) - set(ADIOS2_USE_CUDA ${QUEST_ENABLE_CUDA} CACHE BOOL "" FORCE) + # Match ADIOS2's MPI to QuEST's so distributed runs write per-rank slices + # into one shared file. ADIOS2's CUDA support is deliberately left OFF: + # checkpointing copies amps to host memory (syncQuregFromGpu/syncQuregToGpu) + # before any I/O, so ADIOS2 never touches device pointers. Building it with + # CUDA is unnecessary and stalls the Windows CUDA CI job. + set(ADIOS2_USE_MPI ${QUEST_ENABLE_MPI} CACHE BOOL "" FORCE) + set(ADIOS2_USE_CUDA OFF CACHE BOOL "" FORCE) # Forego unused facilities set(ADIOS2_BUILD_TESTING OFF CACHE BOOL "" FORCE) @@ -589,7 +593,14 @@ if (ENABLE_CHECKPOINTING) find_package(adios2 REQUIRED) endif() - target_link_libraries(QuEST PRIVATE adios2::cxx) + # In distributed builds link ADIOS2's MPI-enabled C++ interface: it defines + # ADIOS2_USE_MPI, which exposes the adios2::ADIOS(MPI_Comm) constructor used in + # qureg.cpp for collective per-rank I/O. The serial target lacks it. + if (QUEST_ENABLE_MPI) + target_link_libraries(QuEST PRIVATE adios2::cxx_mpi) + else() + target_link_libraries(QuEST PRIVATE adios2::cxx) + endif() set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) endif() diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index 7d1eec3aa..b82614708 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -28,6 +28,20 @@ #if QUEST_COMPILE_CHECKPOINTING #include +#if QUEST_COMPILE_MPI +#include +#endif +#endif + +// In distributed builds, ADIOS2 must be given QuEST's communicator so that each +// node's call collectively writes/reads its own slice of the shared file. Without +// it, ADIOS2 runs serially per rank and the per-node slices never form one file. +#if QUEST_COMPILE_CHECKPOINTING +#if QUEST_COMPILE_MPI +#define QUEST_MAKE_ADIOS() adios2::ADIOS(MPI_COMM_WORLD) +#else +#define QUEST_MAKE_ADIOS() adios2::ADIOS() +#endif #endif using std::string; @@ -586,7 +600,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // ensure the CPU amplitudes reflect any GPU-resident state before writing syncQuregFromGpu(qureg); - adios2::ADIOS adios; + adios2::ADIOS adios = QUEST_MAKE_ADIOS(); adios2::IO io = adios.DeclareIO("QuESTQuregSave"); adios2::Engine engine = io.Open(fn, adios2::Mode::Write); @@ -626,7 +640,7 @@ Qureg createQuregFromFile(const char* fn) { validate_quregCheckpointingIsCompiled(__func__); #if QUEST_COMPILE_CHECKPOINTING - adios2::ADIOS adios; + adios2::ADIOS adios = QUEST_MAKE_ADIOS(); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); adios2::Engine engine = io.Open(fn, adios2::Mode::Read); diff --git a/tests/unit/checkpoint.cpp b/tests/unit/checkpoint.cpp index b11083e4c..561b0467d 100644 --- a/tests/unit/checkpoint.cpp +++ b/tests/unit/checkpoint.cpp @@ -60,7 +60,15 @@ TEST_CASE( "saveQuregToFile and createQuregFromFile", "[checkpoint]" ) { destroyQureg(q); destroyQureg(r); - std::filesystem::remove_all(SV_FILE); + + // In distributed runs every node opened the same shared file, so only one + // may delete it; a barrier first guarantees all nodes have finished + // reading, and a barrier after keeps the next section's collective write + // from racing a half-removed directory. + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(SV_FILE); + syncQuESTEnv(); } SECTION( "density-matrix round-trip preserves dimension and amplitudes" ) { @@ -81,7 +89,12 @@ TEST_CASE( "saveQuregToFile and createQuregFromFile", "[checkpoint]" ) { destroyQureg(q); destroyQureg(r); - std::filesystem::remove_all(DM_FILE); + + // see the statevector section: one node deletes, barriers bracket cleanup + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(DM_FILE); + syncQuESTEnv(); } } From 921d24d63c8987b1db5897d1caea4e6b2f50984b Mon Sep 17 00:00:00 2001 From: Ashmit JaiSarita Gupta Date: Tue, 9 Jun 2026 02:50:46 +0530 Subject: [PATCH 06/27] renamed option to QUEST_ENABLE_CHECKPOINTING, fixed extern C linkage and MPI adios2 fallback, restored test_free CI, broadened checkpoint tests across deployments --- .github/workflows/compile.yml | 3 +- .github/workflows/test_free.yml | 9 +-- CMakeLists.txt | 26 ++++--- docs/compile.md | 9 ++- quest/include/qureg.h | 2 +- quest/src/api/environment.cpp | 6 +- quest/src/api/qureg.cpp | 22 +++--- quest/src/core/validation.cpp | 2 +- tests/unit/checkpoint.cpp | 116 +++++++++++++++++++++----------- 9 files changed, 121 insertions(+), 74 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 23087911d..3c0b95420 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -47,6 +47,7 @@ jobs: ${{ matrix.cuda == 'ON' && 'CUDA' || '' }} ${{ matrix.hip == 'ON' && 'HIP' || '' }} ${{ matrix.cuquantum == 'ON' && 'CUQ' || '' }} + ${{ matrix.adios2 == 'ON' && 'CKPT' || '' }} runs-on: ${{ matrix.os }} @@ -250,7 +251,7 @@ jobs: -DQUEST_ENABLE_CUDA=${{ matrix.cuda }} -DQUEST_ENABLE_HIP=${{ matrix.hip }} -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }} - -DENABLE_CHECKPOINTING=${{ matrix.adios2 }} + -DQUEST_ENABLE_CHECKPOINTING=${{ matrix.adios2 }} -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }} -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index 7d6ab8642..01311140a 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -41,7 +41,7 @@ jobs: # we will compile QuEST with all precisions but no parallelisation matrix: os: [ubuntu-latest, macos-latest, windows-latest] - version: [4] # [3, 4] + version: [3, 4] precision: [1, 2, 4] # MSVC cannot compile deprecated v3 tests @@ -68,7 +68,7 @@ jobs: -DQUEST_ENABLE_DEPRECATED_API=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_FLOAT_PRECISION=${{ matrix.precision }} - -DENABLE_CHECKPOINTING=ON + -DQUEST_ENABLE_CHECKPOINTING=ON # force 'Release' build (needed by MSVC to enable optimisations) - name: Compile @@ -79,13 +79,10 @@ jobs: # TODO: # ctest currently doesn't know of our Catch2 tags, so we # are manually excluding each integration test by name - - # DEBUG: - # runining ONLY the checkpoint flags - name: Run v4 tests if: ${{ matrix.version == 4 }} - run: ctest -j2 --output-on-failure --schedule-random -C Release -E "density evolution" -R "saveQuregToFile" + run: ctest -j2 --output-on-failure --schedule-random -C Release -E "density evolution" working-directory: ${{ env.build_dir }} # run v3 unit tests in random order diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cf5b7bd5..d70ac04fd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -543,17 +543,24 @@ endif() -### DRAFT BELOW - - # Checkpointing (ADIOS2) -option(ENABLE_CHECKPOINTING "Enable Qureg checkpointing (saveQuregToFile / createQuregFromFile) via ADIOS2. Turned OFF by default." OFF) -if (ENABLE_CHECKPOINTING) +option(QUEST_ENABLE_CHECKPOINTING "Enable Qureg checkpointing (saveQuregToFile / createQuregFromFile) via ADIOS2. Turned OFF by default." OFF) +message(STATUS "Checkpointing is turned ${QUEST_ENABLE_CHECKPOINTING}. Set QUEST_ENABLE_CHECKPOINTING to modify.") +if (QUEST_ENABLE_CHECKPOINTING) find_package(adios2 QUIET) - if(NOT adios2_FOUND) - message(STATUS "adios2 not found: fetching ADIOS2 via FetchContent") + # A distributed QuEST needs an MPI-enabled ADIOS2 (which provides the + # adios2::cxx_mpi target). A serial system install lacks it, so in that case we + # ignore the found package and fetch an MPI-enabled build instead of failing. + set(quest_use_found_adios2 ${adios2_FOUND}) + if (adios2_FOUND AND QUEST_ENABLE_MPI AND NOT TARGET adios2::cxx_mpi) + message(STATUS "Found ADIOS2 lacks MPI support (no adios2::cxx_mpi target); fetching an MPI-enabled build instead") + set(quest_use_found_adios2 FALSE) + endif() + + if(NOT quest_use_found_adios2) + message(STATUS "fetching ADIOS2 via FetchContent") include(FetchContent) FetchContent_Declare( @@ -589,7 +596,8 @@ if (ENABLE_CHECKPOINTING) FetchContent_MakeAvailable(adios2) else() - # force failure (see Oliver's Catch2 trick) + # re-run non-QUIET so configuration fails with a clear error if the package + # somehow became unavailable between the two calls find_package(adios2 REQUIRED) endif() @@ -616,7 +624,7 @@ set(QUEST_COMPILE_OMP ${QUEST_ENABLE_OMP}) set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI}) set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM}) set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM}) -set(QUEST_COMPILE_CHECKPOINTING ${ENABLE_CHECKPOINTING}) +set(QUEST_COMPILE_CHECKPOINTING ${QUEST_ENABLE_CHECKPOINTING}) set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API}) diff --git a/docs/compile.md b/docs/compile.md index 56157ce72..664ac56a0 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -696,14 +696,17 @@ Note that distributed executables are launched in a distinct way to the other de ------------------ + + + ## Checkpointing QuEST can optionally _checkpoint_ a `Qureg` to disk; writing its state to a file with `saveQuregToFile()`, to later be restored into a new `Qureg` with `createQuregFromFile()`. This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. The file records only the `Qureg` dimension (the number of qubits, and whether it is a density matrix) and its amplitudes; never the incidental deployment configuration. A `Qureg` saved by one deployment (say, distributed over `8` nodes) can therefore be restored by any other (say, a single GPU-accelerated node). -Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `ENABLE_CHECKPOINTING` at configuration: +Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `QUEST_ENABLE_CHECKPOINTING` at configuration: ```bash # configure -cmake .. -D ENABLE_CHECKPOINTING=ON +cmake .. -D QUEST_ENABLE_CHECKPOINTING=ON # build cmake --build . --parallel @@ -712,7 +715,7 @@ cmake --build . --parallel > [!IMPORTANT] > ADIOS2 must be discoverable by CMake. If it was installed to a non-standard location (such as `~/.local`), pass its prefix via `CMAKE_PREFIX_PATH`: > ```bash -> cmake .. -D ENABLE_CHECKPOINTING=ON -D CMAKE_PREFIX_PATH=$HOME/.local +> cmake .. -D QUEST_ENABLE_CHECKPOINTING=ON -D CMAKE_PREFIX_PATH=$HOME/.local > ``` Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled throws a validation error. diff --git a/quest/include/qureg.h b/quest/include/qureg.h index 042bf5676..b0e33aa1d 100644 --- a/quest/include/qureg.h +++ b/quest/include/qureg.h @@ -493,7 +493,7 @@ void getDensityQuregAmps(qcomp** outAmps, Qureg qureg, qindex startRow, qindex s * @defgroup qureg_checkpoint Checkpointing * @brief Functions for saving a Qureg to file and restoring it later. * @details These functions are only available when QuEST is compiled with - * checkpointing support (CMake variable @c ENABLE_CHECKPOINTING=ON), + * checkpointing support (CMake variable @c QUEST_ENABLE_CHECKPOINTING=ON), * which additionally requires the ADIOS2 library. Calling them in a * build without checkpointing support throws a validation error. * @{ diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 2685d1494..700ece439 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -207,11 +207,7 @@ void printPrecisionInfo() { // reports whether QuEST was compiled with Qureg checkpointing support (ADIOS2) static bool isCheckpointingCompiled() { -#if QUEST_COMPILE_CHECKPOINTING - return true; -#else - return false; -#endif + return (bool) QUEST_COMPILE_CHECKPOINTING; } diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index b82614708..70be4fd62 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -33,15 +33,17 @@ #endif #endif +#if QUEST_COMPILE_CHECKPOINTING // In distributed builds, ADIOS2 must be given QuEST's communicator so that each // node's call collectively writes/reads its own slice of the shared file. Without // it, ADIOS2 runs serially per rank and the per-node slices never form one file. -#if QUEST_COMPILE_CHECKPOINTING +static adios2::ADIOS makeAdios() { #if QUEST_COMPILE_MPI -#define QUEST_MAKE_ADIOS() adios2::ADIOS(MPI_COMM_WORLD) + return adios2::ADIOS(MPI_COMM_WORLD); #else -#define QUEST_MAKE_ADIOS() adios2::ADIOS() + return adios2::ADIOS(); #endif +} #endif using std::string; @@ -585,13 +587,17 @@ vector> getDensityQuregAmps(Qureg qureg, qindex startRow, qindex s /* * CHECKPOINTING * - * which is compiled only when ENABLE_CHECKPOINTING=ON (requiring ADIOS2). + * which is compiled only when QUEST_ENABLE_CHECKPOINTING=ON (requiring ADIOS2). * The API functions are always defined so that the validation layer can throw * a clear error in non-checkpointing builds, rather than failing to link. + * + * These are defined with C linkage (matching their extern "C" declarations in + * qureg.h) so they remain callable from C consumers; the signatures pass no + * qcomp by value and so stay C-ABI-safe. */ -void saveQuregToFile(Qureg qureg, const char* fn) { +extern "C" void saveQuregToFile(Qureg qureg, const char* fn) { validate_quregCheckpointingIsCompiled(__func__); #if QUEST_COMPILE_CHECKPOINTING @@ -600,7 +606,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // ensure the CPU amplitudes reflect any GPU-resident state before writing syncQuregFromGpu(qureg); - adios2::ADIOS adios = QUEST_MAKE_ADIOS(); + adios2::ADIOS adios = makeAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregSave"); adios2::Engine engine = io.Open(fn, adios2::Mode::Write); @@ -636,11 +642,11 @@ void saveQuregToFile(Qureg qureg, const char* fn) { } -Qureg createQuregFromFile(const char* fn) { +extern "C" Qureg createQuregFromFile(const char* fn) { validate_quregCheckpointingIsCompiled(__func__); #if QUEST_COMPILE_CHECKPOINTING - adios2::ADIOS adios = QUEST_MAKE_ADIOS(); + adios2::ADIOS adios = makeAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); adios2::Engine engine = io.Open(fn, adios2::Mode::Read); diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index c0e010bc0..aa1d0b4ec 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -279,7 +279,7 @@ namespace report { "Expected a statevector Qureg but received a density matrix."; string QUREG_CHECKPOINTING_NOT_COMPILED = - "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with checkpointing support. Reconfigure with the CMake option -DENABLE_CHECKPOINTING=ON, which additionally requires the ADIOS2 library."; + "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with checkpointing support. Reconfigure with the CMake option -DQUEST_ENABLE_CHECKPOINTING=ON, which additionally requires the ADIOS2 library."; string QUREG_FILE_PRECISION_MISMATCH = "The checkpoint file was written with a qreal precision of ${FILE_BYTES} bytes, but this QuEST build uses ${EXEC_BYTES} bytes. A Qureg can only be restored by a QuEST build using the same floating-point precision (QUEST_FLOAT_PRECISION) as the build which saved it."; diff --git a/tests/unit/checkpoint.cpp b/tests/unit/checkpoint.cpp index 561b0467d..121cbd5fb 100644 --- a/tests/unit/checkpoint.cpp +++ b/tests/unit/checkpoint.cpp @@ -2,7 +2,7 @@ * Unit tests of Qureg checkpointing (saveQuregToFile / createQuregFromFile). * * These tests are only compiled when QuEST is built with the CMake option - * -DENABLE_CHECKPOINTING=ON (which additionally requires the ADIOS2 library). + * -DQUEST_ENABLE_CHECKPOINTING=ON (which additionally requires the ADIOS2 library). * * @author Ashmit JaiSarita Gupta * @@ -16,12 +16,24 @@ #include +#include "tests/utils/macros.hpp" +#include "tests/utils/cache.hpp" + #include #include #include #include #include + + +/* + * file constants and helpers + */ + +#define TEST_CATEGORY \ + LABEL_UNIT_TAG "[checkpoint]" + namespace { const char* SV_FILE = "test_checkpoint_statevector.bp"; @@ -42,60 +54,84 @@ namespace { m = std::max(m, std::abs(getDensityQuregAmp(a, r, c) - getDensityQuregAmp(b, r, c))); return m; } + + // distributed-safe cleanup: a barrier guarantees every node has finished + // reading the shared file, only rank 0 deletes it (concurrent removal races), + // and a second barrier stops the next write racing a half-removed directory. + void removeCheckpointFile(const char* fn) { + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(fn); + syncQuESTEnv(); + } } -TEST_CASE( "saveQuregToFile and createQuregFromFile", "[checkpoint]" ) { - SECTION( "statevector round-trip preserves dimension and amplitudes" ) { - Qureg q = createQureg(6); - initRandomPureState(q); +/** TESTS + * + * @ingroup unitcheckpoint + * @{ + */ - saveQuregToFile(q, SV_FILE); - Qureg r = createQuregFromFile(SV_FILE); +TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { - CHECK( r.numQubits == q.numQubits ); - CHECK( r.isDensityMatrix == q.isDensityMatrix ); - CHECK( maxStatevectorAmpDiff(q, r) < 1e-12 ); + SECTION( LABEL_CORRECTNESS ) { - destroyQureg(q); - destroyQureg(r); + // iterate the cached Quregs so the save path is exercised under every + // deployment combination (serial, OMP, MPI, GPU and their mixtures); + // each restored Qureg chooses its own deployment independently + SECTION( LABEL_STATEVEC ) { - // In distributed runs every node opened the same shared file, so only one - // may delete it; a barrier first guarantees all nodes have finished - // reading, and a barrier after keeps the next section's collective write - // from racing a half-removed directory. - syncQuESTEnv(); - if (getQuESTEnv().rank == 0) - std::filesystem::remove_all(SV_FILE); - syncQuESTEnv(); - } + for (auto& [label, q] : getCachedStatevecs()) { + DYNAMIC_SECTION( label ) { - SECTION( "density-matrix round-trip preserves dimension and amplitudes" ) { + initRandomPureState(q); - Qureg q = createDensityQureg(4); - initZeroState(q); - for (int t = 0; t < q.numQubits; t++) - applyHadamard(q, t); - applyT(q, 0); - applyControlledPauliX(q, 0, 1); + saveQuregToFile(q, SV_FILE); + Qureg r = createQuregFromFile(SV_FILE); - saveQuregToFile(q, DM_FILE); - Qureg r = createQuregFromFile(DM_FILE); + CHECK( r.numQubits == q.numQubits ); + CHECK( r.isDensityMatrix == q.isDensityMatrix ); + CHECK( maxStatevectorAmpDiff(q, r) < 1e-12 ); - CHECK( r.numQubits == q.numQubits ); - CHECK( r.isDensityMatrix == q.isDensityMatrix ); - CHECK( maxDensityMatrixAmpDiff(q, r) < 1e-12 ); + destroyQureg(r); + removeCheckpointFile(SV_FILE); + } + } + } - destroyQureg(q); - destroyQureg(r); + SECTION( LABEL_DENSMATR ) { - // see the statevector section: one node deletes, barriers bracket cleanup - syncQuESTEnv(); - if (getQuESTEnv().rank == 0) - std::filesystem::remove_all(DM_FILE); - syncQuESTEnv(); + for (auto& [label, q] : getCachedDensmatrs()) { + DYNAMIC_SECTION( label ) { + + initRandomPureState(q); // works even for density matrices + + saveQuregToFile(q, DM_FILE); + Qureg r = createQuregFromFile(DM_FILE); + + CHECK( r.numQubits == q.numQubits ); + CHECK( r.isDensityMatrix == q.isDensityMatrix ); + CHECK( maxDensityMatrixAmpDiff(q, r) < 1e-12 ); + + destroyQureg(r); + removeCheckpointFile(DM_FILE); + } + } + } + } + + SECTION( LABEL_VALIDATION ) { + + // The only checkpointing-specific validation - calling the API when QuEST + // was compiled without checkpointing - is unreachable here, since this + // file only compiles under QUEST_COMPILE_CHECKPOINTING. ADIOS2's own + // runtime errors (e.g. a missing file) are not QuEST validation errors. + SUCCEED( ); } } +/** @} (end defgroup) */ + #endif // QUEST_COMPILE_CHECKPOINTING From 24096866d9b0bda88d7c7cbd3e9aa86b25a9a21f Mon Sep 17 00:00:00 2001 From: Ashmit JaiSarita Gupta Date: Tue, 9 Jun 2026 12:50:37 +0530 Subject: [PATCH 07/27] fix: disable ADIOS2 streaming engines to stop Linux CI OOM The FetchContent ADIOS2 build was OOM-killed (exit 143) on the Linux CI runners while compiling the EVPath/atl/ffs/dill/enet third-party stack pulled in by the DataMan/SSC/MHS/SST network engines. Checkpointing only uses the local BP5 file engine, so disable all streaming/staging engines (plus MGARD and Blosc2). Verified the slimmed ADIOS2 still builds and the [checkpoint] round-trip passes serially and under MPI. --- CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index d70ac04fd..b99161fc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -585,8 +585,14 @@ if (QUEST_ENABLE_CHECKPOINTING) set(ADIOS2_USE_HDF5 OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_ZeroMQ OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_SST OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_DataMan OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SSC OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_MHS OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_DAOS OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_MGARD OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_BZip2 OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_Blosc OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Blosc2 OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_SZ OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_ZFP OFF CACHE BOOL "" FORCE) set(ADIOS2_USE_PNG OFF CACHE BOOL "" FORCE) From be3de773df662248a9197bf8b729ff97d5e3157c Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Wed, 10 Jun 2026 23:55:48 -0400 Subject: [PATCH 08/27] Free space before ADIOS2 installation --- .github/workflows/compile.yml | 4 ++-- .github/workflows/test_free.yml | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 3c0b95420..82e23b17e 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -159,9 +159,9 @@ jobs: # perform the job steps: - # free space for big-chungus ROCm compiler + # free space for big-chungus ROCm compiler, and ADIOS2 installation - name: Free disk space - if: ${{ matrix.hip == 'ON' }} + if: ${{ matrix.hip == 'ON' || matrix.adios2 == 'ON' }} uses: jlumbroso/free-disk-space@main with: tool-cache: false diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index 01311140a..edb5c1fc5 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -59,6 +59,12 @@ jobs: - name: Get QuEST uses: actions/checkout@main + # free space for big-chungus ADIOS2 installation + - name: Free disk space + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + # compile serial unit tests, optionally include deprecated test - name: Configure CMake run: > From b849ba411b4531b17d3ab4485f32b7ef35e8e3c8 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Thu, 11 Jun 2026 00:00:34 -0400 Subject: [PATCH 09/27] restrict pre-ADIOS2 memory free to linux since that was the only OS seeing the timeout anyhow --- .github/workflows/compile.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 82e23b17e..f320a0cf8 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -159,9 +159,9 @@ jobs: # perform the job steps: - # free space for big-chungus ROCm compiler, and ADIOS2 installation + # free space for big-chungus ROCm compiler, and ADIOS2 installation (only times out on Linux) - name: Free disk space - if: ${{ matrix.hip == 'ON' || matrix.adios2 == 'ON' }} + if: ${{ (matrix.hip == 'ON' || matrix.adios2 == 'ON') && matrix.os == 'ubuntu-latest' }} uses: jlumbroso/free-disk-space@main with: tool-cache: false From 122e35e633eda5c615003f9c07210ee194b9076a Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Thu, 11 Jun 2026 00:10:50 -0400 Subject: [PATCH 10/27] Force serial compilation to shrink memory to attemptedly avoid ADIOS2 OOM --- .github/workflows/compile.yml | 9 +++++---- .github/workflows/test_free.yml | 10 ++-------- 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index f320a0cf8..75a0111cb 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -159,9 +159,9 @@ jobs: # perform the job steps: - # free space for big-chungus ROCm compiler, and ADIOS2 installation (only times out on Linux) + # free space for big-chungus ROCm compiler - name: Free disk space - if: ${{ (matrix.hip == 'ON' || matrix.adios2 == 'ON') && matrix.os == 'ubuntu-latest' }} + if: ${{ matrix.hip == 'ON' }} uses: jlumbroso/free-disk-space@main with: tool-cache: false @@ -257,9 +257,10 @@ jobs: -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_CXX_FLAGS=${{ matrix.mpi == 'ON' && matrix.cuda == 'ON' && '-fno-lto' || '' }} - # force 'Release' build (needed by MSVC to enable optimisations) + # force 'Release' build (needed by MSVC to enable optimisations), + # temporarily forcing serial compilation to avoid ADIOS2 OOM error - name: Compile - run: cmake --build ${{ env.build_dir }} --config Release --parallel + run: cmake --build ${{ env.build_dir }} --config Release --parallel 1 # run all compiled isolated examples to test for link-time errors, # continuing if any fail (since some deliberately fail) diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index edb5c1fc5..0f12cae6b 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -59,12 +59,6 @@ jobs: - name: Get QuEST uses: actions/checkout@main - # free space for big-chungus ADIOS2 installation - - name: Free disk space - uses: jlumbroso/free-disk-space@main - with: - tool-cache: false - # compile serial unit tests, optionally include deprecated test - name: Configure CMake run: > @@ -76,9 +70,9 @@ jobs: -DQUEST_FLOAT_PRECISION=${{ matrix.precision }} -DQUEST_ENABLE_CHECKPOINTING=ON - # force 'Release' build (needed by MSVC to enable optimisations) + # force 'Release' build (needed by MSVC to enable optimisations), and force serial (to avoid ADIOS2 OOM) - name: Compile - run: cmake --build ${{ env.build_dir }} --config Release --parallel + run: cmake --build ${{ env.build_dir }} --config Release --parallel 1 # run v4 unit tests in random order, excluding the integration tests, # using the default environment variables (e.g. test all permutations) From f0dcb21297503d6e3e50ca8e9c4f68214d674b30 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 19 Jun 2026 14:50:49 -0400 Subject: [PATCH 11/27] renamed CHECKPOINTING to ADIOS2 for consistency with other options, like _MPI and _OMP. Also separates file loading from "checkpointing", since maybe it can be used more generally --- .github/workflows/compile.yml | 2 +- .github/workflows/test_free.yml | 2 +- CMakeLists.txt | 16 +++++++++++----- docs/compile.md | 11 ++++++++--- quest/include/config.h.in | 8 ++++---- quest/include/qureg.h | 8 +++++++- quest/src/api/environment.cpp | 6 +++++- quest/src/api/qureg.cpp | 19 ++++++++++++++----- quest/src/core/validation.cpp | 20 +++++++++++++------- tests/unit/checkpoint.cpp | 8 ++++---- 10 files changed, 68 insertions(+), 32 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 75a0111cb..9587de067 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -251,7 +251,7 @@ jobs: -DQUEST_ENABLE_CUDA=${{ matrix.cuda }} -DQUEST_ENABLE_HIP=${{ matrix.hip }} -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }} - -DQUEST_ENABLE_CHECKPOINTING=${{ matrix.adios2 }} + -DQUEST_ENABLE_ADIOS2=${{ matrix.adios2 }} -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }} -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index 0f12cae6b..cdd06ecfd 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -68,7 +68,7 @@ jobs: -DQUEST_ENABLE_DEPRECATED_API=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_FLOAT_PRECISION=${{ matrix.precision }} - -DQUEST_ENABLE_CHECKPOINTING=ON + -DQUEST_ENABLE_ADIOS2=ON # force 'Release' build (needed by MSVC to enable optimisations), and force serial (to avoid ADIOS2 OOM) - name: Compile diff --git a/CMakeLists.txt b/CMakeLists.txt index b99161fc4..54f6994e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,7 +187,6 @@ message(STATUS "AMD GPU acceleration is turned ${QUEST_ENABLE_HIP}. Set QUEST_EN # GPU Performance Tuning # (We do not print this value when configuring CMake as it is for advanced users only) - set(quest_tpb_description # (the games we play for multi-line set() strings!) "The default number of threads per block QuEST will use when offloading to a GPU. Set to 128 by default. " "Must be a multiple of 32 (on NVIDIA GPUs) or 64 (on AMD GPUs). Can be overridden at executable launch " @@ -199,6 +198,15 @@ set(QUEST_DEFAULT_NUM_GPU_THREADS_PER_BLOCK 128 mark_as_advanced(QUEST_DEFAULT_NUM_GPU_THREADS_PER_BLOCK) +# Checkpointing with ADIOS2 +option( + QUEST_ENABLE_ADIOS2 + "Whether QuEST will be built with ADIOS2, enabling checkpointing (via saveQuregToFile / createQuregFromFile). Turned OFF by default." + OFF +) +message(STATUS "ADIOS2 integration is turned ${QUEST_ENABLE_ADIOS2}. Set QUEST_ENABLE_ADIOS2 to modify.") + + # Deprecated API option( QUEST_ENABLE_DEPRECATED_API @@ -544,9 +552,7 @@ endif() # Checkpointing (ADIOS2) -option(QUEST_ENABLE_CHECKPOINTING "Enable Qureg checkpointing (saveQuregToFile / createQuregFromFile) via ADIOS2. Turned OFF by default." OFF) -message(STATUS "Checkpointing is turned ${QUEST_ENABLE_CHECKPOINTING}. Set QUEST_ENABLE_CHECKPOINTING to modify.") -if (QUEST_ENABLE_CHECKPOINTING) +if (QUEST_ENABLE_ADIOS2) find_package(adios2 QUIET) @@ -630,7 +636,7 @@ set(QUEST_COMPILE_OMP ${QUEST_ENABLE_OMP}) set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI}) set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM}) set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM}) -set(QUEST_COMPILE_CHECKPOINTING ${QUEST_ENABLE_CHECKPOINTING}) +set(QUEST_COMPILE_ADIOS2 ${QUEST_ENABLE_ADIOS2}) set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API}) diff --git a/docs/compile.md b/docs/compile.md index 664ac56a0..3a40bff2b 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -701,12 +701,17 @@ Note that distributed executables are launched in a distinct way to the other de ## Checkpointing + + TODO: + Update below to mention automatic ADIOS2 download and build + + QuEST can optionally _checkpoint_ a `Qureg` to disk; writing its state to a file with `saveQuregToFile()`, to later be restored into a new `Qureg` with `createQuregFromFile()`. This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. The file records only the `Qureg` dimension (the number of qubits, and whether it is a density matrix) and its amplitudes; never the incidental deployment configuration. A `Qureg` saved by one deployment (say, distributed over `8` nodes) can therefore be restored by any other (say, a single GPU-accelerated node). -Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `QUEST_ENABLE_CHECKPOINTING` at configuration: +Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `QUEST_ENABLE_ADIOS2` at configuration: ```bash # configure -cmake .. -D QUEST_ENABLE_CHECKPOINTING=ON +cmake .. -D QUEST_ENABLE_ADIOS2=ON # build cmake --build . --parallel @@ -715,7 +720,7 @@ cmake --build . --parallel > [!IMPORTANT] > ADIOS2 must be discoverable by CMake. If it was installed to a non-standard location (such as `~/.local`), pass its prefix via `CMAKE_PREFIX_PATH`: > ```bash -> cmake .. -D QUEST_ENABLE_CHECKPOINTING=ON -D CMAKE_PREFIX_PATH=$HOME/.local +> cmake .. -D QUEST_ENABLE_ADIOS2=ON -D CMAKE_PREFIX_PATH=$HOME/.local > ``` Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled throws a validation error. diff --git a/quest/include/config.h.in b/quest/include/config.h.in index ef40e4e91..d89df4bfc 100644 --- a/quest/include/config.h.in +++ b/quest/include/config.h.in @@ -41,7 +41,7 @@ defined(QUEST_COMPILE_CUDA) || \ defined(QUEST_COMPILE_HIP) || \ defined(QUEST_COMPILE_CUQUANTUM) || \ - defined(QUEST_COMPILE_CHECKPOINTING) || \ + defined(QUEST_COMPILE_ADIOS2) || \ defined(QUEST_ENABLE_NUMA) || \ defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -85,7 +85,7 @@ #cmakedefine01 QUEST_COMPILE_CUDA #cmakedefine01 QUEST_COMPILE_CUQUANTUM #cmakedefine01 QUEST_COMPILE_HIP -#cmakedefine01 QUEST_COMPILE_CHECKPOINTING +#cmakedefine01 QUEST_COMPILE_ADIOS2 // crucial to QuEST source (informs optional NUMA usage) @@ -127,7 +127,7 @@ ! defined(QUEST_COMPILE_CUDA) || \ ! defined(QUEST_COMPILE_HIP) || \ ! defined(QUEST_COMPILE_CUQUANTUM) || \ - ! defined(QUEST_COMPILE_CHECKPOINTING) || \ + ! defined(QUEST_COMPILE_ADIOS2) || \ ! defined(QUEST_ENABLE_NUMA) || \ ! defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ ! defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -155,7 +155,7 @@ ! (QUEST_COMPILE_CUDA == 0 || QUEST_COMPILE_CUDA == 1) || \ ! (QUEST_COMPILE_HIP == 0 || QUEST_COMPILE_HIP == 1) || \ ! (QUEST_COMPILE_CUQUANTUM == 0 || QUEST_COMPILE_CUQUANTUM == 1) || \ - ! (QUEST_COMPILE_CHECKPOINTING == 0 || QUEST_COMPILE_CHECKPOINTING == 1) || \ + ! (QUEST_COMPILE_ADIOS2 == 0 || QUEST_COMPILE_ADIOS2 == 1) || \ ! (QUEST_ENABLE_NUMA == 0 || QUEST_ENABLE_NUMA == 1) || \ ! (QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 0 || QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 1) || \ ! (QUEST_DISABLE_DEPRECATION_WARNINGS == 0 || QUEST_DISABLE_DEPRECATION_WARNINGS == 1) diff --git a/quest/include/qureg.h b/quest/include/qureg.h index b0e33aa1d..3e7427f4a 100644 --- a/quest/include/qureg.h +++ b/quest/include/qureg.h @@ -489,11 +489,17 @@ void getDensityQuregAmps(qcomp** outAmps, Qureg qureg, qindex startRow, qindex s + + + + // TODO + // move below to experimental (doc group 'checkpoint' should have been in a separate file anyway) + /** * @defgroup qureg_checkpoint Checkpointing * @brief Functions for saving a Qureg to file and restoring it later. * @details These functions are only available when QuEST is compiled with - * checkpointing support (CMake variable @c QUEST_ENABLE_CHECKPOINTING=ON), + * checkpointing support (CMake variable @c QUEST_ENABLE_ADIOS2=ON), * which additionally requires the ADIOS2 library. Calling them in a * build without checkpointing support throws a validation error. * @{ diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index 700ece439..bb13b52d7 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -205,12 +205,16 @@ void printPrecisionInfo() { } + +// TODO: possibly move this + // reports whether QuEST was compiled with Qureg checkpointing support (ADIOS2) static bool isCheckpointingCompiled() { - return (bool) QUEST_COMPILE_CHECKPOINTING; + return (bool) QUEST_COMPILE_ADIOS2; } + void printCompilationInfo() { print_table( diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index 70be4fd62..7a32d8bbd 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -26,14 +26,18 @@ #include #include -#if QUEST_COMPILE_CHECKPOINTING + + +// TODO: +// move this to experimental +#ifdef QUEST_COMPILE_ADIOS2 #include #if QUEST_COMPILE_MPI #include #endif #endif -#if QUEST_COMPILE_CHECKPOINTING +#if QUEST_COMPILE_ADIOS2 // In distributed builds, ADIOS2 must be given QuEST's communicator so that each // node's call collectively writes/reads its own slice of the shared file. Without // it, ADIOS2 runs serially per rank and the per-node slices never form one file. @@ -584,10 +588,15 @@ vector> getDensityQuregAmps(Qureg qureg, qindex startRow, qindex s + + +// TODO: +// move this to experimental + /* * CHECKPOINTING * - * which is compiled only when QUEST_ENABLE_CHECKPOINTING=ON (requiring ADIOS2). + * which is compiled only when QUEST_COMPILE_ADIOS2=ON (requiring ADIOS2). * The API functions are always defined so that the validation layer can throw * a clear error in non-checkpointing builds, rather than failing to link. * @@ -600,7 +609,7 @@ vector> getDensityQuregAmps(Qureg qureg, qindex startRow, qindex s extern "C" void saveQuregToFile(Qureg qureg, const char* fn) { validate_quregCheckpointingIsCompiled(__func__); -#if QUEST_COMPILE_CHECKPOINTING +#ifdef QUEST_COMPILE_ADIOS2 validate_quregFields(qureg, __func__); // ensure the CPU amplitudes reflect any GPU-resident state before writing @@ -645,7 +654,7 @@ extern "C" void saveQuregToFile(Qureg qureg, const char* fn) { extern "C" Qureg createQuregFromFile(const char* fn) { validate_quregCheckpointingIsCompiled(__func__); -#if QUEST_COMPILE_CHECKPOINTING +#ifdef QUEST_COMPILE_ADIOS2 adios2::ADIOS adios = makeAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); adios2::Engine engine = io.Open(fn, adios2::Mode::Read); diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index aa1d0b4ec..cf004d223 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -278,13 +278,19 @@ namespace report { string QUREG_NOT_STATE_VECTOR = "Expected a statevector Qureg but received a density matrix."; - string QUREG_CHECKPOINTING_NOT_COMPILED = - "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with checkpointing support. Reconfigure with the CMake option -DQUEST_ENABLE_CHECKPOINTING=ON, which additionally requires the ADIOS2 library."; - string QUREG_FILE_PRECISION_MISMATCH = "The checkpoint file was written with a qreal precision of ${FILE_BYTES} bytes, but this QuEST build uses ${EXEC_BYTES} bytes. A Qureg can only be restored by a QuEST build using the same floating-point precision (QUEST_FLOAT_PRECISION) as the build which saved it."; + + // TODO: move this + + string ADIOS2_NOT_COMPILED = + "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with ADIOS2. Reconfigure with the CMake option -DQUEST_ENABLE_ADIOS2=ON."; + + + + /* * MUTABLE OBJECT FLAGS */ @@ -2002,16 +2008,16 @@ void validate_quregCheckpointingIsCompiled(const char* caller) { if (!global_isValidationEnabled) return; - // this validation must fire regardless of QUEST_COMPILE_CHECKPOINTING, so the - // user receives a clear error (rather than a linker error) when calling the + // this validation must fire regardless of QUEST_ENABLE_ADIOS2, so the user + // receives a clear error (rather than a linker error) when calling the // checkpointing API in a build which did not compile it - #if QUEST_COMPILE_CHECKPOINTING + #ifdef QUEST_COMPILE_ADIOS2 bool isCompiled = true; #else bool isCompiled = false; #endif - assertThat(isCompiled, report::QUREG_CHECKPOINTING_NOT_COMPILED, caller); + assertThat(isCompiled, report::ADIOS2_NOT_COMPILED, caller); } void validate_quregFileMatchesPrecision(int fileQrealBytes, const char* caller) { diff --git a/tests/unit/checkpoint.cpp b/tests/unit/checkpoint.cpp index 121cbd5fb..62ad0e7d5 100644 --- a/tests/unit/checkpoint.cpp +++ b/tests/unit/checkpoint.cpp @@ -2,7 +2,7 @@ * Unit tests of Qureg checkpointing (saveQuregToFile / createQuregFromFile). * * These tests are only compiled when QuEST is built with the CMake option - * -DQUEST_ENABLE_CHECKPOINTING=ON (which additionally requires the ADIOS2 library). + * -DQUEST_ENABLE_ADIOS2=ON (which additionally requires the ADIOS2 library). * * @author Ashmit JaiSarita Gupta * @@ -12,7 +12,7 @@ #include "quest.h" -#if QUEST_COMPILE_CHECKPOINTING +#ifdef QUEST_COMPILE_ADIOS2 #include @@ -126,7 +126,7 @@ TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { // The only checkpointing-specific validation - calling the API when QuEST // was compiled without checkpointing - is unreachable here, since this - // file only compiles under QUEST_COMPILE_CHECKPOINTING. ADIOS2's own + // file only compiles under QUEST_COMPILE_ADIOS2. ADIOS2's own // runtime errors (e.g. a missing file) are not QuEST validation errors. SUCCEED( ); } @@ -134,4 +134,4 @@ TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { /** @} (end defgroup) */ -#endif // QUEST_COMPILE_CHECKPOINTING +#endif // QUEST_COMPILE_ADIOS2 From 8d5399db923191bc01db682d8fda86aedd6a2ba4 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 19 Jun 2026 15:15:22 -0400 Subject: [PATCH 12/27] move from qureg to experimental group --- quest/include/experimental.h | 40 ++++++++++ quest/include/qureg.h | 52 ------------- quest/src/api/experimental.cpp | 131 ++++++++++++++++++++++++++++++- quest/src/api/qureg.cpp | 136 -------------------------------- tests/unit/CMakeLists.txt | 1 - tests/unit/checkpoint.cpp | 137 --------------------------------- tests/unit/experimental.cpp | 108 ++++++++++++++++++++++++++ 7 files changed, 278 insertions(+), 327 deletions(-) delete mode 100644 tests/unit/checkpoint.cpp diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 8c2cc4e0a..75db4ed5d 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -27,6 +27,9 @@ #include #endif +#include "quest/include/qureg.h" + + // enable invocation by both C and C++ binaries #ifdef __cplusplus extern "C" { @@ -100,6 +103,43 @@ int getQuESTNumGpuThreadsPerBlock(); void setQuESTNumGpuThreadsPerBlock(int numThreadsPerBlock); + + // TODO: + // - change 'fn' to 'dir' + // - note only enabled when QUEST_ENABLE_ADIOS2=ON + // - also link/add to the 'qureg' API module? (Then need to mark this as experimental explicitly?!) + + +/** Writes the contents of @p qureg to the file @p fn, so that it may later be + * restored with createQuregFromFile(). The file records only the @p qureg + * dimension (number of qubits and whether it is a density matrix) and its full + * set of amplitudes; incidental deployment information (e.g. multithreading, + * GPU-acceleration, distribution) is not recorded. + * + * @param[in] qureg the Qureg to write to disk. + * @param[in] fn the output file path. + * @notyetdoced + * @notyettested + * @see + * - createQuregFromFile() to restore a Qureg saved by this function. + */ +void saveQuregToFile(Qureg qureg, const char* fn); + + +/** Creates a new Qureg from a file previously written by saveQuregToFile(), + * with automatically chosen deployments (independent of those used when the + * file was saved), and populates it with the stored amplitudes. + * + * @param[in] fn the input file path. + * @returns A new Qureg instance matching the saved dimension and amplitudes. + * @notyetdoced + * @notyettested + * @see + * - saveQuregToFile() to create a file readable by this function. + */ +Qureg createQuregFromFile(const char* fn); + + // end de-mangler #ifdef __cplusplus } diff --git a/quest/include/qureg.h b/quest/include/qureg.h index 3e7427f4a..3b70e502b 100644 --- a/quest/include/qureg.h +++ b/quest/include/qureg.h @@ -487,58 +487,6 @@ void getDensityQuregAmps(qcomp** outAmps, Qureg qureg, qindex startRow, qindex s /** @} */ - - - - - - // TODO - // move below to experimental (doc group 'checkpoint' should have been in a separate file anyway) - -/** - * @defgroup qureg_checkpoint Checkpointing - * @brief Functions for saving a Qureg to file and restoring it later. - * @details These functions are only available when QuEST is compiled with - * checkpointing support (CMake variable @c QUEST_ENABLE_ADIOS2=ON), - * which additionally requires the ADIOS2 library. Calling them in a - * build without checkpointing support throws a validation error. - * @{ - */ - - -/** Writes the contents of @p qureg to the file @p fn, so that it may later be - * restored with createQuregFromFile(). The file records only the @p qureg - * dimension (number of qubits and whether it is a density matrix) and its full - * set of amplitudes; incidental deployment information (e.g. multithreading, - * GPU-acceleration, distribution) is not recorded. - * - * @param[in] qureg the Qureg to write to disk. - * @param[in] fn the output file path. - * @notyetdoced - * @notyettested - * @see - * - createQuregFromFile() to restore a Qureg saved by this function. - */ -void saveQuregToFile(Qureg qureg, const char* fn); - - -/** Creates a new Qureg from a file previously written by saveQuregToFile(), - * with automatically chosen deployments (independent of those used when the - * file was saved), and populates it with the stored amplitudes. - * - * @param[in] fn the input file path. - * @returns A new Qureg instance matching the saved dimension and amplitudes. - * @notyetdoced - * @notyettested - * @see - * - saveQuregToFile() to create a file readable by this function. - */ -Qureg createQuregFromFile(const char* fn); - - -/** @} */ - - // end de-mangler #ifdef __cplusplus } diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index a6f883656..711c786a9 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -23,6 +23,14 @@ #include #endif +#ifdef QUEST_COMPILE_ADIOS2 + #include + + #if QUEST_COMPILE_MPI + #include + #endif +#endif + /* @@ -45,6 +53,33 @@ extern void validateAndInitCustomQuESTEnv( +/* + * INTERNAL FUNCTIONS + */ + + +// TODO: +// below is broken; we must not give COMM_WORLD, but instead the QuEST +// subcommunicator. Must get this from comm somehow, though this requires +// exposing an MPI type across QuEST translation units. Hmm!!! + + +#if QUEST_COMPILE_ADIOS2 +// In distributed builds, ADIOS2 must be given QuEST's communicator so that each +// node's call collectively writes/reads its own slice of the shared file. Without +// it, ADIOS2 runs serially per rank and the per-node slices never form one file. +static adios2::ADIOS makeAdios() { +#if QUEST_COMPILE_MPI + return adios2::ADIOS(MPI_COMM_WORLD); +#else + return adios2::ADIOS(); +#endif +} +#endif + + + + /* * API FUNCTIONS */ @@ -60,7 +95,6 @@ void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, in #if QUEST_COMPILE_SUBCOMM // hide MPI_Comm - void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) { // useDistrib and userOwnsMpi are implied by the user of this initialiser @@ -103,5 +137,100 @@ void setQuESTNumGpuThreadsPerBlock(int numTPB) { } +void saveQuregToFile(Qureg qureg, const char* fn) { + validate_quregCheckpointingIsCompiled(__func__); + +#ifdef QUEST_COMPILE_ADIOS2 + validate_quregFields(qureg, __func__); + + // ensure the CPU amplitudes reflect any GPU-resident state before writing + syncQuregFromGpu(qureg); + + adios2::ADIOS adios = makeAdios(); + adios2::IO io = adios.DeclareIO("QuESTQuregSave"); + adios2::Engine engine = io.Open(fn, adios2::Mode::Write); + + // global single-value metadata; we deliberately record only the dimension + // and precision, never incidental deployment fields (the loader chooses its + // own deployment) nor derivable fields (like numAmps) + adios2::Variable vNumQubits = io.DefineVariable("numQubits"); + adios2::Variable vIsDensMatr = io.DefineVariable("isDensityMatrix"); + adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); + + // amplitudes are stored as interleaved (real, imag) reals to stay agnostic + // to precision and to ADIOS2's complex-type support; each node writes only + // its local slice into the global array, avoiding excessive memory use + qindex globalReals = 2 * qureg.numAmps; + qindex localReals = 2 * qureg.numAmpsPerNode; + qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; + adios2::Variable vAmps = io.DefineVariable( + "amps", + { (size_t) globalReals }, + { (size_t) startReal }, + { (size_t) localReals }); + + int qrealBytes = (int) sizeof(qreal); + + engine.BeginStep(); + engine.Put(vNumQubits, qureg.numQubits); + engine.Put(vIsDensMatr, qureg.isDensityMatrix); + engine.Put(vQrealBytes, qrealBytes); + engine.Put(vAmps, reinterpret_cast(qureg.cpuAmps)); + engine.EndStep(); + engine.Close(); +#endif +} + + +Qureg createQuregFromFile(const char* fn) { + validate_quregCheckpointingIsCompiled(__func__); + +#ifdef QUEST_COMPILE_ADIOS2 + adios2::ADIOS adios = makeAdios(); + adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); + adios2::Engine engine = io.Open(fn, adios2::Mode::Read); + + engine.BeginStep(); + + // read dimension + precision metadata first, so we can size the new Qureg + int numQubits = 0; + int isDensMatr = 0; + int fileQrealBytes = 0; + engine.Get(io.InquireVariable("numQubits"), numQubits); + engine.Get(io.InquireVariable("isDensityMatrix"), isDensMatr); + engine.Get(io.InquireVariable("qrealBytes"), fileQrealBytes); + engine.PerformGets(); + + validate_quregFileMatchesPrecision(fileQrealBytes, __func__); + + // create a matching-dimension Qureg with automatically chosen deployments, + // independent of those used when the file was saved + Qureg qureg = (isDensMatr)? + createDensityQureg(numQubits) : + createQureg(numQubits); + + // read only this node's slice of the global amplitude array into its buffer + qindex localReals = 2 * qureg.numAmpsPerNode; + qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; + adios2::Variable vAmps = io.InquireVariable("amps"); + vAmps.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); + engine.Get(vAmps, reinterpret_cast(qureg.cpuAmps)); + + engine.EndStep(); + engine.Close(); + + // propagate the restored CPU amplitudes to the GPU, if deployed + syncQuregToGpu(qureg); + + return qureg; +#else + // unreachable: the validation above always throws in non-checkpointing builds + return Qureg{}; +#endif + + // TODO: fix above!!! Will warn non-init? +} + + // end de-mangler } diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index 7a32d8bbd..83565d51f 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -26,30 +26,6 @@ #include #include - - -// TODO: -// move this to experimental -#ifdef QUEST_COMPILE_ADIOS2 -#include -#if QUEST_COMPILE_MPI -#include -#endif -#endif - -#if QUEST_COMPILE_ADIOS2 -// In distributed builds, ADIOS2 must be given QuEST's communicator so that each -// node's call collectively writes/reads its own slice of the shared file. Without -// it, ADIOS2 runs serially per rank and the per-node slices never form one file. -static adios2::ADIOS makeAdios() { -#if QUEST_COMPILE_MPI - return adios2::ADIOS(MPI_COMM_WORLD); -#else - return adios2::ADIOS(); -#endif -} -#endif - using std::string; using std::vector; @@ -585,115 +561,3 @@ vector> getDensityQuregAmps(Qureg qureg, qindex startRow, qindex s getDensityQuregAmps(ptrs.data(), qureg, startRow, startCol, numRows, numCols); return out; } - - - - - -// TODO: -// move this to experimental - -/* - * CHECKPOINTING - * - * which is compiled only when QUEST_COMPILE_ADIOS2=ON (requiring ADIOS2). - * The API functions are always defined so that the validation layer can throw - * a clear error in non-checkpointing builds, rather than failing to link. - * - * These are defined with C linkage (matching their extern "C" declarations in - * qureg.h) so they remain callable from C consumers; the signatures pass no - * qcomp by value and so stay C-ABI-safe. - */ - - -extern "C" void saveQuregToFile(Qureg qureg, const char* fn) { - validate_quregCheckpointingIsCompiled(__func__); - -#ifdef QUEST_COMPILE_ADIOS2 - validate_quregFields(qureg, __func__); - - // ensure the CPU amplitudes reflect any GPU-resident state before writing - syncQuregFromGpu(qureg); - - adios2::ADIOS adios = makeAdios(); - adios2::IO io = adios.DeclareIO("QuESTQuregSave"); - adios2::Engine engine = io.Open(fn, adios2::Mode::Write); - - // global single-value metadata; we deliberately record only the dimension - // and precision, never incidental deployment fields (the loader chooses its - // own deployment) nor derivable fields (like numAmps) - adios2::Variable vNumQubits = io.DefineVariable("numQubits"); - adios2::Variable vIsDensMatr = io.DefineVariable("isDensityMatrix"); - adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); - - // amplitudes are stored as interleaved (real, imag) reals to stay agnostic - // to precision and to ADIOS2's complex-type support; each node writes only - // its local slice into the global array, avoiding excessive memory use - qindex globalReals = 2 * qureg.numAmps; - qindex localReals = 2 * qureg.numAmpsPerNode; - qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; - adios2::Variable vAmps = io.DefineVariable( - "amps", - { (size_t) globalReals }, - { (size_t) startReal }, - { (size_t) localReals }); - - int qrealBytes = (int) sizeof(qreal); - - engine.BeginStep(); - engine.Put(vNumQubits, qureg.numQubits); - engine.Put(vIsDensMatr, qureg.isDensityMatrix); - engine.Put(vQrealBytes, qrealBytes); - engine.Put(vAmps, reinterpret_cast(qureg.cpuAmps)); - engine.EndStep(); - engine.Close(); -#endif -} - - -extern "C" Qureg createQuregFromFile(const char* fn) { - validate_quregCheckpointingIsCompiled(__func__); - -#ifdef QUEST_COMPILE_ADIOS2 - adios2::ADIOS adios = makeAdios(); - adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); - adios2::Engine engine = io.Open(fn, adios2::Mode::Read); - - engine.BeginStep(); - - // read dimension + precision metadata first, so we can size the new Qureg - int numQubits = 0; - int isDensMatr = 0; - int fileQrealBytes = 0; - engine.Get(io.InquireVariable("numQubits"), numQubits); - engine.Get(io.InquireVariable("isDensityMatrix"), isDensMatr); - engine.Get(io.InquireVariable("qrealBytes"), fileQrealBytes); - engine.PerformGets(); - - validate_quregFileMatchesPrecision(fileQrealBytes, __func__); - - // create a matching-dimension Qureg with automatically chosen deployments, - // independent of those used when the file was saved - Qureg qureg = (isDensMatr)? - createDensityQureg(numQubits) : - createQureg(numQubits); - - // read only this node's slice of the global amplitude array into its buffer - qindex localReals = 2 * qureg.numAmpsPerNode; - qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; - adios2::Variable vAmps = io.InquireVariable("amps"); - vAmps.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); - engine.Get(vAmps, reinterpret_cast(qureg.cpuAmps)); - - engine.EndStep(); - engine.Close(); - - // propagate the restored CPU amplitudes to the GPU, if deployed - syncQuregToGpu(qureg); - - return qureg; -#else - // unreachable: the validation above always throws in non-checkpointing builds - return Qureg{}; -#endif -} diff --git a/tests/unit/CMakeLists.txt b/tests/unit/CMakeLists.txt index 4e06fac9d..59341759f 100644 --- a/tests/unit/CMakeLists.txt +++ b/tests/unit/CMakeLists.txt @@ -4,7 +4,6 @@ target_sources(tests PUBLIC calculations.cpp channels.cpp - checkpoint.cpp debug.cpp decoherence.cpp environment.cpp diff --git a/tests/unit/checkpoint.cpp b/tests/unit/checkpoint.cpp deleted file mode 100644 index 62ad0e7d5..000000000 --- a/tests/unit/checkpoint.cpp +++ /dev/null @@ -1,137 +0,0 @@ -/** @file - * Unit tests of Qureg checkpointing (saveQuregToFile / createQuregFromFile). - * - * These tests are only compiled when QuEST is built with the CMake option - * -DQUEST_ENABLE_ADIOS2=ON (which additionally requires the ADIOS2 library). - * - * @author Ashmit JaiSarita Gupta - * - * @defgroup unitcheckpoint Checkpointing - * @ingroup unittests - */ - -#include "quest.h" - -#ifdef QUEST_COMPILE_ADIOS2 - -#include - -#include "tests/utils/macros.hpp" -#include "tests/utils/cache.hpp" - -#include -#include -#include -#include -#include - - - -/* - * file constants and helpers - */ - -#define TEST_CATEGORY \ - LABEL_UNIT_TAG "[checkpoint]" - -namespace { - - const char* SV_FILE = "test_checkpoint_statevector.bp"; - const char* DM_FILE = "test_checkpoint_densitymatrix.bp"; - - qreal maxStatevectorAmpDiff(Qureg a, Qureg b) { - qreal m = 0; - for (qindex i = 0; i < a.numAmps; i++) - m = std::max(m, std::abs(getQuregAmp(a, i) - getQuregAmp(b, i))); - return m; - } - - qreal maxDensityMatrixAmpDiff(Qureg a, Qureg b) { - qreal m = 0; - qindex dim = (qindex) 1 << a.numQubits; - for (qindex r = 0; r < dim; r++) - for (qindex c = 0; c < dim; c++) - m = std::max(m, std::abs(getDensityQuregAmp(a, r, c) - getDensityQuregAmp(b, r, c))); - return m; - } - - // distributed-safe cleanup: a barrier guarantees every node has finished - // reading the shared file, only rank 0 deletes it (concurrent removal races), - // and a second barrier stops the next write racing a half-removed directory. - void removeCheckpointFile(const char* fn) { - syncQuESTEnv(); - if (getQuESTEnv().rank == 0) - std::filesystem::remove_all(fn); - syncQuESTEnv(); - } -} - - - -/** TESTS - * - * @ingroup unitcheckpoint - * @{ - */ - -TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { - - SECTION( LABEL_CORRECTNESS ) { - - // iterate the cached Quregs so the save path is exercised under every - // deployment combination (serial, OMP, MPI, GPU and their mixtures); - // each restored Qureg chooses its own deployment independently - SECTION( LABEL_STATEVEC ) { - - for (auto& [label, q] : getCachedStatevecs()) { - DYNAMIC_SECTION( label ) { - - initRandomPureState(q); - - saveQuregToFile(q, SV_FILE); - Qureg r = createQuregFromFile(SV_FILE); - - CHECK( r.numQubits == q.numQubits ); - CHECK( r.isDensityMatrix == q.isDensityMatrix ); - CHECK( maxStatevectorAmpDiff(q, r) < 1e-12 ); - - destroyQureg(r); - removeCheckpointFile(SV_FILE); - } - } - } - - SECTION( LABEL_DENSMATR ) { - - for (auto& [label, q] : getCachedDensmatrs()) { - DYNAMIC_SECTION( label ) { - - initRandomPureState(q); // works even for density matrices - - saveQuregToFile(q, DM_FILE); - Qureg r = createQuregFromFile(DM_FILE); - - CHECK( r.numQubits == q.numQubits ); - CHECK( r.isDensityMatrix == q.isDensityMatrix ); - CHECK( maxDensityMatrixAmpDiff(q, r) < 1e-12 ); - - destroyQureg(r); - removeCheckpointFile(DM_FILE); - } - } - } - } - - SECTION( LABEL_VALIDATION ) { - - // The only checkpointing-specific validation - calling the API when QuEST - // was compiled without checkpointing - is unreachable here, since this - // file only compiles under QUEST_COMPILE_ADIOS2. ADIOS2's own - // runtime errors (e.g. a missing file) are not QuEST validation errors. - SUCCEED( ); - } -} - -/** @} (end defgroup) */ - -#endif // QUEST_COMPILE_ADIOS2 diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 943645831..7fa4686aa 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -16,6 +16,7 @@ #include "tests/utils/macros.hpp" #include "tests/utils/config.hpp" +#include "tests/utils/cache.hpp" using Catch::Matchers::ContainsSubstring; @@ -121,6 +122,113 @@ TEST_CASE( "getQuESTNumGpuThreadsPerBlock", TEST_CATEGORY ) { } + // TODO: + // - fix this guard! Just runtime skip + // - fix test + + +#ifdef QUEST_COMPILE_ADIOS2 + +#include +#include +#include +#include +#include + +namespace { + + const char* SV_FILE = "test_checkpoint_statevector.bp"; + const char* DM_FILE = "test_checkpoint_densitymatrix.bp"; + + qreal maxStatevectorAmpDiff(Qureg a, Qureg b) { + qreal m = 0; + for (qindex i = 0; i < a.numAmps; i++) + m = std::max(m, std::abs(getQuregAmp(a, i) - getQuregAmp(b, i))); + return m; + } + + qreal maxDensityMatrixAmpDiff(Qureg a, Qureg b) { + qreal m = 0; + qindex dim = (qindex) 1 << a.numQubits; + for (qindex r = 0; r < dim; r++) + for (qindex c = 0; c < dim; c++) + m = std::max(m, std::abs(getDensityQuregAmp(a, r, c) - getDensityQuregAmp(b, r, c))); + return m; + } + + // distributed-safe cleanup: a barrier guarantees every node has finished + // reading the shared file, only rank 0 deletes it (concurrent removal races), + // and a second barrier stops the next write racing a half-removed directory. + void removeCheckpointFile(const char* fn) { + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(fn); + syncQuESTEnv(); + } +} + +TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { + + SECTION( LABEL_CORRECTNESS ) { + + // iterate the cached Quregs so the save path is exercised under every + // deployment combination (serial, OMP, MPI, GPU and their mixtures); + // each restored Qureg chooses its own deployment independently + SECTION( LABEL_STATEVEC ) { + + for (auto& [label, q] : getCachedStatevecs()) { + DYNAMIC_SECTION( label ) { + + initRandomPureState(q); + + saveQuregToFile(q, SV_FILE); + Qureg r = createQuregFromFile(SV_FILE); + + CHECK( r.numQubits == q.numQubits ); + CHECK( r.isDensityMatrix == q.isDensityMatrix ); + CHECK( maxStatevectorAmpDiff(q, r) < 1e-12 ); + + destroyQureg(r); + removeCheckpointFile(SV_FILE); + } + } + } + + SECTION( LABEL_DENSMATR ) { + + for (auto& [label, q] : getCachedDensmatrs()) { + DYNAMIC_SECTION( label ) { + + initRandomPureState(q); // works even for density matrices + + saveQuregToFile(q, DM_FILE); + Qureg r = createQuregFromFile(DM_FILE); + + CHECK( r.numQubits == q.numQubits ); + CHECK( r.isDensityMatrix == q.isDensityMatrix ); + CHECK( maxDensityMatrixAmpDiff(q, r) < 1e-12 ); + + destroyQureg(r); + removeCheckpointFile(DM_FILE); + } + } + } + } + + SECTION( LABEL_VALIDATION ) { + + // The only checkpointing-specific validation - calling the API when QuEST + // was compiled without checkpointing - is unreachable here, since this + // file only compiles under QUEST_COMPILE_ADIOS2. ADIOS2's own + // runtime errors (e.g. a missing file) are not QuEST validation errors. + SUCCEED( ); + } +} + +#endif // QUEST_COMPILE_ADIOS2 + + + /** @} (end defgroup) */ From 6c3d9c40df945642d16f19cc4cf8e93375ab5132 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 19 Jun 2026 15:33:37 -0400 Subject: [PATCH 13/27] make ADIOS use QuEST communicator rather than COMM_WORLD --- quest/src/api/experimental.cpp | 46 +++++++++++++++++++--------------- tests/unit/experimental.cpp | 3 ++- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index 711c786a9..6cbc2b14e 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -52,34 +52,33 @@ extern void validateAndInitCustomQuESTEnv( #endif +#if (QUEST_COMPILE_ADIOS2 && QUEST_COMPILE_MPI) // hide MPI_Comm + extern MPI_Comm comm_getMpiComm(); +#endif + + /* * INTERNAL FUNCTIONS */ -// TODO: -// below is broken; we must not give COMM_WORLD, but instead the QuEST -// subcommunicator. Must get this from comm somehow, though this requires -// exposing an MPI type across QuEST translation units. Hmm!!! - - #if QUEST_COMPILE_ADIOS2 -// In distributed builds, ADIOS2 must be given QuEST's communicator so that each -// node's call collectively writes/reads its own slice of the shared file. Without -// it, ADIOS2 runs serially per rank and the per-node slices never form one file. -static adios2::ADIOS makeAdios() { -#if QUEST_COMPILE_MPI - return adios2::ADIOS(MPI_COMM_WORLD); -#else - return adios2::ADIOS(); -#endif +auto createAdios() { + + // In distributed builds, ADIOS2 must be given QuEST's communicator so that each + // node's call collectively writes/reads its own slice of the shared file. Without + // it, ADIOS2 runs serially per rank and the per-node slices never form one file. + #if QUEST_COMPILE_MPI + return adios2::ADIOS(comm_getMpiComm()); + #else + return adios2::ADIOS(); + #endif } #endif - /* * API FUNCTIONS */ @@ -137,6 +136,15 @@ void setQuESTNumGpuThreadsPerBlock(int numTPB) { } + + // TODO: + // - fix subcomm issue + // - make comment about gratuitous re-creation of ADIOS2 (fine for simplicity) + // - make comment about size_t overflow risk + // - fix Qureg{} return warning issue + // - check restoration to a DISTRIBUTED qureg is correct + + void saveQuregToFile(Qureg qureg, const char* fn) { validate_quregCheckpointingIsCompiled(__func__); @@ -146,7 +154,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // ensure the CPU amplitudes reflect any GPU-resident state before writing syncQuregFromGpu(qureg); - adios2::ADIOS adios = makeAdios(); + adios2::ADIOS adios = createAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregSave"); adios2::Engine engine = io.Open(fn, adios2::Mode::Write); @@ -186,7 +194,7 @@ Qureg createQuregFromFile(const char* fn) { validate_quregCheckpointingIsCompiled(__func__); #ifdef QUEST_COMPILE_ADIOS2 - adios2::ADIOS adios = makeAdios(); + adios2::ADIOS adios = createAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); adios2::Engine engine = io.Open(fn, adios2::Mode::Read); @@ -227,8 +235,6 @@ Qureg createQuregFromFile(const char* fn) { // unreachable: the validation above always throws in non-checkpointing builds return Qureg{}; #endif - - // TODO: fix above!!! Will warn non-init? } diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 7fa4686aa..cb07e8f4f 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -124,7 +124,8 @@ TEST_CASE( "getQuESTNumGpuThreadsPerBlock", TEST_CATEGORY ) { // TODO: // - fix this guard! Just runtime skip - // - fix test + // - fix tests + // - extend tests to CHANGE DEPLOYMENT of the Qureg pre and post restoration! #ifdef QUEST_COMPILE_ADIOS2 From 0858ef26ee30d90a0379f7bdec20d87260acc6ef Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 19 Jun 2026 17:25:48 -0400 Subject: [PATCH 14/27] improve validation --- quest/include/experimental.h | 57 ++++++++++----- quest/include/qureg.h | 1 + quest/src/api/environment.cpp | 12 +-- quest/src/api/experimental.cpp | 130 +++++++++++++++++++++++---------- quest/src/core/validation.cpp | 85 ++++++++++++++++----- quest/src/core/validation.hpp | 14 +++- tests/unit/experimental.cpp | 6 +- 7 files changed, 216 insertions(+), 89 deletions(-) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 75db4ed5d..f5525310c 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -57,8 +57,9 @@ void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, in * The user-provided MPI communicator undergoes the same validation procedure as any that QuEST * would use, and so must contain a power-of-2 number of processes. * - * This function is only compiled and exposed when macro QUEST_COMPILE_SUBCOMM is 1, as is - * defined when providing CMake option QUEST_ENABLE_SUBCOMM during building. + * > [!IMPORTANT] + * > This function is only compiled and exposed when macro QUEST_COMPILE_SUBCOMM is 1, as is + * > defined when providing CMake option QUEST_ENABLE_SUBCOMM during building. * * @author Oliver Brown */ @@ -105,37 +106,59 @@ void setQuESTNumGpuThreadsPerBlock(int numThreadsPerBlock); // TODO: - // - change 'fn' to 'dir' - // - note only enabled when QUEST_ENABLE_ADIOS2=ON // - also link/add to the 'qureg' API module? (Then need to mark this as experimental explicitly?!) + // - add note about file extension???? -/** Writes the contents of @p qureg to the file @p fn, so that it may later be - * restored with createQuregFromFile(). The file records only the @p qureg - * dimension (number of qubits and whether it is a density matrix) and its full - * set of amplitudes; incidental deployment information (e.g. multithreading, - * GPU-acceleration, distribution) is not recorded. +/** Writes the contents of @p qureg to the file (or folder) @p fn, so that it may later be + * restored with createQuregFromFile(), potentially in another process. + * + * The output records only the @p qureg dimension (number of qubits and whether it is a density matrix), + * the amplitude precision, and the Qureg's full set of amplitudes. Deployment information (such as whether + * the Qureg is distributed, or GPU-accelerated) is not recorded. + * + * There is no particular file extension or folder name suffix required, though since saving is + * performed with ADIOS2, a suffix of @p .bp is conventional. + * + * > [!IMPORTANT] + * > This function is only callable when QuEST is compiled with CMake option QUEST_ENABLE_ADIOS2=1. * * @param[in] qureg the Qureg to write to disk. - * @param[in] fn the output file path. - * @notyetdoced - * @notyettested + * @param[in] fn the output file (or folder) path. + * @throws @validationerror + * - if @p qureg is uninitialised. + * - if QuEST was not compiled with CMake option QUEST_ENABLE_ADIOS2=1. + * - if opening or writing to @p fn fails. * @see * - createQuregFromFile() to restore a Qureg saved by this function. + * @author Ashmit JaiSarita Gupta */ void saveQuregToFile(Qureg qureg, const char* fn); -/** Creates a new Qureg from a file previously written by saveQuregToFile(), +/** Creates a new Qureg from a file (or folder) previously created by saveQuregToFile(), * with automatically chosen deployments (independent of those used when the - * file was saved), and populates it with the stored amplitudes. + * file was saved), and populates the Qureg with the saved amplitudes. + * + * The chosen deployments are identical to those chosen by createQureg() and createDensityQureg(). + * + * > [!IMPORTANT] + * > This function is only callable when QuEST is compiled with CMake option QUEST_ENABLE_ADIOS2=1. * - * @param[in] fn the input file path. + * @param[in] fn the file (or folder) path previously created by saveQuregToFile(). * @returns A new Qureg instance matching the saved dimension and amplitudes. - * @notyetdoced - * @notyettested + * @throws @validationerror + * - if QuEST was not compiled with CMake option QUEST_ENABLE_ADIOS2=1. + * - if @p fn cannot be read (since, for example, it does not exist). + * - if the precision of the saved Qureg differs from the current QuEST precision. + * - if the recorded Qureg dimensions would overflow the @c qindex type. + * - if the recorded toatal Qureg memory would overflow the @c size_t type. + * - if the system contains insufficient RAM (or VRAM) to store the Qureg in any deployment. + * - if any Qureg memory allocation unexpectedly fails. * @see * - saveQuregToFile() to create a file readable by this function. + * @author Ashmit JaiSarita Gupta + * @author Tyson Jones (validation) */ Qureg createQuregFromFile(const char* fn); diff --git a/quest/include/qureg.h b/quest/include/qureg.h index 3b70e502b..bd2bc7129 100644 --- a/quest/include/qureg.h +++ b/quest/include/qureg.h @@ -135,6 +135,7 @@ typedef struct { * - createDensityQureg() to create a density matrix which can additionally undergo decoherence. * - createForcedQureg() to create a statevector which is forced to make use of all available deployments. * - createCustomQureg() to explicitly set the used deployments. + * - createQuregFromFile() to create a Qureg from a checkpoint file. * @author Tyson Jones */ Qureg createQureg(int numQubits); diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index bb13b52d7..e6fd2903a 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -205,16 +205,6 @@ void printPrecisionInfo() { } - -// TODO: possibly move this - -// reports whether QuEST was compiled with Qureg checkpointing support (ADIOS2) -static bool isCheckpointingCompiled() { - return (bool) QUEST_COMPILE_ADIOS2; -} - - - void printCompilationInfo() { print_table( @@ -225,7 +215,7 @@ void printCompilationInfo() { {"isGpuCompiled", gpu_isGpuCompiled()}, {"isHipCompiled", gpu_isHipCompiled()}, {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, - {"isCheckpointingCompiled", isCheckpointingCompiled()}, + {"isCheckpointingCompiled", QUEST_COMPILE_ADIOS2}, }); } diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index 6cbc2b14e..ab1e96291 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -10,6 +10,8 @@ #include "quest/include/config.h" #include "quest/include/environment.h" +#include "quest/include/qureg.h" +#include "quest/include/modes.h" #include "quest/src/core/validation.hpp" #include "quest/src/comm/comm_config.hpp" @@ -47,6 +49,10 @@ extern void validateAndInitCustomQuESTEnv( int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller); +extern Qureg validateAndCreateCustomQureg( + int numQubits, int isDensMatr, int useDistrib, int useGpuAccel, int useMultithread, const char* caller); + + #if QUEST_COMPILE_SUBCOMM // hide MPI_Comm extern bool comm_setMpiComm(MPI_Comm newComm, bool userOwnsMpi); #endif @@ -138,97 +144,145 @@ void setQuESTNumGpuThreadsPerBlock(int numTPB) { // TODO: - // - fix subcomm issue - // - make comment about gratuitous re-creation of ADIOS2 (fine for simplicity) // - make comment about size_t overflow risk // - fix Qureg{} return warning issue // - check restoration to a DISTRIBUTED qureg is correct void saveQuregToFile(Qureg qureg, const char* fn) { - validate_quregCheckpointingIsCompiled(__func__); + validate_adios2IsCompiled(__func__); + validate_quregFields(qureg, __func__); #ifdef QUEST_COMPILE_ADIOS2 - validate_quregFields(qureg, __func__); - // ensure the CPU amplitudes reflect any GPU-resident state before writing - syncQuregFromGpu(qureg); + // Pedantic but safe - don't let ADIOS2 start reading amps prematurely + if (qureg.isDistributed) + comm_sync(); + + // TODO: + // We can optimise in GPU settings by giving ADIOS2 the device memory + // pointers; but for now, we simply stage into CPU memory first + if (qureg.isGpuAccelerated) + gpu_copyGpuToCpu(qureg); + // gratuitously re-create ADIOS2 at every call, for simplicity (occluded by IO) adios2::ADIOS adios = createAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregSave"); - adios2::Engine engine = io.Open(fn, adios2::Mode::Write); + + // attempt to open the file + adios2::Engine engine; // default ctor + try { + engine = io.Open(fn, adios2::Mode::Write); + } catch (...) { + validate_adiosCanOpenFile(false, fn, __func__); + } // global single-value metadata; we deliberately record only the dimension // and precision, never incidental deployment fields (the loader chooses its // own deployment) nor derivable fields (like numAmps) adios2::Variable vNumQubits = io.DefineVariable("numQubits"); adios2::Variable vIsDensMatr = io.DefineVariable("isDensityMatrix"); - adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); + adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); // also encodes precision // amplitudes are stored as interleaved (real, imag) reals to stay agnostic // to precision and to ADIOS2's complex-type support; each node writes only // its local slice into the global array, avoiding excessive memory use + // (these scalars are guaranteed not to overflow by createQureg validation) qindex globalReals = 2 * qureg.numAmps; qindex localReals = 2 * qureg.numAmpsPerNode; qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; - adios2::Variable vAmps = io.DefineVariable( - "amps", + adios2::Variable vAmpComponents = io.DefineVariable( + "ampComponents", { (size_t) globalReals }, { (size_t) startReal }, { (size_t) localReals }); - int qrealBytes = (int) sizeof(qreal); + // attempt to write to file + try { + engine.BeginStep(); + engine.Put(vNumQubits, qureg.numQubits); + engine.Put(vIsDensMatr, qureg.isDensityMatrix); + engine.Put(vQrealBytes, sizeof(qreal)); + engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); + engine.EndStep(); + engine.Close(); + } catch (...) { + // no need for a finally; RAII frees engine + validate_adiosCanWriteToFile(false, fn, __func__); + } - engine.BeginStep(); - engine.Put(vNumQubits, qureg.numQubits); - engine.Put(vIsDensMatr, qureg.isDensityMatrix); - engine.Put(vQrealBytes, qrealBytes); - engine.Put(vAmps, reinterpret_cast(qureg.cpuAmps)); - engine.EndStep(); - engine.Close(); #endif } Qureg createQuregFromFile(const char* fn) { - validate_quregCheckpointingIsCompiled(__func__); + validate_adios2IsCompiled(__func__); #ifdef QUEST_COMPILE_ADIOS2 + + // gratuitously re-create ADIOS2 at every call, for simplicity (occluded by IO) adios2::ADIOS adios = createAdios(); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); - adios2::Engine engine = io.Open(fn, adios2::Mode::Read); - engine.BeginStep(); + // attempt to open the file, and prepare to parse + adios2::Engine engine; // default ctor + try { + engine = io.Open(fn, adios2::Mode::Read); + engine.BeginStep(); + } catch (...) { + validate_adiosCanOpenFile(false, fn, __func__); + } + + // check that the file contains the expected variables + auto vNumQubits = io.InquireVariable("numQubits"); + auto vIsDensMatr = io.InquireVariable("isDensityMatrix"); + auto vQrealBytes = io.InquireVariable("qrealBytes"); + auto vAmpComponents = io.InquireVariable("ampComponents"); + bool areAllVarsPresent = vNumQubits && vIsDensMatr && vQrealBytes && vAmpComponents; + validate_adiosFileContainsFields(areAllVarsPresent, __func__); // read dimension + precision metadata first, so we can size the new Qureg int numQubits = 0; int isDensMatr = 0; - int fileQrealBytes = 0; - engine.Get(io.InquireVariable("numQubits"), numQubits); - engine.Get(io.InquireVariable("isDensityMatrix"), isDensMatr); - engine.Get(io.InquireVariable("qrealBytes"), fileQrealBytes); - engine.PerformGets(); + size_t fileQrealBytes = 0; + try { + engine.Get(vNumQubits, numQubits); + engine.Get(vIsDensMatr, isDensMatr); + engine.Get(vQrealBytes, fileQrealBytes); + engine.PerformGets(); + } catch(...) { + validate_adiosCanReadFile(false, fn, __func__); + } - validate_quregFileMatchesPrecision(fileQrealBytes, __func__); + // check the amps are of the expected precision, and so are parsable + validate_newQuregFileMatchesPrecision(fileQrealBytes, __func__); - // create a matching-dimension Qureg with automatically chosen deployments, - // independent of those used when the file was saved - Qureg qureg = (isDensMatr)? - createDensityQureg(numQubits) : - createQureg(numQubits); + // attempt to create a matching-dimension Qureg with automatically chosen deployments + Qureg qureg = validateAndCreateCustomQureg(numQubits, isDensMatr, + modeflag::USE_AUTO, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__); // read only this node's slice of the global amplitude array into its buffer + // (guaranteed not to overflow by above validateAndCreateCustomQureg validation) qindex localReals = 2 * qureg.numAmpsPerNode; qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; - adios2::Variable vAmps = io.InquireVariable("amps"); - vAmps.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); - engine.Get(vAmps, reinterpret_cast(qureg.cpuAmps)); + vAmpComponents.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); + try { + engine.Get(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); // immediate; PerformGets redundant + } catch(...) { + validate_adiosCanReadFile(false, fn, __func__); + } - engine.EndStep(); - engine.Close(); + // complete ADIOS2 work + try { + engine.EndStep(); + engine.Close(); + } catch(...) { + validate_adiosCanReadFile(false, fn, __func__); + } // propagate the restored CPU amplitudes to the GPU, if deployed - syncQuregToGpu(qureg); + if (qureg.isGpuAccelerated) + gpu_copyCpuToGpu(qureg); return qureg; #else diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index cf004d223..a4a5f3792 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -281,16 +281,10 @@ namespace report { string QUREG_FILE_PRECISION_MISMATCH = "The checkpoint file was written with a qreal precision of ${FILE_BYTES} bytes, but this QuEST build uses ${EXEC_BYTES} bytes. A Qureg can only be restored by a QuEST build using the same floating-point precision (QUEST_FLOAT_PRECISION) as the build which saved it."; - - - // TODO: move this - string ADIOS2_NOT_COMPILED = "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with ADIOS2. Reconfigure with the CMake option -DQUEST_ENABLE_ADIOS2=ON."; - - /* * MUTABLE OBJECT FLAGS */ @@ -1161,6 +1155,18 @@ namespace report { string CANNOT_READ_FILE = "Could not load and read the given file. Make sure the file exists and is readable as plaintext."; + string ADIOS2_CANNOT_OPEN_FILE = + "The specified file (or folder) could not be opened by ADIOS2."; + + string ADIOS2_CANNOT_READ_FILE = + "The specified file (or folder) was opened by ADIOS2, but the contents could not be read or loaded."; + + string ADIOS2_CANNOT_WRITE_TO_FILE = + "ADIOS2 failed to write to the specified file (or folder)."; + + string ADIOS2_FILE_INVALID = + "The specified file (or folder) did not contain the expected AIODS2 variables, suggesting it was not created with saveQuregToFile()."; + /* * TEMPORARY ALLOCATIONS @@ -1949,6 +1955,18 @@ void validate_newQuregAllocs(Qureg qureg, const char* caller) { assertAllNodesAgreeThat(mem_isAllocated(qureg.gpuCommBuffer), report::NEW_QUREG_GPU_COMM_BUFFER_ALLOC_FAILED, caller); } +void validate_newQuregFileMatchesPrecision(size_t fileQrealBytes, const char* caller) { + + if (!global_isValidationEnabled) + return; + + tokenSubs vars = { + {"${FILE_BYTES}", (int) fileQrealBytes}, + {"${EXEC_BYTES}", (int) sizeof(qreal)}}; + + assertThat(fileQrealBytes == (int) sizeof(qreal), report::QUREG_FILE_PRECISION_MISMATCH, vars, caller); +} + /* @@ -2003,7 +2021,7 @@ void validate_quregIsDensityMatrix(Qureg qureg, const char* caller) { assertThat(qureg.isDensityMatrix, report::QUREG_NOT_DENSITY_MATRIX, caller); } -void validate_quregCheckpointingIsCompiled(const char* caller) { +void validate_adios2IsCompiled(const char* caller) { if (!global_isValidationEnabled) return; @@ -2020,18 +2038,6 @@ void validate_quregCheckpointingIsCompiled(const char* caller) { assertThat(isCompiled, report::ADIOS2_NOT_COMPILED, caller); } -void validate_quregFileMatchesPrecision(int fileQrealBytes, const char* caller) { - - if (!global_isValidationEnabled) - return; - - tokenSubs vars = { - {"${FILE_BYTES}", fileQrealBytes}, - {"${EXEC_BYTES}", (int) sizeof(qreal)}}; - - assertThat(fileQrealBytes == (int) sizeof(qreal), report::QUREG_FILE_PRECISION_MISMATCH, vars, caller); -} - /* @@ -5082,6 +5088,47 @@ void validate_canReadFile(string fn, const char* caller) { assertThat(parser_canReadFile(fn), report::CANNOT_READ_FILE, caller); } +void validate_adiosCanOpenFile(bool canOpen, string fn, const char* caller) { + + if (!global_isValidationEnabled) + return; + + /// @todo embed filename into error message when tokenSubs is updated to permit strings + (void) fn; + + assertThat(canOpen, report::ADIOS2_CANNOT_OPEN_FILE, caller); +} + +void validate_adiosCanReadFile(bool canRead, string fn, const char* caller) { + + if (!global_isValidationEnabled) + return; + + /// @todo embed filename into error message when tokenSubs is updated to permit strings + (void) fn; + + assertThat(canRead, report::ADIOS2_CANNOT_READ_FILE, caller); +} + +void validate_adiosCanWriteToFile(bool canWrite, string fn, const char* caller) { + + if (!global_isValidationEnabled) + return; + + /// @todo embed filename into error message when tokenSubs is updated to permit strings + (void) fn; + + assertThat(canWrite, report::ADIOS2_CANNOT_WRITE_TO_FILE, caller); +} + +void validate_adiosFileContainsFields(bool areAllVarsPresent, const char* caller) { + + if (!global_isValidationEnabled) + return; + + assertThat(areAllVarsPresent, report::ADIOS2_FILE_INVALID, caller); +} + /* diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index e8eb7306d..522422ae1 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -125,6 +125,8 @@ void validate_newQuregParams(int numQubits, int isDensMatr, int isDistrib, int i void validate_newQuregAllocs(Qureg qureg, const char* caller); +void validate_newQuregFileMatchesPrecision(size_t fileQrealBytes, const char* caller); + /* @@ -137,9 +139,7 @@ void validate_quregIsStateVector(Qureg qureg, const char* caller); void validate_quregIsDensityMatrix(Qureg qureg, const char* caller); -void validate_quregCheckpointingIsCompiled(const char* caller); - -void validate_quregFileMatchesPrecision(int fileQrealBytes, const char* caller); +void validate_adios2IsCompiled(const char* caller); @@ -540,6 +540,14 @@ void validate_quregCanBeSetToReducedDensMatr(Qureg out, Qureg in, int numTraceQu void validate_canReadFile(string fn, const char* caller); +void validate_adiosCanOpenFile(bool canOpen, string fn, const char* caller); + +void validate_adiosCanReadFile(bool canRead, string fn, const char* caller); + +void validate_adiosCanWriteToFile(bool canWrite, string fn, const char* caller); + +void validate_adiosFileContainsFields(bool areAllVarsPresent, const char* caller); + /* diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index cb07e8f4f..15b9af753 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -124,8 +124,12 @@ TEST_CASE( "getQuESTNumGpuThreadsPerBlock", TEST_CATEGORY ) { // TODO: // - fix this guard! Just runtime skip - // - fix tests + // - fix tests; don't use custom comparison, use existing utils + // - negative test of when PRECISION CHANGES + // (can we invoke a QuEST subprocess to WRITE to file?!?! Probs not ) // - extend tests to CHANGE DEPLOYMENT of the Qureg pre and post restoration! + // - note we cannot actually make negative test changes of precision! + // - separate test into two functions, for each API func #ifdef QUEST_COMPILE_ADIOS2 From 8ad17ba286483445f6a84ce451a483f729b21bb2 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 19 Jun 2026 19:58:43 -0400 Subject: [PATCH 15/27] demand saved and new distributions match because the existing implementation was incorrect in the circumstance that a previously non-distributed Qureg was auto-deployed to be distributed. --- quest/include/experimental.h | 27 ++++++++++++------ quest/src/api/experimental.cpp | 51 ++++++++++++++++++++++------------ quest/src/core/validation.cpp | 19 ++++++++++++- quest/src/core/validation.hpp | 2 ++ tests/unit/experimental.cpp | 37 ++++++++++++++++++++---- 5 files changed, 103 insertions(+), 33 deletions(-) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index f5525310c..c7e59ccc0 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -7,6 +7,7 @@ * * @author Oliver Brown * @author Tyson Jones (formatting) + * @author Ashmit JaiSarita Gupta (checkpointing) * * @defgroup experimental Experimental * @ingroup api @@ -104,18 +105,16 @@ int getQuESTNumGpuThreadsPerBlock(); void setQuESTNumGpuThreadsPerBlock(int numThreadsPerBlock); - - // TODO: - // - also link/add to the 'qureg' API module? (Then need to mark this as experimental explicitly?!) - // - add note about file extension???? - - /** Writes the contents of @p qureg to the file (or folder) @p fn, so that it may later be * restored with createQuregFromFile(), potentially in another process. * - * The output records only the @p qureg dimension (number of qubits and whether it is a density matrix), - * the amplitude precision, and the Qureg's full set of amplitudes. Deployment information (such as whether - * the Qureg is distributed, or GPU-accelerated) is not recorded. + * @notyettested + * @notyetvalidated + * + * The output records the @p qureg dimension (number of qubits and whether it is a density matrix), + * the amplitude precision, the Qureg's distribution, and the Qureg's full set of amplitudes. Other + * deployment information, such as whether the Qureg is multithreaded or GPU-accelerated, is not + * recorded. * * There is no particular file extension or folder name suffix required, though since saving is * performed with ADIOS2, a suffix of @p .bp is conventional. @@ -140,8 +139,17 @@ void saveQuregToFile(Qureg qureg, const char* fn); * with automatically chosen deployments (independent of those used when the * file was saved), and populates the Qureg with the saved amplitudes. * + * @notyettested + * @notyetvalidated + * * The chosen deployments are identical to those chosen by createQureg() and createDensityQureg(). * + * > [!NOTE] + * > The number of distributed nodes chosen by the autodeployer must agree with the + * > number of nodes of the originally saved Qureg, else a @validationerror is thrown. Therefore, + * > the number of MPI processes calling these functions cannot be changed between saveQuregToFile() + * > and createQuregFromFile(), unless the Qureg was non-distributed in both settings. + * * > [!IMPORTANT] * > This function is only callable when QuEST is compiled with CMake option QUEST_ENABLE_ADIOS2=1. * @@ -151,6 +159,7 @@ void saveQuregToFile(Qureg qureg, const char* fn); * - if QuEST was not compiled with CMake option QUEST_ENABLE_ADIOS2=1. * - if @p fn cannot be read (since, for example, it does not exist). * - if the precision of the saved Qureg differs from the current QuEST precision. + * - if the number of distributed nodes of the saved Qureg differs from the autodeployer's chosen number. * - if the recorded Qureg dimensions would overflow the @c qindex type. * - if the recorded toatal Qureg memory would overflow the @c size_t type. * - if the system contains insufficient RAM (or VRAM) to store the Qureg in any deployment. diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index ab1e96291..6c1c52f0c 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -5,7 +5,9 @@ * file against MPI, despite being outside of /comm/, * and so require opt-in macros (QUEST_COMPILE_SUBCOMM) * - * @author Oliver Brown + * @author Oliver Brown (custom QuESTEnv) + * @author Ashmit JaiSarita Gupta (checkpointing) + * @author Tyson Jones (structure) */ #include "quest/include/config.h" @@ -70,15 +72,16 @@ extern Qureg validateAndCreateCustomQureg( #if QUEST_COMPILE_ADIOS2 -auto createAdios() { +auto createAdios(bool useMpi) { - // In distributed builds, ADIOS2 must be given QuEST's communicator so that each - // node's call collectively writes/reads its own slice of the shared file. Without - // it, ADIOS2 runs serially per rank and the per-node slices never form one file. + // When the Qureg is distributed, ADIOS2 must be given QuEST's communicator so that each + // node writes/reads its own slice of the shared file #if QUEST_COMPILE_MPI - return adios2::ADIOS(comm_getMpiComm()); + return useMpi? + adios2::ADIOS(comm_getMpiComm()) : + adios2::ADIOS(); #else - return adios2::ADIOS(); + return adios2::ADIOS(); // implies useMpi=0 #endif } #endif @@ -142,19 +145,18 @@ void setQuESTNumGpuThreadsPerBlock(int numTPB) { } - - // TODO: - // - make comment about size_t overflow risk - // - fix Qureg{} return warning issue - // - check restoration to a DISTRIBUTED qureg is correct - - void saveQuregToFile(Qureg qureg, const char* fn) { validate_adios2IsCompiled(__func__); validate_quregFields(qureg, __func__); #ifdef QUEST_COMPILE_ADIOS2 + // when qureg is duplicated in a distributed QuEST env, only root proceeds, + // to avoid ADIOS2 processes racing to file. Note that we cannot prevent the + // race when user's code is distributed but QuEST is not - user must be careful! + if (!qureg.isDistributed && comm_getRank() > ROOT_RANK) + return; + // Pedantic but safe - don't let ADIOS2 start reading amps prematurely if (qureg.isDistributed) comm_sync(); @@ -166,7 +168,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { gpu_copyGpuToCpu(qureg); // gratuitously re-create ADIOS2 at every call, for simplicity (occluded by IO) - adios2::ADIOS adios = createAdios(); + adios2::ADIOS adios = createAdios(qureg.isDistributed); adios2::IO io = adios.DeclareIO("QuESTQuregSave"); // attempt to open the file @@ -181,6 +183,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // and precision, never incidental deployment fields (the loader chooses its // own deployment) nor derivable fields (like numAmps) adios2::Variable vNumQubits = io.DefineVariable("numQubits"); + adios2::Variable vNumNodes = io.DefineVariable("numNodes"); adios2::Variable vIsDensMatr = io.DefineVariable("isDensityMatrix"); adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); // also encodes precision @@ -190,7 +193,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // (these scalars are guaranteed not to overflow by createQureg validation) qindex globalReals = 2 * qureg.numAmps; qindex localReals = 2 * qureg.numAmpsPerNode; - qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; + qindex startReal = localReals * qureg.rank; adios2::Variable vAmpComponents = io.DefineVariable( "ampComponents", { (size_t) globalReals }, @@ -201,6 +204,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { try { engine.BeginStep(); engine.Put(vNumQubits, qureg.numQubits); + engine.Put(vNumNodes, qureg.numNodes); engine.Put(vIsDensMatr, qureg.isDensityMatrix); engine.Put(vQrealBytes, sizeof(qreal)); engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); @@ -220,8 +224,13 @@ Qureg createQuregFromFile(const char* fn) { #ifdef QUEST_COMPILE_ADIOS2 + // make ADIOS2 MPI-aware even when the subsequently-loaded Qureg is + // auto-deployed to be non-distributed; every process will safely + // parse the file and independently update its Qureg copy + bool giveAdiosMpi = comm_isActive(); + // gratuitously re-create ADIOS2 at every call, for simplicity (occluded by IO) - adios2::ADIOS adios = createAdios(); + adios2::ADIOS adios = createAdios(giveAdiosMpi); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); // attempt to open the file, and prepare to parse @@ -235,6 +244,7 @@ Qureg createQuregFromFile(const char* fn) { // check that the file contains the expected variables auto vNumQubits = io.InquireVariable("numQubits"); + auto vNumNodes = io.InquireVariable("numNodes"); auto vIsDensMatr = io.InquireVariable("isDensityMatrix"); auto vQrealBytes = io.InquireVariable("qrealBytes"); auto vAmpComponents = io.InquireVariable("ampComponents"); @@ -243,10 +253,12 @@ Qureg createQuregFromFile(const char* fn) { // read dimension + precision metadata first, so we can size the new Qureg int numQubits = 0; + int numNodes = 0; int isDensMatr = 0; size_t fileQrealBytes = 0; try { engine.Get(vNumQubits, numQubits); + engine.Get(vNumNodes, numNodes); engine.Get(vIsDensMatr, isDensMatr); engine.Get(vQrealBytes, fileQrealBytes); engine.PerformGets(); @@ -261,6 +273,11 @@ Qureg createQuregFromFile(const char* fn) { Qureg qureg = validateAndCreateCustomQureg(numQubits, isDensMatr, modeflag::USE_AUTO, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__); + // auto-distribution MUST match checkpointed distribution (pre-free to avoid leak) + if (qureg.numNodes != numNodes) + destroyQureg(qureg); + validate_newQuregNumNodesMatchesSavedFile(numNodes, qureg.numNodes, comm_getNumNodes(), numQubits, isDensMatr, __func__); + // read only this node's slice of the global amplitude array into its buffer // (guaranteed not to overflow by above validateAndCreateCustomQureg validation) qindex localReals = 2 * qureg.numAmpsPerNode; diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index a4a5f3792..2f2188298 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -281,6 +281,9 @@ namespace report { string QUREG_FILE_PRECISION_MISMATCH = "The checkpoint file was written with a qreal precision of ${FILE_BYTES} bytes, but this QuEST build uses ${EXEC_BYTES} bytes. A Qureg can only be restored by a QuEST build using the same floating-point precision (QUEST_FLOAT_PRECISION) as the build which saved it."; + string QUREG_FILE_NUM_NODES_MISMATCH = + "The autodeployer chose to distribute the ${NUM_QUBITS}-qubit Qureg (isDensityMatrix=${IS_DENS_MATR}) over ${NUM_AUTODEPLOYED_NODES} nodes (of the ${NUM_AVAILABLE_NODES} available to QuEST), but the saved Qureg was distributed over ${NUM_SAVED_NODES} nodes. The distributions must match."; + string ADIOS2_NOT_COMPILED = "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with ADIOS2. Reconfigure with the CMake option -DQUEST_ENABLE_ADIOS2=ON."; @@ -1964,7 +1967,21 @@ void validate_newQuregFileMatchesPrecision(size_t fileQrealBytes, const char* ca {"${FILE_BYTES}", (int) fileQrealBytes}, {"${EXEC_BYTES}", (int) sizeof(qreal)}}; - assertThat(fileQrealBytes == (int) sizeof(qreal), report::QUREG_FILE_PRECISION_MISMATCH, vars, caller); + assertThat(fileQrealBytes == sizeof(qreal), report::QUREG_FILE_PRECISION_MISMATCH, vars, caller); +} + +void validate_newQuregNumNodesMatchesSavedFile(int numSavedNodes, int numAutoDeployedNodes, int numAvailableNodes, int numQubits, bool isDensMatr, const char* caller) { + + if (!global_isValidationEnabled) + return; + + tokenSubs vars = { + {"${NUM_QUBITS}", numQubits}, + {"${IS_DENS_MATR}", isDensMatr}, + {"${NUM_SAVED_NODES}", numSavedNodes}, + {"${NUM_AUTODEPLOYED_NODES}", numAutoDeployedNodes}, + {"${NUM_AVAILABLE_NODES}", numAvailableNodes}}; + assertThat(numSavedNodes == numAutoDeployedNodes, report::QUREG_FILE_NUM_NODES_MISMATCH, vars, caller); } diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 522422ae1..486893740 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -127,6 +127,8 @@ void validate_newQuregAllocs(Qureg qureg, const char* caller); void validate_newQuregFileMatchesPrecision(size_t fileQrealBytes, const char* caller); +void validate_newQuregNumNodesMatchesSavedFile(int numSavedNodes, int numAutoDeployedNodes, int numAvailableNodes, int numQubits, bool isDensMatr, const char* caller); + /* diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 15b9af753..529095c14 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -174,19 +174,39 @@ namespace { TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { + // TODO / DEBUG / BEWARE! + // These tests are insufficient! They only ever test createQuregFromFile() + // (and ergo validate saveQuregToFile() worked properly) for non-distributed + // Quregs! This is because createQuregFromFile() uses the distribution of the + // autodeployer, which for our tiny unit-test Quregs, will always default to + // non-distributed. Distributed Qureg restoration is totally untested! + SECTION( LABEL_CORRECTNESS ) { - // iterate the cached Quregs so the save path is exercised under every - // deployment combination (serial, OMP, MPI, GPU and their mixtures); - // each restored Qureg chooses its own deployment independently + // We will iterate the cached Quregs so the save path is exercised under every + // deployment combination (serial, OMP, MPI, GPU and their mixtures). However, + // the restored Qureg uses a distribution chosen by the auto-deployer, which is + // not permitted to differ from the checkpointed distribution; we skip those! + Qureg svDummy = createQureg(getNumCachedQubits()); + Qureg dmDummy = createDensityQureg(getNumCachedQubits()); + int legalSvNumNodes = svDummy.numNodes; + int legalDmNumNodes = dmDummy.numNodes; + destroyQureg(svDummy); + destroyQureg(dmDummy); + SECTION( LABEL_STATEVEC ) { for (auto& [label, q] : getCachedStatevecs()) { DYNAMIC_SECTION( label ) { + // always test writing succeeds initRandomPureState(q); + REQUIRE_NOTHROW( saveQuregToFile(q, SV_FILE) ); + + // skip restoration when new Qureg distribution would disagree with old + if (q.numNodes != legalSvNumNodes) + continue; - saveQuregToFile(q, SV_FILE); Qureg r = createQuregFromFile(SV_FILE); CHECK( r.numQubits == q.numQubits ); @@ -204,9 +224,14 @@ TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { for (auto& [label, q] : getCachedDensmatrs()) { DYNAMIC_SECTION( label ) { - initRandomPureState(q); // works even for density matrices + // always test writing succeeds + initRandomMixedState(q, /*numPureStates=*/10); + REQUIRE_NOTHROW( saveQuregToFile(q, DM_FILE) ); + + // skip cached quregs with illegal distributions + if (q.numNodes != legalDmNumNodes) + continue; - saveQuregToFile(q, DM_FILE); Qureg r = createQuregFromFile(DM_FILE); CHECK( r.numQubits == q.numQubits ); From e12a9bd113e2d657a4813ef92ecc398d2077d711 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Fri, 19 Jun 2026 21:02:48 -0400 Subject: [PATCH 16/27] add saveQuregToFile test --- quest/include/experimental.h | 6 +++ tests/unit/experimental.cpp | 75 ++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index c7e59ccc0..548756803 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -177,6 +177,12 @@ Qureg createQuregFromFile(const char* fn); } #endif + + +// TODO: C++ only (accepts std::string) + + + #endif // EXPERIMENTAL_H /** @} */ // (end file-wide doxygen defgroup) diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 529095c14..6cea6a621 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -18,6 +18,8 @@ #include "tests/utils/config.hpp" #include "tests/utils/cache.hpp" +#include + using Catch::Matchers::ContainsSubstring; @@ -26,10 +28,23 @@ using Catch::Matchers::ContainsSubstring; * UTILITIES */ + #define TEST_CATEGORY \ LABEL_UNIT_TAG "[experimental]" +void TEST_ON_CACHED_QUREGS(quregCache quregs, auto testFunc) { + + for (auto& [label, qureg]: quregs) { + + DYNAMIC_SECTION( label ) { + + testFunc(qureg); + } + } +} + + /** * TESTS @@ -122,6 +137,66 @@ TEST_CASE( "getQuESTNumGpuThreadsPerBlock", TEST_CATEGORY ) { } +TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { + + SECTION( LABEL_CORRECTNESS ) { + + const char* outFn = "test_checkpoint.bp"; + + auto testFunc = [&](Qureg qureg) { + initRandomPureState(qureg); + REQUIRE_NOTHROW( saveQuregToFile(qureg, outFn) ); + + // note that we are NOT validating the contents was correct; + // that will be performed by the createQuregFromFile() test + }; + + // skip correctness tests if ADIOS2 not compiled + SECTION( LABEL_STATEVEC ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedStatevecs(), testFunc); SUCCEED( ); } + SECTION( LABEL_DENSMATR ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedDensmatrs(), testFunc); SUCCEED( ); } + + // single process deletes checkpoint file (assumes a shared filesystem; if not, who cares about the scraps?) + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(outFn); + } + + SECTION( LABEL_VALIDATION ) { + + Qureg qureg = getArbitraryCachedStatevec(); + + SECTION( "adios2 not compiled" ) { + + if (!QUEST_COMPILE_ADIOS2) + REQUIRE_THROWS_WITH( saveQuregToFile(qureg, "dummy.bp"), ContainsSubstring("blah") ); + + SUCCEED( ); + } + + SECTION( "qureg uninitialised" ) { + + if (QUEST_COMPILE_ADIOS2) { + Qureg badQureg; + badQureg.numQubits = -123; + REQUIRE_THROWS_WITH( saveQuregToFile(badQureg, "dummy.bp"), ContainsSubstring("Received an invalid Qureg") ); + } + + SUCCEED( ); + } + + SECTION( "bad name" ) { + + if (QUEST_COMPILE_ADIOS2) { + auto badFn = GENERATE( "" ); // surprisingly hard to find cross-OS illegal names! + REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); + } + + SUCCEED( ); + } + } +} + + // TODO: // - fix this guard! Just runtime skip // - fix tests; don't use custom comparison, use existing utils From 64976db446071d972c30306014869371391df15b Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 01:08:17 -0400 Subject: [PATCH 17/27] workaround ADIOS2 bug --- quest/include/experimental.h | 7 ++- quest/src/api/experimental.cpp | 110 ++++++++++++++++++++++++--------- quest/src/comm/comm_config.cpp | 7 +++ quest/src/comm/comm_config.hpp | 3 + quest/src/core/validation.cpp | 27 +++----- quest/src/core/validation.hpp | 8 +-- 6 files changed, 110 insertions(+), 52 deletions(-) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 548756803..1786b5328 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -144,6 +144,11 @@ void saveQuregToFile(Qureg qureg, const char* fn); * * The chosen deployments are identical to those chosen by createQureg() and createDensityQureg(). * + * > [!CAUTION] + * > Specifying @fn equal to an existing directory or file will cause erasure and overwriting of + * > its contents. It is especially dangerous to pass @fn equal to a system directory, such as + * > @c / on Unix, and may cause system corruption. + * * > [!NOTE] * > The number of distributed nodes chosen by the autodeployer must agree with the * > number of nodes of the originally saved Qureg, else a @validationerror is thrown. Therefore, @@ -161,7 +166,7 @@ void saveQuregToFile(Qureg qureg, const char* fn); * - if the precision of the saved Qureg differs from the current QuEST precision. * - if the number of distributed nodes of the saved Qureg differs from the autodeployer's chosen number. * - if the recorded Qureg dimensions would overflow the @c qindex type. - * - if the recorded toatal Qureg memory would overflow the @c size_t type. + * - if the recorded total Qureg memory would overflow the @c size_t type. * - if the system contains insufficient RAM (or VRAM) to store the Qureg in any deployment. * - if any Qureg memory allocation unexpectedly fails. * @see diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index 6c1c52f0c..6a8f6bf24 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -27,7 +27,7 @@ #include #endif -#ifdef QUEST_COMPILE_ADIOS2 +#if QUEST_COMPILE_ADIOS2 #include #if QUEST_COMPILE_MPI @@ -87,6 +87,38 @@ auto createAdios(bool useMpi) { #endif +// Temp workaround ADIOS2 foot-gun +#include +#include +void DEBUG_ungracefullyExitMpiAwareAdios2(const std::exception& e) { + + // TODO: + // Surely we can avoid this madness?! (We could prior validate non-distributed? Blegh!) + + // For some ungodly reason, ADIOS2 hangs on non-root processes when throwing an exception + // from the root process; see https://github.com/ornladios/ADIOS2/issues/5098 + // This means that we cannot ever recover from an ADIOS2 error in MPI settings, and must + // ungracefully catastrophically abort MPI. Otherwise, the user will see a hang and no error! + + // When QuEST is not distributed, caller will reach safe/graceful validation + if (!comm_isActive()) + return; + + // By here, every non-root process is hung; so root will print... + std::cout + << "The below ADIOS2 exception occurred, which currently cannot be gracefully handled by QuEST's input validation; " + << "MPI Abort will be called. " + << std::endl + << e.what() + << std::endl; + std::cout.flush(); + + // and all processes will crash! + comm_abort(); + exit(EXIT_FAILURE); +} + + /* * API FUNCTIONS @@ -148,16 +180,18 @@ void setQuESTNumGpuThreadsPerBlock(int numTPB) { void saveQuregToFile(Qureg qureg, const char* fn) { validate_adios2IsCompiled(__func__); validate_quregFields(qureg, __func__); + + (void) fn; // suppress unused warning -#ifdef QUEST_COMPILE_ADIOS2 +#if QUEST_COMPILE_ADIOS2 + + + // TODO: + // need a new way to avoid race when ADIOS2 is saving a duplicated Qureg in a distributed env + // (cannot exit early due to validation syncs) - // when qureg is duplicated in a distributed QuEST env, only root proceeds, - // to avoid ADIOS2 processes racing to file. Note that we cannot prevent the - // race when user's code is distributed but QuEST is not - user must be careful! - if (!qureg.isDistributed && comm_getRank() > ROOT_RANK) - return; - // Pedantic but safe - don't let ADIOS2 start reading amps prematurely + // pedantic but safe - don't let ADIOS2 start reading amps prematurely if (qureg.isDistributed) comm_sync(); @@ -173,11 +207,12 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // attempt to open the file adios2::Engine engine; // default ctor + bool success = false; try { engine = io.Open(fn, adios2::Mode::Write); - } catch (...) { - validate_adiosCanOpenFile(false, fn, __func__); - } + success = true; + } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); // global single-value metadata; we deliberately record only the dimension // and precision, never incidental deployment fields (the loader chooses its @@ -201,6 +236,7 @@ void saveQuregToFile(Qureg qureg, const char* fn) { { (size_t) localReals }); // attempt to write to file + success = false; try { engine.BeginStep(); engine.Put(vNumQubits, qureg.numQubits); @@ -210,10 +246,13 @@ void saveQuregToFile(Qureg qureg, const char* fn) { engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); engine.EndStep(); engine.Close(); - } catch (...) { - // no need for a finally; RAII frees engine - validate_adiosCanWriteToFile(false, fn, __func__); - } + success = true; + } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + validate_adiosCanWriteToFileOnAllNodes(success, fn, __func__); + + // prevent any process from continuing until ADIOS2 is fully finished + if (qureg.isDistributed) + comm_sync(); #endif } @@ -222,7 +261,11 @@ void saveQuregToFile(Qureg qureg, const char* fn) { Qureg createQuregFromFile(const char* fn) { validate_adios2IsCompiled(__func__); -#ifdef QUEST_COMPILE_ADIOS2 +#if QUEST_COMPILE_ADIOS2 + + // pedantic but safe - don't let ADIOS2 start reading while other processes are working + if (comm_isActive()) + comm_sync(); // make ADIOS2 MPI-aware even when the subsequently-loaded Qureg is // auto-deployed to be non-distributed; every process will safely @@ -235,12 +278,13 @@ Qureg createQuregFromFile(const char* fn) { // attempt to open the file, and prepare to parse adios2::Engine engine; // default ctor + bool success = false; try { engine = io.Open(fn, adios2::Mode::Read); engine.BeginStep(); - } catch (...) { - validate_adiosCanOpenFile(false, fn, __func__); - } + success = true; + } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); // check that the file contains the expected variables auto vNumQubits = io.InquireVariable("numQubits"); @@ -249,22 +293,23 @@ Qureg createQuregFromFile(const char* fn) { auto vQrealBytes = io.InquireVariable("qrealBytes"); auto vAmpComponents = io.InquireVariable("ampComponents"); bool areAllVarsPresent = vNumQubits && vIsDensMatr && vQrealBytes && vAmpComponents; - validate_adiosFileContainsFields(areAllVarsPresent, __func__); + validate_adiosFileContainsFieldsOnAllNodes(areAllVarsPresent, __func__); // read dimension + precision metadata first, so we can size the new Qureg int numQubits = 0; int numNodes = 0; int isDensMatr = 0; size_t fileQrealBytes = 0; + success = false; try { engine.Get(vNumQubits, numQubits); engine.Get(vNumNodes, numNodes); engine.Get(vIsDensMatr, isDensMatr); engine.Get(vQrealBytes, fileQrealBytes); engine.PerformGets(); - } catch(...) { - validate_adiosCanReadFile(false, fn, __func__); - } + success = true; + } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + validate_adiosCanReadFileOnAllNodes(success, fn, __func__); // check the amps are of the expected precision, and so are parsable validate_newQuregFileMatchesPrecision(fileQrealBytes, __func__); @@ -273,6 +318,11 @@ Qureg createQuregFromFile(const char* fn) { Qureg qureg = validateAndCreateCustomQureg(numQubits, isDensMatr, modeflag::USE_AUTO, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__); + + // DEBUG + // can manually check this works in distributed by forcing those flags from AUTO above + + // auto-distribution MUST match checkpointed distribution (pre-free to avoid leak) if (qureg.numNodes != numNodes) destroyQureg(qureg); @@ -283,19 +333,21 @@ Qureg createQuregFromFile(const char* fn) { qindex localReals = 2 * qureg.numAmpsPerNode; qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; vAmpComponents.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); + success = false; try { engine.Get(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); // immediate; PerformGets redundant - } catch(...) { - validate_adiosCanReadFile(false, fn, __func__); - } + success = true; + } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + validate_adiosCanReadFileOnAllNodes(success, fn, __func__); // complete ADIOS2 work + success = false; try { engine.EndStep(); engine.Close(); - } catch(...) { - validate_adiosCanReadFile(false, fn, __func__); - } + success = true; + } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + validate_adiosCanReadFileOnAllNodes(success, fn, __func__); // propagate the restored CPU amplitudes to the GPU, if deployed if (qureg.isGpuAccelerated) diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 4b76ca71e..0d039161f 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -217,6 +217,13 @@ bool comm_isMpiUserOwned() { } +void comm_abort() { +#if QUEST_COMPILE_MPI + MPI_Abort(global_mpiComm, 1); // 1 = generic failure +#endif +} + + /* * QUEST COMMUNICATION MANAGEMENT diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index cc009ab9a..a6ccaa8dc 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -19,6 +19,9 @@ bool comm_isMpiGpuAware(); bool comm_isMpiInit(); bool comm_isMpiUserOwned(); +// control of global MPI env (dangerous!!) +void comm_abort(); + // control of QuEST's (possibly more limited) MPI env bool comm_isActive(); void comm_init(bool userOwnsMpi); diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 2f2188298..beeae12f1 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -2043,16 +2043,7 @@ void validate_adios2IsCompiled(const char* caller) { if (!global_isValidationEnabled) return; - // this validation must fire regardless of QUEST_ENABLE_ADIOS2, so the user - // receives a clear error (rather than a linker error) when calling the - // checkpointing API in a build which did not compile it - #ifdef QUEST_COMPILE_ADIOS2 - bool isCompiled = true; - #else - bool isCompiled = false; - #endif - - assertThat(isCompiled, report::ADIOS2_NOT_COMPILED, caller); + assertThat(QUEST_COMPILE_ADIOS2, report::ADIOS2_NOT_COMPILED, caller); } @@ -5105,7 +5096,7 @@ void validate_canReadFile(string fn, const char* caller) { assertThat(parser_canReadFile(fn), report::CANNOT_READ_FILE, caller); } -void validate_adiosCanOpenFile(bool canOpen, string fn, const char* caller) { +void validate_adiosCanOpenFileOnAllNodes(bool canOpenInThisNode, string fn, const char* caller) { if (!global_isValidationEnabled) return; @@ -5113,10 +5104,10 @@ void validate_adiosCanOpenFile(bool canOpen, string fn, const char* caller) { /// @todo embed filename into error message when tokenSubs is updated to permit strings (void) fn; - assertThat(canOpen, report::ADIOS2_CANNOT_OPEN_FILE, caller); + assertAllNodesAgreeThat(canOpenInThisNode, report::ADIOS2_CANNOT_OPEN_FILE, caller); } -void validate_adiosCanReadFile(bool canRead, string fn, const char* caller) { +void validate_adiosCanReadFileOnAllNodes(bool canReadInThisNode, string fn, const char* caller) { if (!global_isValidationEnabled) return; @@ -5124,10 +5115,10 @@ void validate_adiosCanReadFile(bool canRead, string fn, const char* caller) { /// @todo embed filename into error message when tokenSubs is updated to permit strings (void) fn; - assertThat(canRead, report::ADIOS2_CANNOT_READ_FILE, caller); + assertAllNodesAgreeThat(canReadInThisNode, report::ADIOS2_CANNOT_READ_FILE, caller); } -void validate_adiosCanWriteToFile(bool canWrite, string fn, const char* caller) { +void validate_adiosCanWriteToFileOnAllNodes(bool canWriteInThisNode, string fn, const char* caller) { if (!global_isValidationEnabled) return; @@ -5135,15 +5126,15 @@ void validate_adiosCanWriteToFile(bool canWrite, string fn, const char* caller) /// @todo embed filename into error message when tokenSubs is updated to permit strings (void) fn; - assertThat(canWrite, report::ADIOS2_CANNOT_WRITE_TO_FILE, caller); + assertAllNodesAgreeThat(canWriteInThisNode, report::ADIOS2_CANNOT_WRITE_TO_FILE, caller); } -void validate_adiosFileContainsFields(bool areAllVarsPresent, const char* caller) { +void validate_adiosFileContainsFieldsOnAllNodes(bool areAllVarsPresentInThisNode, const char* caller) { if (!global_isValidationEnabled) return; - assertThat(areAllVarsPresent, report::ADIOS2_FILE_INVALID, caller); + assertAllNodesAgreeThat(areAllVarsPresentInThisNode, report::ADIOS2_FILE_INVALID, caller); } diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 486893740..64d3541f1 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -542,13 +542,13 @@ void validate_quregCanBeSetToReducedDensMatr(Qureg out, Qureg in, int numTraceQu void validate_canReadFile(string fn, const char* caller); -void validate_adiosCanOpenFile(bool canOpen, string fn, const char* caller); +void validate_adiosCanOpenFileOnAllNodes(bool canOpen, string fn, const char* caller); -void validate_adiosCanReadFile(bool canRead, string fn, const char* caller); +void validate_adiosCanReadFileOnAllNodes(bool canRead, string fn, const char* caller); -void validate_adiosCanWriteToFile(bool canWrite, string fn, const char* caller); +void validate_adiosCanWriteToFileOnAllNodes(bool canWrite, string fn, const char* caller); -void validate_adiosFileContainsFields(bool areAllVarsPresent, const char* caller); +void validate_adiosFileContainsFieldsOnAllNodes(bool areAllVarsPresent, const char* caller); From 72ec86a3f8cd49a8d886fe5b0a0430feffa805f1 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 01:09:07 -0400 Subject: [PATCH 18/27] add createQuregFromFile test --- tests/unit/experimental.cpp | 200 ++++++++++++++++-------------------- 1 file changed, 90 insertions(+), 110 deletions(-) diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 6cea6a621..1187e9cdd 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -17,6 +17,7 @@ #include "tests/utils/macros.hpp" #include "tests/utils/config.hpp" #include "tests/utils/cache.hpp" +#include "tests/utils/compare.hpp" #include @@ -155,10 +156,13 @@ TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { SECTION( LABEL_STATEVEC ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedStatevecs(), testFunc); SUCCEED( ); } SECTION( LABEL_DENSMATR ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedDensmatrs(), testFunc); SUCCEED( ); } - // single process deletes checkpoint file (assumes a shared filesystem; if not, who cares about the scraps?) + // Single process deletes checkpoint file (assumes a shared filesystem; if not, who cares about the scraps?) + // Note these syncs are ESSENTIAL for correct behaviour, else root can begin deletion while a subsequent node + // proceeds to the below validation and re-creates some files within the same direc, causing MPI hangs. Ouch! syncQuESTEnv(); if (getQuESTEnv().rank == 0) std::filesystem::remove_all(outFn); + syncQuESTEnv(); } SECTION( LABEL_VALIDATION ) { @@ -168,7 +172,7 @@ TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { SECTION( "adios2 not compiled" ) { if (!QUEST_COMPILE_ADIOS2) - REQUIRE_THROWS_WITH( saveQuregToFile(qureg, "dummy.bp"), ContainsSubstring("blah") ); + REQUIRE_THROWS_WITH( saveQuregToFile(qureg, "dummy.bp"), ContainsSubstring("compiled with ADIOS2") ); SUCCEED( ); } @@ -186,10 +190,20 @@ TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { SECTION( "bad name" ) { - if (QUEST_COMPILE_ADIOS2) { - auto badFn = GENERATE( "" ); // surprisingly hard to find cross-OS illegal names! - REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); - } + // TODO: + // This negative test currently hangs execution, because it relies + // upon an ADIOS2-invoked exception, which causes non-root nodes to + // hang! See https://github.com/ornladios/ADIOS2/issues/5098 + + // NOTE: + // Actually, this particular exception DID NOT cause non-root nodes + // to hang! But our hotfix (to ungracefully exit when ADIOS2 errors) + // breaks this validation, so it must be skippec + + // if (QUEST_COMPILE_ADIOS2) { + // auto badFn = GENERATE( "" ); // surprisingly hard to find cross-OS illegal names! + // REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); + // } SUCCEED( ); } @@ -197,142 +211,108 @@ TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { } - // TODO: - // - fix this guard! Just runtime skip - // - fix tests; don't use custom comparison, use existing utils - // - negative test of when PRECISION CHANGES - // (can we invoke a QuEST subprocess to WRITE to file?!?! Probs not ) - // - extend tests to CHANGE DEPLOYMENT of the Qureg pre and post restoration! - // - note we cannot actually make negative test changes of precision! - // - separate test into two functions, for each API func - +TEST_CASE( "createQuregFromFile", TEST_CATEGORY ) { + + SECTION( LABEL_CORRECTNESS ) { + + const char* checkpointFn = "test_checkpoint.bp"; -#ifdef QUEST_COMPILE_ADIOS2 + // We will iterate the cached Quregs so the save path is exercised under every + // deployment combination (serial, OMP, MPI, GPU and their mixtures). However, + // the restored Qureg uses a distribution chosen by the auto-deployer, which is + // not permitted to differ from the checkpointed distribution. We know, given + // the unit test Quregs are so small, that distribution is NEVER automatically + // enabled; so we will forbid testing with distributed Quregs + int legalNumNodes = 1; -#include -#include -#include -#include -#include + auto testFunc = [&](Qureg qureg) { -namespace { + initRandomPureState(qureg); + REQUIRE_NOTHROW( saveQuregToFile(qureg, checkpointFn) ); - const char* SV_FILE = "test_checkpoint_statevector.bp"; - const char* DM_FILE = "test_checkpoint_densitymatrix.bp"; + // skip restoration when new Qureg distribution would disagree with old + if (qureg.numNodes != legalNumNodes) + return; - qreal maxStatevectorAmpDiff(Qureg a, Qureg b) { - qreal m = 0; - for (qindex i = 0; i < a.numAmps; i++) - m = std::max(m, std::abs(getQuregAmp(a, i) - getQuregAmp(b, i))); - return m; - } + Qureg newQureg = createQuregFromFile(checkpointFn); + REQUIRE_AGREE(qureg, newQureg); - qreal maxDensityMatrixAmpDiff(Qureg a, Qureg b) { - qreal m = 0; - qindex dim = (qindex) 1 << a.numQubits; - for (qindex r = 0; r < dim; r++) - for (qindex c = 0; c < dim; c++) - m = std::max(m, std::abs(getDensityQuregAmp(a, r, c) - getDensityQuregAmp(b, r, c))); - return m; - } + destroyQureg(newQureg); + }; + + // skip correctness tests if ADIOS2 not compiled + SECTION( LABEL_STATEVEC ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedStatevecs(), testFunc); SUCCEED( ); } + SECTION( LABEL_DENSMATR ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedDensmatrs(), testFunc); SUCCEED( ); } - // distributed-safe cleanup: a barrier guarantees every node has finished - // reading the shared file, only rank 0 deletes it (concurrent removal races), - // and a second barrier stops the next write racing a half-removed directory. - void removeCheckpointFile(const char* fn) { + CAPTURE( checkpointFn ); + + // Single process deletes checkpoint file (assumes a shared filesystem; if not, who cares about the scraps?). + // Note these syncs are ESSENTIAL for correct behaviour, else root can begin deletion while a subsequent node + // proceeds to the below validation and re-creates some files within the same direc, causing MPI hangs. Ouch! syncQuESTEnv(); if (getQuESTEnv().rank == 0) - std::filesystem::remove_all(fn); + std::filesystem::remove_all(checkpointFn); syncQuESTEnv(); } -} - -TEST_CASE( "saveQuregToFile and createQuregFromFile", TEST_CATEGORY ) { - // TODO / DEBUG / BEWARE! - // These tests are insufficient! They only ever test createQuregFromFile() - // (and ergo validate saveQuregToFile() worked properly) for non-distributed - // Quregs! This is because createQuregFromFile() uses the distribution of the - // autodeployer, which for our tiny unit-test Quregs, will always default to - // non-distributed. Distributed Qureg restoration is totally untested! - - SECTION( LABEL_CORRECTNESS ) { - - // We will iterate the cached Quregs so the save path is exercised under every - // deployment combination (serial, OMP, MPI, GPU and their mixtures). However, - // the restored Qureg uses a distribution chosen by the auto-deployer, which is - // not permitted to differ from the checkpointed distribution; we skip those! - Qureg svDummy = createQureg(getNumCachedQubits()); - Qureg dmDummy = createDensityQureg(getNumCachedQubits()); - int legalSvNumNodes = svDummy.numNodes; - int legalDmNumNodes = dmDummy.numNodes; - destroyQureg(svDummy); - destroyQureg(dmDummy); + SECTION( LABEL_VALIDATION ) { - SECTION( LABEL_STATEVEC ) { + SECTION( "adios2 not compiled" ) { - for (auto& [label, q] : getCachedStatevecs()) { - DYNAMIC_SECTION( label ) { + if (!QUEST_COMPILE_ADIOS2) + REQUIRE_THROWS_WITH( createQuregFromFile("dummy.bp"), ContainsSubstring("compiled with ADIOS2") ); - // always test writing succeeds - initRandomPureState(q); - REQUIRE_NOTHROW( saveQuregToFile(q, SV_FILE) ); + SUCCEED( ); + } - // skip restoration when new Qureg distribution would disagree with old - if (q.numNodes != legalSvNumNodes) - continue; + SECTION( "bad name" ) { - Qureg r = createQuregFromFile(SV_FILE); + // TODO: + // This negative test currently hangs execution, because it relies + // upon an ADIOS2-invoked exception, which causes non-root nodes to + // hang! See https://github.com/ornladios/ADIOS2/issues/5098 - CHECK( r.numQubits == q.numQubits ); - CHECK( r.isDensityMatrix == q.isDensityMatrix ); - CHECK( maxStatevectorAmpDiff(q, r) < 1e-12 ); + // if (QUEST_COMPILE_ADIOS2) + // REQUIRE_THROWS_WITH( createQuregFromFile("BAD_FILENAME"), ContainsSubstring("could not be opened") ); - destroyQureg(r); - removeCheckpointFile(SV_FILE); - } - } + SUCCEED( ); } - SECTION( LABEL_DENSMATR ) { - - for (auto& [label, q] : getCachedDensmatrs()) { - DYNAMIC_SECTION( label ) { + SECTION( "differing distributions" ) { - // always test writing succeeds - initRandomMixedState(q, /*numPureStates=*/10); - REQUIRE_NOTHROW( saveQuregToFile(q, DM_FILE) ); + // Distributions can only differ when QuEST is distributed over more than 1 node + if (QUEST_COMPILE_ADIOS2 && getQuESTEnv().numNodes > 1) { - // skip cached quregs with illegal distributions - if (q.numNodes != legalDmNumNodes) - continue; + // Create a new distributed qureg; we know createQuregFromFile() will create + // non-distributed, since unit-test-size Quregs auto-deploy to non-distributed + Qureg quregDistrib = createCustomQureg(getNumCachedQubits(), 0, /*useDistrib=*/1, 0, 0); - Qureg r = createQuregFromFile(DM_FILE); + CAPTURE( quregDistrib.numNodes ); - CHECK( r.numQubits == q.numQubits ); - CHECK( r.isDensityMatrix == q.isDensityMatrix ); - CHECK( maxDensityMatrixAmpDiff(q, r) < 1e-12 ); + // Write qureg to file, then deliberately fail to restore it + const char* fn = "test_checkpoint.nb"; + saveQuregToFile(quregDistrib, fn); + REQUIRE_THROWS_WITH( createQuregFromFile(fn), ContainsSubstring("distributions must match") ); - destroyQureg(r); - removeCheckpointFile(DM_FILE); - } + // cleanup + destroyQureg(quregDistrib); + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(fn); + syncQuESTEnv(); } - } - } - SECTION( LABEL_VALIDATION ) { + SUCCEED( ); + } - // The only checkpointing-specific validation - calling the API when QuEST - // was compiled without checkpointing - is unreachable here, since this - // file only compiles under QUEST_COMPILE_ADIOS2. ADIOS2's own - // runtime errors (e.g. a missing file) are not QuEST validation errors. - SUCCEED( ); + // We do not presently test the below validations, since it will require + // externally generating and saving ADIOS2 files; quite a pain! + // SECTION( "differing precision" ) { } + // SECTION( "overflow" ) { } + // SECTION( "insufficient RAM" ) { } } } -#endif // QUEST_COMPILE_ADIOS2 - - /** @} (end defgroup) */ From c3bd8a56da1dc9ffe84785ad6d20980672bfe946 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 11:29:54 -0400 Subject: [PATCH 19/27] avoid ADIOS2 hang thanks to help from @eisenhauer in https://github.com/ornladios/ADIOS2/issues/5098 --- quest/src/api/experimental.cpp | 58 ++++++++-------------------------- tests/unit/experimental.cpp | 29 ++++------------- 2 files changed, 21 insertions(+), 66 deletions(-) diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index 6a8f6bf24..e0066da6e 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -87,38 +87,6 @@ auto createAdios(bool useMpi) { #endif -// Temp workaround ADIOS2 foot-gun -#include -#include -void DEBUG_ungracefullyExitMpiAwareAdios2(const std::exception& e) { - - // TODO: - // Surely we can avoid this madness?! (We could prior validate non-distributed? Blegh!) - - // For some ungodly reason, ADIOS2 hangs on non-root processes when throwing an exception - // from the root process; see https://github.com/ornladios/ADIOS2/issues/5098 - // This means that we cannot ever recover from an ADIOS2 error in MPI settings, and must - // ungracefully catastrophically abort MPI. Otherwise, the user will see a hang and no error! - - // When QuEST is not distributed, caller will reach safe/graceful validation - if (!comm_isActive()) - return; - - // By here, every non-root process is hung; so root will print... - std::cout - << "The below ADIOS2 exception occurred, which currently cannot be gracefully handled by QuEST's input validation; " - << "MPI Abort will be called. " - << std::endl - << e.what() - << std::endl; - std::cout.flush(); - - // and all processes will crash! - comm_abort(); - exit(EXIT_FAILURE); -} - - /* * API FUNCTIONS @@ -205,13 +173,16 @@ void saveQuregToFile(Qureg qureg, const char* fn) { adios2::ADIOS adios = createAdios(qureg.isDistributed); adios2::IO io = adios.DeclareIO("QuESTQuregSave"); + // use BP5 specifically to avoid non-root-hangs upon rank exceptions + io.SetEngine("BP5"); + // attempt to open the file adios2::Engine engine; // default ctor bool success = false; try { engine = io.Open(fn, adios2::Mode::Write); success = true; - } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + } catch (...) {} validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); // global single-value metadata; we deliberately record only the dimension @@ -238,16 +209,14 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // attempt to write to file success = false; try { - engine.BeginStep(); engine.Put(vNumQubits, qureg.numQubits); engine.Put(vNumNodes, qureg.numNodes); engine.Put(vIsDensMatr, qureg.isDensityMatrix); engine.Put(vQrealBytes, sizeof(qreal)); engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); - engine.EndStep(); engine.Close(); success = true; - } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + } catch (...) {} validate_adiosCanWriteToFileOnAllNodes(success, fn, __func__); // prevent any process from continuing until ADIOS2 is fully finished @@ -276,14 +245,16 @@ Qureg createQuregFromFile(const char* fn) { adios2::ADIOS adios = createAdios(giveAdiosMpi); adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); + // use BP5 specifically to avoid non-root-hangs upon rank exceptions + io.SetEngine("BP5"); + // attempt to open the file, and prepare to parse adios2::Engine engine; // default ctor bool success = false; try { - engine = io.Open(fn, adios2::Mode::Read); - engine.BeginStep(); + engine = io.Open(fn, adios2::Mode::ReadRandomAccess); success = true; - } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + } catch (...) {} validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); // check that the file contains the expected variables @@ -292,7 +263,7 @@ Qureg createQuregFromFile(const char* fn) { auto vIsDensMatr = io.InquireVariable("isDensityMatrix"); auto vQrealBytes = io.InquireVariable("qrealBytes"); auto vAmpComponents = io.InquireVariable("ampComponents"); - bool areAllVarsPresent = vNumQubits && vIsDensMatr && vQrealBytes && vAmpComponents; + bool areAllVarsPresent = vNumQubits && vNumNodes && vIsDensMatr && vQrealBytes && vAmpComponents; validate_adiosFileContainsFieldsOnAllNodes(areAllVarsPresent, __func__); // read dimension + precision metadata first, so we can size the new Qureg @@ -308,7 +279,7 @@ Qureg createQuregFromFile(const char* fn) { engine.Get(vQrealBytes, fileQrealBytes); engine.PerformGets(); success = true; - } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + } catch (...) {} validate_adiosCanReadFileOnAllNodes(success, fn, __func__); // check the amps are of the expected precision, and so are parsable @@ -337,16 +308,15 @@ Qureg createQuregFromFile(const char* fn) { try { engine.Get(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); // immediate; PerformGets redundant success = true; - } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + } catch (...) {} validate_adiosCanReadFileOnAllNodes(success, fn, __func__); // complete ADIOS2 work success = false; try { - engine.EndStep(); engine.Close(); success = true; - } catch (const std::exception& e) { DEBUG_ungracefullyExitMpiAwareAdios2(e); } + } catch (...) {} validate_adiosCanReadFileOnAllNodes(success, fn, __func__); // propagate the restored CPU amplitudes to the GPU, if deployed diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 1187e9cdd..a3dc4d880 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -190,20 +190,10 @@ TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { SECTION( "bad name" ) { - // TODO: - // This negative test currently hangs execution, because it relies - // upon an ADIOS2-invoked exception, which causes non-root nodes to - // hang! See https://github.com/ornladios/ADIOS2/issues/5098 - - // NOTE: - // Actually, this particular exception DID NOT cause non-root nodes - // to hang! But our hotfix (to ungracefully exit when ADIOS2 errors) - // breaks this validation, so it must be skippec - - // if (QUEST_COMPILE_ADIOS2) { - // auto badFn = GENERATE( "" ); // surprisingly hard to find cross-OS illegal names! - // REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); - // } + if (QUEST_COMPILE_ADIOS2) { + auto badFn = GENERATE( "" ); // surprisingly hard to find cross-OS illegal names! + REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); + } SUCCEED( ); } @@ -267,13 +257,8 @@ TEST_CASE( "createQuregFromFile", TEST_CATEGORY ) { SECTION( "bad name" ) { - // TODO: - // This negative test currently hangs execution, because it relies - // upon an ADIOS2-invoked exception, which causes non-root nodes to - // hang! See https://github.com/ornladios/ADIOS2/issues/5098 - - // if (QUEST_COMPILE_ADIOS2) - // REQUIRE_THROWS_WITH( createQuregFromFile("BAD_FILENAME"), ContainsSubstring("could not be opened") ); + if (QUEST_COMPILE_ADIOS2) + REQUIRE_THROWS_WITH( createQuregFromFile("BAD_FILENAME"), ContainsSubstring("could not be opened") ); SUCCEED( ); } @@ -290,7 +275,7 @@ TEST_CASE( "createQuregFromFile", TEST_CATEGORY ) { CAPTURE( quregDistrib.numNodes ); // Write qureg to file, then deliberately fail to restore it - const char* fn = "test_checkpoint.nb"; + const char* fn = "test_checkpoint.bp"; saveQuregToFile(quregDistrib, fn); REQUIRE_THROWS_WITH( createQuregFromFile(fn), ContainsSubstring("distributions must match") ); From 5e571885e44e4b0843e5bedf1e76846cfcc27a69 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 11:55:07 -0400 Subject: [PATCH 20/27] added QUEST_DOWNLOAD_ADIOS2, updated doc --- docs/cmake.md | 2 ++ docs/compile.md | 15 +++++---------- quest/include/experimental.h | 28 +++++++++++----------------- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/docs/cmake.md b/docs/cmake.md index fec90d76a..7f03d1055 100644 --- a/docs/cmake.md +++ b/docs/cmake.md @@ -44,6 +44,8 @@ make | `QUEST_ENABLE_CUDA` | (`OFF`), `ON` | Determines whether QuEST will be built with support for NVIDIA GPU acceleration. If turned on, `CMAKE_CUDA_ARCHITECTURES` should probably also be set. | | `QUEST_ENABLE_CUQUANTUM` | (`OFF`), `ON` | Determines whether QuEST will make use of the NVIDIA CuQuantum library. Cannot be turned on if `QUEST_ENABLE_CUDA` is off. | | `QUEST_ENABLE_HIP` | (`OFF`), `ON` | Determines whether QuEST will be built with support for AMD GPU acceleration. If turned on, `CMAKE_HIP_ARCHITECTURES` should probably also be set. | +| `QUEST_ENABLE_ADIOS2` | (`OFF`), `ON` | Determines whether QuEST will be built with ADIOS2 to enable checkpointing, via functions `saveQuregToFile()` and `createQuregFromFile()`. | +| `QUEST_DOWNLOAD_ADIOS2` | (`ON`), `OFF` | Determines whether to download ADIOS2 from Github, when ADIOS2 is enabled but not found. | | `QUEST_ENABLE_DEPRECATED_API` | (`OFF`), `ON` | Determines whether QuEST will be built with support for the deprecated (v3) API. ***Note**: this will generate compiler warnings and is not supported by MSVC.* | | `QUEST_DISABLE_DEPRECATION_WARNINGS` | (`OFF`), `ON` | Whether to disable the compile-time deprecation warnings when using the deprecated (v3) API. | | `USER_SOURCE_NAMES` | (Undefined), String | The source file for a user program which will be compiled alongside QuEST. `USER_OUTPUT_EXE_NAME` *must* also be defined. | diff --git a/docs/compile.md b/docs/compile.md index 3a40bff2b..af0bc9550 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -45,6 +45,7 @@ Compiling is configured with variables supplied by the [`-D` flag](https://cmake > - cuQuantum > - Distribution > - Multi-GPU +> - Checkpointing > **See also**: > - [`cmake.md`](cmake.md) for the full list of passable compiler variables. @@ -701,14 +702,9 @@ Note that distributed executables are launched in a distinct way to the other de ## Checkpointing +QuEST has optional facilities for _checkpointing_ a `Qureg`; writing its state to a file with [`saveQuregToFile()`](https://quest-kit.github.io/QuEST/group__experimental.html#gaf9a1aec34fdfdb3c650dc60e5a8ac9d9), to be later restored into a new `Qureg` with [`createQuregFromFile()`](https://quest-kit.github.io/QuEST/group__experimental.html#gab1ebe89e2ff15470fa340d4c2ced5703). This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. - TODO: - Update below to mention automatic ADIOS2 download and build - - -QuEST can optionally _checkpoint_ a `Qureg` to disk; writing its state to a file with `saveQuregToFile()`, to later be restored into a new `Qureg` with `createQuregFromFile()`. This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. The file records only the `Qureg` dimension (the number of qubits, and whether it is a density matrix) and its amplitudes; never the incidental deployment configuration. A `Qureg` saved by one deployment (say, distributed over `8` nodes) can therefore be restored by any other (say, a single GPU-accelerated node). - -Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, install ADIOS2 and specify `QUEST_ENABLE_ADIOS2` at configuration: +Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, simply specify `QUEST_ENABLE_ADIOS2` at configuration: ```bash # configure cmake .. -D QUEST_ENABLE_ADIOS2=ON @@ -717,10 +713,9 @@ cmake .. -D QUEST_ENABLE_ADIOS2=ON cmake --build . --parallel ``` -> [!IMPORTANT] -> ADIOS2 must be discoverable by CMake. If it was installed to a non-standard location (such as `~/.local`), pass its prefix via `CMAKE_PREFIX_PATH`: +If a compatible ADIOS2 is not found, it will be automatically downloaded and installed from the ADIOS2 [Github](https://github.com/ornladios/ADIOS2), unless `QUEST_DOWNLOAD_ADIOS2` is overridden to be `OFF`. If an existing ADIOS2 is installed in a non-standard location (such as `~/.local`), pass its prefix via [`CMAKE_PREFIX_PATH`](https://cmake.org/cmake/help/latest/variable/CMAKE_PREFIX_PATH.html): > ```bash > cmake .. -D QUEST_ENABLE_ADIOS2=ON -D CMAKE_PREFIX_PATH=$HOME/.local > ``` -Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled throws a validation error. +Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled will trigger a runtime validation error. diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 1786b5328..9efd91ac1 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -108,25 +108,27 @@ void setQuESTNumGpuThreadsPerBlock(int numThreadsPerBlock); /** Writes the contents of @p qureg to the file (or folder) @p fn, so that it may later be * restored with createQuregFromFile(), potentially in another process. * - * @notyettested - * @notyetvalidated - * * The output records the @p qureg dimension (number of qubits and whether it is a density matrix), * the amplitude precision, the Qureg's distribution, and the Qureg's full set of amplitudes. Other * deployment information, such as whether the Qureg is multithreaded or GPU-accelerated, is not * recorded. * * There is no particular file extension or folder name suffix required, though since saving is - * performed with ADIOS2, a suffix of @p .bp is conventional. + * performed with ADIOS2, a suffix of `.bp` is conventional. + * + * > [!CAUTION] + * > Specifying @p fn equal to an existing directory or file will cause erasure and overwriting of + * > its contents. It is especially dangerous to pass @p fn equal to a system directory, such as + * > @c / on Unix, and may cause system corruption. * * > [!IMPORTANT] - * > This function is only callable when QuEST is compiled with CMake option QUEST_ENABLE_ADIOS2=1. + * > This function is only callable when QuEST is compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. * * @param[in] qureg the Qureg to write to disk. * @param[in] fn the output file (or folder) path. * @throws @validationerror * - if @p qureg is uninitialised. - * - if QuEST was not compiled with CMake option QUEST_ENABLE_ADIOS2=1. + * - if QuEST was not compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. * - if opening or writing to @p fn fails. * @see * - createQuregFromFile() to restore a Qureg saved by this function. @@ -139,16 +141,8 @@ void saveQuregToFile(Qureg qureg, const char* fn); * with automatically chosen deployments (independent of those used when the * file was saved), and populates the Qureg with the saved amplitudes. * - * @notyettested - * @notyetvalidated - * * The chosen deployments are identical to those chosen by createQureg() and createDensityQureg(). * - * > [!CAUTION] - * > Specifying @fn equal to an existing directory or file will cause erasure and overwriting of - * > its contents. It is especially dangerous to pass @fn equal to a system directory, such as - * > @c / on Unix, and may cause system corruption. - * * > [!NOTE] * > The number of distributed nodes chosen by the autodeployer must agree with the * > number of nodes of the originally saved Qureg, else a @validationerror is thrown. Therefore, @@ -156,12 +150,12 @@ void saveQuregToFile(Qureg qureg, const char* fn); * > and createQuregFromFile(), unless the Qureg was non-distributed in both settings. * * > [!IMPORTANT] - * > This function is only callable when QuEST is compiled with CMake option QUEST_ENABLE_ADIOS2=1. + * > This function is only callable when QuEST is compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. * * @param[in] fn the file (or folder) path previously created by saveQuregToFile(). * @returns A new Qureg instance matching the saved dimension and amplitudes. * @throws @validationerror - * - if QuEST was not compiled with CMake option QUEST_ENABLE_ADIOS2=1. + * - if QuEST was not compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. * - if @p fn cannot be read (since, for example, it does not exist). * - if the precision of the saved Qureg differs from the current QuEST precision. * - if the number of distributed nodes of the saved Qureg differs from the autodeployer's chosen number. @@ -172,7 +166,7 @@ void saveQuregToFile(Qureg qureg, const char* fn); * @see * - saveQuregToFile() to create a file readable by this function. * @author Ashmit JaiSarita Gupta - * @author Tyson Jones (validation) + * @author Tyson Jones (input validation) */ Qureg createQuregFromFile(const char* fn); From 72e6d178f1f8a5230f0d7255ceca41617b26ffef Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 12:06:22 -0400 Subject: [PATCH 21/27] add C++ std::string overloads --- quest/include/experimental.h | 22 +++++++++++++++++++++- quest/src/api/experimental.cpp | 22 +++++++++++++++++++++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 9efd91ac1..8d7a8eb28 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -30,6 +30,11 @@ #include "quest/include/qureg.h" +// C++ gets string overloads +#ifdef __cplusplus + #include +#endif + // enable invocation by both C and C++ binaries #ifdef __cplusplus @@ -177,9 +182,24 @@ Qureg createQuregFromFile(const char* fn); #endif +/** + * @notyetdoced + * @cpponly + * + * @see + * - saveQuregToFile() + */ +void saveQuregToFile(Qureg qureg, std::string); -// TODO: C++ only (accepts std::string) +/** + * @notyetdoced + * @cpponly + * + * @see + * - createQuregFromFile() + */ +Qureg createQuregFromFile(std::string fn); #endif // EXPERIMENTAL_H diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index e0066da6e..9c2aafcb0 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -19,6 +19,8 @@ #include "quest/src/comm/comm_config.hpp" #include "quest/src/gpu/gpu_config.hpp" +#include + #if QUEST_COMPILE_SUBCOMM && ! QUEST_COMPILE_MPI #error "Macro QUEST_COMPILE_SUBCOMM was true, but QUEST_COMPILE_MPI was illegally false." #endif @@ -74,6 +76,9 @@ extern Qureg validateAndCreateCustomQureg( #if QUEST_COMPILE_ADIOS2 auto createAdios(bool useMpi) { + // suppress unused warning when MPI not compiled (implies useMpi=false) + (void) useMpi; + // When the Qureg is distributed, ADIOS2 must be given QuEST's communicator so that each // node writes/reads its own slice of the shared file #if QUEST_COMPILE_MPI @@ -89,7 +94,7 @@ auto createAdios(bool useMpi) { /* - * API FUNCTIONS + * C API FUNCTIONS */ @@ -333,3 +338,18 @@ Qureg createQuregFromFile(const char* fn) { // end de-mangler } + + +/* + * C++ API FUNCTIONS + */ + +void saveQuregToFile(Qureg qureg, std::string fn) { + + saveQuregToFile(qureg, fn.c_str()); +} + +Qureg createQuregFromFile(std::string fn) { + + return createQuregFromFile(fn.c_str()); +} From 21871055a410a87d5d43dca0d36698b470ceffdf Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 12:22:45 -0400 Subject: [PATCH 22/27] addressed TODOs --- CMakeLists.txt | 8 +++++++- quest/src/api/experimental.cpp | 36 +++++++++++++++++----------------- quest/src/comm/comm_config.cpp | 7 ------- quest/src/comm/comm_config.hpp | 3 --- 4 files changed, 25 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 54f6994e6..ac3c000cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -206,6 +206,12 @@ option( ) message(STATUS "ADIOS2 integration is turned ${QUEST_ENABLE_ADIOS2}. Set QUEST_ENABLE_ADIOS2 to modify.") +option( + QUEST_DOWNLOAD_ADIOS2 + "Whether ADIOS2 will be downloaded if it is enabled but not found. Turned ON by default." + ON +) + # Deprecated API option( @@ -565,7 +571,7 @@ if (QUEST_ENABLE_ADIOS2) set(quest_use_found_adios2 FALSE) endif() - if(NOT quest_use_found_adios2) + if(NOT quest_use_found_adios2 AND QUEST_DOWNLOAD_ADIOS2) message(STATUS "fetching ADIOS2 via FetchContent") include(FetchContent) diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index 9c2aafcb0..bb260f17c 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -7,7 +7,7 @@ * * @author Oliver Brown (custom QuESTEnv) * @author Ashmit JaiSarita Gupta (checkpointing) - * @author Tyson Jones (structure) + * @author Tyson Jones (structure, validation) */ #include "quest/include/config.h" @@ -88,6 +88,8 @@ auto createAdios(bool useMpi) { #else return adios2::ADIOS(); // implies useMpi=0 #endif + + // caller need not call destructor; ADIOS2 uses RAII } #endif @@ -158,11 +160,11 @@ void saveQuregToFile(Qureg qureg, const char* fn) { #if QUEST_COMPILE_ADIOS2 - - // TODO: - // need a new way to avoid race when ADIOS2 is saving a duplicated Qureg in a distributed env - // (cannot exit early due to validation syncs) - + // When the QuEST env is distributed, but the given Qureg is not (and is instead + // duplicated upon every node), we should permit only a single process (the root) + // to use ADIOS2 to write to the (assumably, shared) filesystem. Note that non-root + // nodes must not exit; they need to participate in validation syncs + bool shouldSkipAdios = (! qureg.isDistributed) && (comm_getRank() > ROOT_RANK); // pedantic but safe - don't let ADIOS2 start reading amps prematurely if (qureg.isDistributed) @@ -185,7 +187,8 @@ void saveQuregToFile(Qureg qureg, const char* fn) { adios2::Engine engine; // default ctor bool success = false; try { - engine = io.Open(fn, adios2::Mode::Write); + if (!shouldSkipAdios) + engine = io.Open(fn, adios2::Mode::Write); success = true; } catch (...) {} validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); @@ -214,12 +217,14 @@ void saveQuregToFile(Qureg qureg, const char* fn) { // attempt to write to file success = false; try { - engine.Put(vNumQubits, qureg.numQubits); - engine.Put(vNumNodes, qureg.numNodes); - engine.Put(vIsDensMatr, qureg.isDensityMatrix); - engine.Put(vQrealBytes, sizeof(qreal)); - engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); - engine.Close(); + if (!shouldSkipAdios) { + engine.Put(vNumQubits, qureg.numQubits); + engine.Put(vNumNodes, qureg.numNodes); + engine.Put(vIsDensMatr, qureg.isDensityMatrix); + engine.Put(vQrealBytes, sizeof(qreal)); + engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); + engine.Close(); + } success = true; } catch (...) {} validate_adiosCanWriteToFileOnAllNodes(success, fn, __func__); @@ -294,11 +299,6 @@ Qureg createQuregFromFile(const char* fn) { Qureg qureg = validateAndCreateCustomQureg(numQubits, isDensMatr, modeflag::USE_AUTO, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__); - - // DEBUG - // can manually check this works in distributed by forcing those flags from AUTO above - - // auto-distribution MUST match checkpointed distribution (pre-free to avoid leak) if (qureg.numNodes != numNodes) destroyQureg(qureg); diff --git a/quest/src/comm/comm_config.cpp b/quest/src/comm/comm_config.cpp index 0d039161f..4b76ca71e 100644 --- a/quest/src/comm/comm_config.cpp +++ b/quest/src/comm/comm_config.cpp @@ -217,13 +217,6 @@ bool comm_isMpiUserOwned() { } -void comm_abort() { -#if QUEST_COMPILE_MPI - MPI_Abort(global_mpiComm, 1); // 1 = generic failure -#endif -} - - /* * QUEST COMMUNICATION MANAGEMENT diff --git a/quest/src/comm/comm_config.hpp b/quest/src/comm/comm_config.hpp index a6ccaa8dc..cc009ab9a 100644 --- a/quest/src/comm/comm_config.hpp +++ b/quest/src/comm/comm_config.hpp @@ -19,9 +19,6 @@ bool comm_isMpiGpuAware(); bool comm_isMpiInit(); bool comm_isMpiUserOwned(); -// control of global MPI env (dangerous!!) -void comm_abort(); - // control of QuEST's (possibly more limited) MPI env bool comm_isActive(); void comm_init(bool userOwnsMpi); From 8b670c45cfe4cd2b02b55c4ef2b6d49e948aae75 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 12:28:17 -0400 Subject: [PATCH 23/27] adjusting doc --- .github/workflows/compile.yml | 2 +- .github/workflows/test_free.yml | 2 +- quest/src/api/qureg.cpp | 1 - tests/unit/experimental.cpp | 1 + 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 9587de067..be25b6547 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -258,7 +258,7 @@ jobs: -DCMAKE_CXX_FLAGS=${{ matrix.mpi == 'ON' && matrix.cuda == 'ON' && '-fno-lto' || '' }} # force 'Release' build (needed by MSVC to enable optimisations), - # temporarily forcing serial compilation to avoid ADIOS2 OOM error + # and force serial compilation to avoid ADIOS2 OOM error - name: Compile run: cmake --build ${{ env.build_dir }} --config Release --parallel 1 diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index cdd06ecfd..90c022958 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -59,7 +59,7 @@ jobs: - name: Get QuEST uses: actions/checkout@main - # compile serial unit tests, optionally include deprecated test + # compile serial unit tests, optionally include deprecated test, always including ADIOS2 - name: Configure CMake run: > cmake -B ${{ env.build_dir }} diff --git a/quest/src/api/qureg.cpp b/quest/src/api/qureg.cpp index 83565d51f..84bcd2bd0 100644 --- a/quest/src/api/qureg.cpp +++ b/quest/src/api/qureg.cpp @@ -5,7 +5,6 @@ * @author Tyson Jones */ -#include "quest/include/config.h" #include "quest/include/qureg.h" #include "quest/include/modes.h" #include "quest/include/environment.h" diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index a3dc4d880..18b1b5430 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -3,6 +3,7 @@ * * @author Oliver Brown * @author Tyson Jones + * @author Ashmit JaiSarita Gupta (checkpoint test prototype) * * @defgroup unitexperi Experimental * @ingroup unittests From 099f78bc6103316e58f6adc59821512368890ea9 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 12:52:37 -0400 Subject: [PATCH 24/27] added missing C++ guards --- quest/include/experimental.h | 39 +++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 8d7a8eb28..f9ab1312c 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -182,24 +182,31 @@ Qureg createQuregFromFile(const char* fn); #endif -/** - * @notyetdoced - * @cpponly - * - * @see - * - saveQuregToFile() - */ -void saveQuregToFile(Qureg qureg, std::string); +#if defined(__cplusplus) -/** - * @notyetdoced - * @cpponly - * - * @see - * - createQuregFromFile() - */ -Qureg createQuregFromFile(std::string fn); + + /** + * @notyetdoced + * @cpponly + * + * @see + * - saveQuregToFile() + */ + void saveQuregToFile(Qureg qureg, std::string); + + + /** + * @notyetdoced + * @cpponly + * + * @see + * - createQuregFromFile() + */ + Qureg createQuregFromFile(std::string fn); + + +#endif // __cplusplus #endif // EXPERIMENTAL_H From e0c47562f7748d8db1d57e2bc2dc8a0faf0adabe Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 13:00:06 -0400 Subject: [PATCH 25/27] temporarily duplicate James' CI patch --- .github/workflows/compile.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index be25b6547..696398046 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -61,7 +61,7 @@ jobs: # compile QuEST with all combinations of below flags matrix: - os: [windows-latest, ubuntu-latest, macos-latest] + os: [windows-2022, ubuntu-latest, macos-latest] precision: [1, 2, 4] omp: [ON, OFF] mpi: [ON, OFF] @@ -82,7 +82,7 @@ jobs: - os: macos-latest compiler: clang++ deprecated: ON - - os: windows-latest + - os: windows-2022 compiler: cl deprecated: OFF @@ -109,7 +109,7 @@ jobs: # cannot use cuquantum on Windows or MacOS - cuquantum: ON - os: windows-latest + os: windows-2022 - cuquantum: ON os: macos-latest @@ -132,14 +132,14 @@ jobs: mpilib: 'msmpi' # MacOS: [MPICH, OpenMPI] - os: macos-latest mpilib: 'impi' - - os: windows-latest + - os: windows-2022 mpilib: 'mpich' # Windows: [Intel MPI, MS MPI] - - os: windows-latest + - os: windows-2022 mpilib: 'ompi' # cannot presently install HIP on Windows CI (times out) - hip: ON - os: windows-latest + os: windows-2022 # cannot presently compile HIP + MPI; the linker fails with # "undefined reference to 'vtable for thrust::system::system_error' @@ -265,7 +265,7 @@ jobs: # run all compiled isolated examples to test for link-time errors, # continuing if any fail (since some deliberately fail) - name: Run isolated examples (Windows) - if: ${{ matrix.os == 'windows-latest' }} + if: ${{ matrix.os == 'windows-2022' }} working-directory: ${{ env.isolated_dir }}/Release/ shell: pwsh run: | @@ -275,7 +275,7 @@ jobs: & $_.FullName } - name: Run isolated examples (Unix) - if: ${{ matrix.os != 'windows-latest' }} + if: ${{ matrix.os != 'windows-2022' }} working-directory: ${{ env.isolated_dir }} run: | for fn in *_c *_cpp; do @@ -285,7 +285,7 @@ jobs: # run all compiled 'automated' examples - name: Run automated examples (Windows) - if: ${{ matrix.os == 'windows-latest' }} + if: ${{ matrix.os == 'windows-2022' }} working-directory: ${{ env.automated_dir }}/Release/ shell: pwsh run: | @@ -295,7 +295,7 @@ jobs: & $_.FullName } - name: Run automated examples (Unix) - if: ${{ matrix.os != 'windows-latest' }} + if: ${{ matrix.os != 'windows-2022' }} working-directory: ${{ env.automated_dir }} run: | for fn in *_c *_cpp; do From f3f17593193dc412017fbc3ea97826c6b075dbbf Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Sat, 20 Jun 2026 22:13:34 -0400 Subject: [PATCH 26/27] patch Windows saveQuregToFile validation test --- tests/unit/experimental.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 18b1b5430..df4ebbaa8 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -192,7 +192,12 @@ TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { SECTION( "bad name" ) { if (QUEST_COMPILE_ADIOS2) { - auto badFn = GENERATE( "" ); // surprisingly hard to find cross-OS illegal names! + // surprisingly hard to find cross-OS illegal names! + #if defined(_MSC_VER) + auto badFn = GENERATE( ":", "?", "*" ); + #else + auto badFn = GENERATE( "", "\0" ); + #endif REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); } From 69a1176505e217c308f6ac26fabb1c348a0a29a0 Mon Sep 17 00:00:00 2001 From: Tyson Jones Date: Mon, 22 Jun 2026 02:18:00 -0400 Subject: [PATCH 27/27] revert duplicated CI patch --- .github/workflows/compile.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 696398046..be25b6547 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -61,7 +61,7 @@ jobs: # compile QuEST with all combinations of below flags matrix: - os: [windows-2022, ubuntu-latest, macos-latest] + os: [windows-latest, ubuntu-latest, macos-latest] precision: [1, 2, 4] omp: [ON, OFF] mpi: [ON, OFF] @@ -82,7 +82,7 @@ jobs: - os: macos-latest compiler: clang++ deprecated: ON - - os: windows-2022 + - os: windows-latest compiler: cl deprecated: OFF @@ -109,7 +109,7 @@ jobs: # cannot use cuquantum on Windows or MacOS - cuquantum: ON - os: windows-2022 + os: windows-latest - cuquantum: ON os: macos-latest @@ -132,14 +132,14 @@ jobs: mpilib: 'msmpi' # MacOS: [MPICH, OpenMPI] - os: macos-latest mpilib: 'impi' - - os: windows-2022 + - os: windows-latest mpilib: 'mpich' # Windows: [Intel MPI, MS MPI] - - os: windows-2022 + - os: windows-latest mpilib: 'ompi' # cannot presently install HIP on Windows CI (times out) - hip: ON - os: windows-2022 + os: windows-latest # cannot presently compile HIP + MPI; the linker fails with # "undefined reference to 'vtable for thrust::system::system_error' @@ -265,7 +265,7 @@ jobs: # run all compiled isolated examples to test for link-time errors, # continuing if any fail (since some deliberately fail) - name: Run isolated examples (Windows) - if: ${{ matrix.os == 'windows-2022' }} + if: ${{ matrix.os == 'windows-latest' }} working-directory: ${{ env.isolated_dir }}/Release/ shell: pwsh run: | @@ -275,7 +275,7 @@ jobs: & $_.FullName } - name: Run isolated examples (Unix) - if: ${{ matrix.os != 'windows-2022' }} + if: ${{ matrix.os != 'windows-latest' }} working-directory: ${{ env.isolated_dir }} run: | for fn in *_c *_cpp; do @@ -285,7 +285,7 @@ jobs: # run all compiled 'automated' examples - name: Run automated examples (Windows) - if: ${{ matrix.os == 'windows-2022' }} + if: ${{ matrix.os == 'windows-latest' }} working-directory: ${{ env.automated_dir }}/Release/ shell: pwsh run: | @@ -295,7 +295,7 @@ jobs: & $_.FullName } - name: Run automated examples (Unix) - if: ${{ matrix.os != 'windows-2022' }} + if: ${{ matrix.os != 'windows-latest' }} working-directory: ${{ env.automated_dir }} run: | for fn in *_c *_cpp; do