diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index c86de84f1..be25b6547 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -47,6 +47,7 @@ jobs: ${{ matrix.cuda == 'ON' && 'CUDA' || '' }} ${{ matrix.hip == 'ON' && 'HIP' || '' }} ${{ matrix.cuquantum == 'ON' && 'CUQ' || '' }} + ${{ matrix.adios2 == 'ON' && 'CKPT' || '' }} runs-on: ${{ matrix.os }} @@ -67,6 +68,7 @@ jobs: cuda: [ON, OFF] hip: [ON, OFF] cuquantum: [ON, OFF] + adios2: [ON, OFF] mpilib: ['', 'mpich', 'ompi', 'impi', 'msmpi'] # disable deprecated API on MSVC, and assign unique compilers, @@ -249,14 +251,16 @@ jobs: -DQUEST_ENABLE_CUDA=${{ matrix.cuda }} -DQUEST_ENABLE_HIP=${{ matrix.hip }} -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }} + -DQUEST_ENABLE_ADIOS2=${{ matrix.adios2 }} -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }} -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} -DCMAKE_CXX_FLAGS=${{ matrix.mpi == 'ON' && matrix.cuda == 'ON' && '-fno-lto' || '' }} - # force 'Release' build (needed by MSVC to enable optimisations) + # force 'Release' build (needed by MSVC to enable optimisations), + # and force serial compilation to avoid ADIOS2 OOM error - name: Compile - run: cmake --build ${{ env.build_dir }} --config Release --parallel + run: cmake --build ${{ env.build_dir }} --config Release --parallel 1 # run all compiled isolated examples to test for link-time errors, # continuing if any fail (since some deliberately fail) diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index 2d332e842..90c022958 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -59,7 +59,7 @@ jobs: - name: Get QuEST uses: actions/checkout@main - # compile serial unit tests, optionally include deprecated test + # compile serial unit tests, optionally include deprecated test, always including ADIOS2 - name: Configure CMake run: > cmake -B ${{ env.build_dir }} @@ -68,16 +68,18 @@ jobs: -DQUEST_ENABLE_DEPRECATED_API=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_FLOAT_PRECISION=${{ matrix.precision }} + -DQUEST_ENABLE_ADIOS2=ON - # force 'Release' build (needed by MSVC to enable optimisations) + # force 'Release' build (needed by MSVC to enable optimisations), and force serial (to avoid ADIOS2 OOM) - name: Compile - run: cmake --build ${{ env.build_dir }} --config Release --parallel + run: cmake --build ${{ env.build_dir }} --config Release --parallel 1 # run v4 unit tests in random order, excluding the integration tests, # using the default environment variables (e.g. test all permutations) # TODO: # ctest currently doesn't know of our Catch2 tags, so we # are manually excluding each integration test by name + - name: Run v4 tests if: ${{ matrix.version == 4 }} run: ctest -j2 --output-on-failure --schedule-random -C Release -E "density evolution" diff --git a/CMakeLists.txt b/CMakeLists.txt index b5a438713..ac3c000cb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -187,7 +187,6 @@ message(STATUS "AMD GPU acceleration is turned ${QUEST_ENABLE_HIP}. Set QUEST_EN # GPU Performance Tuning # (We do not print this value when configuring CMake as it is for advanced users only) - set(quest_tpb_description # (the games we play for multi-line set() strings!) "The default number of threads per block QuEST will use when offloading to a GPU. Set to 128 by default. " "Must be a multiple of 32 (on NVIDIA GPUs) or 64 (on AMD GPUs). Can be overridden at executable launch " @@ -199,6 +198,21 @@ set(QUEST_DEFAULT_NUM_GPU_THREADS_PER_BLOCK 128 mark_as_advanced(QUEST_DEFAULT_NUM_GPU_THREADS_PER_BLOCK) +# Checkpointing with ADIOS2 +option( + QUEST_ENABLE_ADIOS2 + "Whether QuEST will be built with ADIOS2, enabling checkpointing (via saveQuregToFile / createQuregFromFile). Turned OFF by default." + OFF +) +message(STATUS "ADIOS2 integration is turned ${QUEST_ENABLE_ADIOS2}. Set QUEST_ENABLE_ADIOS2 to modify.") + +option( + QUEST_DOWNLOAD_ADIOS2 + "Whether ADIOS2 will be downloaded if it is enabled but not found. Turned ON by default." + ON +) + + # Deprecated API option( QUEST_ENABLE_DEPRECATED_API @@ -543,6 +557,81 @@ endif() +# Checkpointing (ADIOS2) +if (QUEST_ENABLE_ADIOS2) + + find_package(adios2 QUIET) + + # A distributed QuEST needs an MPI-enabled ADIOS2 (which provides the + # adios2::cxx_mpi target). A serial system install lacks it, so in that case we + # ignore the found package and fetch an MPI-enabled build instead of failing. + set(quest_use_found_adios2 ${adios2_FOUND}) + if (adios2_FOUND AND QUEST_ENABLE_MPI AND NOT TARGET adios2::cxx_mpi) + message(STATUS "Found ADIOS2 lacks MPI support (no adios2::cxx_mpi target); fetching an MPI-enabled build instead") + set(quest_use_found_adios2 FALSE) + endif() + + if(NOT quest_use_found_adios2 AND QUEST_DOWNLOAD_ADIOS2) + message(STATUS "fetching ADIOS2 via FetchContent") + + include(FetchContent) + FetchContent_Declare( + adios2 + GIT_REPOSITORY https://github.com/ornladios/ADIOS2.git + GIT_TAG v2.12.1 + ) + + # Match ADIOS2's MPI to QuEST's so distributed runs write per-rank slices + # into one shared file. ADIOS2's CUDA support is deliberately left OFF: + # checkpointing copies amps to host memory (syncQuregFromGpu/syncQuregToGpu) + # before any I/O, so ADIOS2 never touches device pointers. Building it with + # CUDA is unnecessary and stalls the Windows CUDA CI job. + set(ADIOS2_USE_MPI ${QUEST_ENABLE_MPI} CACHE BOOL "" FORCE) + set(ADIOS2_USE_CUDA OFF CACHE BOOL "" FORCE) + + # Forego unused facilities + set(ADIOS2_BUILD_TESTING OFF CACHE BOOL "" FORCE) + set(ADIOS2_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SODIUM OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Fortran OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_HDF5 OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_ZeroMQ OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SST OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_DataMan OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SSC OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_MHS OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_DAOS OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_MGARD OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_BZip2 OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Blosc OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Blosc2 OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_SZ OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_ZFP OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_PNG OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Profiling OFF CACHE BOOL "" FORCE) + set(ADIOS2_USE_Python OFF CACHE BOOL "" FORCE) + + FetchContent_MakeAvailable(adios2) + + else() + # re-run non-QUIET so configuration fails with a clear error if the package + # somehow became unavailable between the two calls + find_package(adios2 REQUIRED) + endif() + + # In distributed builds link ADIOS2's MPI-enabled C++ interface: it defines + # ADIOS2_USE_MPI, which exposes the adios2::ADIOS(MPI_Comm) constructor used in + # qureg.cpp for collective per-rank I/O. The serial target lacks it. + if (QUEST_ENABLE_MPI) + target_link_libraries(QuEST PRIVATE adios2::cxx_mpi) + else() + target_link_libraries(QuEST PRIVATE adios2::cxx) + endif() + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH ON) +endif() + + + # =============================== # Set options to save in config.h # =============================== @@ -553,6 +642,7 @@ set(QUEST_COMPILE_OMP ${QUEST_ENABLE_OMP}) set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI}) set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM}) set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM}) +set(QUEST_COMPILE_ADIOS2 ${QUEST_ENABLE_ADIOS2}) set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API}) diff --git a/docs/cmake.md b/docs/cmake.md index fec90d76a..7f03d1055 100644 --- a/docs/cmake.md +++ b/docs/cmake.md @@ -44,6 +44,8 @@ make | `QUEST_ENABLE_CUDA` | (`OFF`), `ON` | Determines whether QuEST will be built with support for NVIDIA GPU acceleration. If turned on, `CMAKE_CUDA_ARCHITECTURES` should probably also be set. | | `QUEST_ENABLE_CUQUANTUM` | (`OFF`), `ON` | Determines whether QuEST will make use of the NVIDIA CuQuantum library. Cannot be turned on if `QUEST_ENABLE_CUDA` is off. | | `QUEST_ENABLE_HIP` | (`OFF`), `ON` | Determines whether QuEST will be built with support for AMD GPU acceleration. If turned on, `CMAKE_HIP_ARCHITECTURES` should probably also be set. | +| `QUEST_ENABLE_ADIOS2` | (`OFF`), `ON` | Determines whether QuEST will be built with ADIOS2 to enable checkpointing, via functions `saveQuregToFile()` and `createQuregFromFile()`. | +| `QUEST_DOWNLOAD_ADIOS2` | (`ON`), `OFF` | Determines whether to download ADIOS2 from Github, when ADIOS2 is enabled but not found. | | `QUEST_ENABLE_DEPRECATED_API` | (`OFF`), `ON` | Determines whether QuEST will be built with support for the deprecated (v3) API. ***Note**: this will generate compiler warnings and is not supported by MSVC.* | | `QUEST_DISABLE_DEPRECATION_WARNINGS` | (`OFF`), `ON` | Whether to disable the compile-time deprecation warnings when using the deprecated (v3) API. | | `USER_SOURCE_NAMES` | (Undefined), String | The source file for a user program which will be compiled alongside QuEST. `USER_OUTPUT_EXE_NAME` *must* also be defined. | diff --git a/docs/compile.md b/docs/compile.md index ba4306a85..af0bc9550 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -45,6 +45,7 @@ Compiling is configured with variables supplied by the [`-D` flag](https://cmake > - cuQuantum > - Distribution > - Multi-GPU +> - Checkpointing > **See also**: > - [`cmake.md`](cmake.md) for the full list of passable compiler variables. @@ -689,3 +690,32 @@ Note that distributed executables are launched in a distinct way to the other de > - UCX > - launch flags > - checking via reportenv + + + + +------------------ + + + + + +## Checkpointing + +QuEST has optional facilities for _checkpointing_ a `Qureg`; writing its state to a file with [`saveQuregToFile()`](https://quest-kit.github.io/QuEST/group__experimental.html#gaf9a1aec34fdfdb3c650dc60e5a8ac9d9), to be later restored into a new `Qureg` with [`createQuregFromFile()`](https://quest-kit.github.io/QuEST/group__experimental.html#gab1ebe89e2ff15470fa340d4c2ced5703). This is useful for long-running jobs which risk timeout or failure - an evolving `Qureg` can be periodically saved and resumed in a subsequent process. + +Checkpointing is built upon [ADIOS2](https://github.com/ornladios/ADIOS2) and is _disabled_ by default. To enable it, simply specify `QUEST_ENABLE_ADIOS2` at configuration: +```bash +# configure +cmake .. -D QUEST_ENABLE_ADIOS2=ON + +# build +cmake --build . --parallel +``` + +If a compatible ADIOS2 is not found, it will be automatically downloaded and installed from the ADIOS2 [Github](https://github.com/ornladios/ADIOS2), unless `QUEST_DOWNLOAD_ADIOS2` is overridden to be `OFF`. If an existing ADIOS2 is installed in a non-standard location (such as `~/.local`), pass its prefix via [`CMAKE_PREFIX_PATH`](https://cmake.org/cmake/help/latest/variable/CMAKE_PREFIX_PATH.html): +> ```bash +> cmake .. -D QUEST_ENABLE_ADIOS2=ON -D CMAKE_PREFIX_PATH=$HOME/.local +> ``` + +Calling `saveQuregToFile()` or `createQuregFromFile()` in a build _without_ checkpointing enabled will trigger a runtime validation error. diff --git a/quest/include/config.h.in b/quest/include/config.h.in index 1bb8a0470..d89df4bfc 100644 --- a/quest/include/config.h.in +++ b/quest/include/config.h.in @@ -41,6 +41,7 @@ defined(QUEST_COMPILE_CUDA) || \ defined(QUEST_COMPILE_HIP) || \ defined(QUEST_COMPILE_CUQUANTUM) || \ + defined(QUEST_COMPILE_ADIOS2) || \ defined(QUEST_ENABLE_NUMA) || \ defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -84,6 +85,7 @@ #cmakedefine01 QUEST_COMPILE_CUDA #cmakedefine01 QUEST_COMPILE_CUQUANTUM #cmakedefine01 QUEST_COMPILE_HIP +#cmakedefine01 QUEST_COMPILE_ADIOS2 // crucial to QuEST source (informs optional NUMA usage) @@ -125,6 +127,7 @@ ! defined(QUEST_COMPILE_CUDA) || \ ! defined(QUEST_COMPILE_HIP) || \ ! defined(QUEST_COMPILE_CUQUANTUM) || \ + ! defined(QUEST_COMPILE_ADIOS2) || \ ! defined(QUEST_ENABLE_NUMA) || \ ! defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ ! defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -152,6 +155,7 @@ ! (QUEST_COMPILE_CUDA == 0 || QUEST_COMPILE_CUDA == 1) || \ ! (QUEST_COMPILE_HIP == 0 || QUEST_COMPILE_HIP == 1) || \ ! (QUEST_COMPILE_CUQUANTUM == 0 || QUEST_COMPILE_CUQUANTUM == 1) || \ + ! (QUEST_COMPILE_ADIOS2 == 0 || QUEST_COMPILE_ADIOS2 == 1) || \ ! (QUEST_ENABLE_NUMA == 0 || QUEST_ENABLE_NUMA == 1) || \ ! (QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 0 || QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 1) || \ ! (QUEST_DISABLE_DEPRECATION_WARNINGS == 0 || QUEST_DISABLE_DEPRECATION_WARNINGS == 1) diff --git a/quest/include/experimental.h b/quest/include/experimental.h index 8c2cc4e0a..f9ab1312c 100644 --- a/quest/include/experimental.h +++ b/quest/include/experimental.h @@ -7,6 +7,7 @@ * * @author Oliver Brown * @author Tyson Jones (formatting) + * @author Ashmit JaiSarita Gupta (checkpointing) * * @defgroup experimental Experimental * @ingroup api @@ -27,6 +28,14 @@ #include #endif +#include "quest/include/qureg.h" + +// C++ gets string overloads +#ifdef __cplusplus + #include +#endif + + // enable invocation by both C and C++ binaries #ifdef __cplusplus extern "C" { @@ -54,8 +63,9 @@ void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, in * The user-provided MPI communicator undergoes the same validation procedure as any that QuEST * would use, and so must contain a power-of-2 number of processes. * - * This function is only compiled and exposed when macro QUEST_COMPILE_SUBCOMM is 1, as is - * defined when providing CMake option QUEST_ENABLE_SUBCOMM during building. + * > [!IMPORTANT] + * > This function is only compiled and exposed when macro QUEST_COMPILE_SUBCOMM is 1, as is + * > defined when providing CMake option QUEST_ENABLE_SUBCOMM during building. * * @author Oliver Brown */ @@ -100,11 +110,105 @@ int getQuESTNumGpuThreadsPerBlock(); void setQuESTNumGpuThreadsPerBlock(int numThreadsPerBlock); +/** Writes the contents of @p qureg to the file (or folder) @p fn, so that it may later be + * restored with createQuregFromFile(), potentially in another process. + * + * The output records the @p qureg dimension (number of qubits and whether it is a density matrix), + * the amplitude precision, the Qureg's distribution, and the Qureg's full set of amplitudes. Other + * deployment information, such as whether the Qureg is multithreaded or GPU-accelerated, is not + * recorded. + * + * There is no particular file extension or folder name suffix required, though since saving is + * performed with ADIOS2, a suffix of `.bp` is conventional. + * + * > [!CAUTION] + * > Specifying @p fn equal to an existing directory or file will cause erasure and overwriting of + * > its contents. It is especially dangerous to pass @p fn equal to a system directory, such as + * > @c / on Unix, and may cause system corruption. + * + * > [!IMPORTANT] + * > This function is only callable when QuEST is compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. + * + * @param[in] qureg the Qureg to write to disk. + * @param[in] fn the output file (or folder) path. + * @throws @validationerror + * - if @p qureg is uninitialised. + * - if QuEST was not compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. + * - if opening or writing to @p fn fails. + * @see + * - createQuregFromFile() to restore a Qureg saved by this function. + * @author Ashmit JaiSarita Gupta + */ +void saveQuregToFile(Qureg qureg, const char* fn); + + +/** Creates a new Qureg from a file (or folder) previously created by saveQuregToFile(), + * with automatically chosen deployments (independent of those used when the + * file was saved), and populates the Qureg with the saved amplitudes. + * + * The chosen deployments are identical to those chosen by createQureg() and createDensityQureg(). + * + * > [!NOTE] + * > The number of distributed nodes chosen by the autodeployer must agree with the + * > number of nodes of the originally saved Qureg, else a @validationerror is thrown. Therefore, + * > the number of MPI processes calling these functions cannot be changed between saveQuregToFile() + * > and createQuregFromFile(), unless the Qureg was non-distributed in both settings. + * + * > [!IMPORTANT] + * > This function is only callable when QuEST is compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. + * + * @param[in] fn the file (or folder) path previously created by saveQuregToFile(). + * @returns A new Qureg instance matching the saved dimension and amplitudes. + * @throws @validationerror + * - if QuEST was not compiled with CMake option @c QUEST_ENABLE_ADIOS2=1. + * - if @p fn cannot be read (since, for example, it does not exist). + * - if the precision of the saved Qureg differs from the current QuEST precision. + * - if the number of distributed nodes of the saved Qureg differs from the autodeployer's chosen number. + * - if the recorded Qureg dimensions would overflow the @c qindex type. + * - if the recorded total Qureg memory would overflow the @c size_t type. + * - if the system contains insufficient RAM (or VRAM) to store the Qureg in any deployment. + * - if any Qureg memory allocation unexpectedly fails. + * @see + * - saveQuregToFile() to create a file readable by this function. + * @author Ashmit JaiSarita Gupta + * @author Tyson Jones (input validation) + */ +Qureg createQuregFromFile(const char* fn); + + // end de-mangler #ifdef __cplusplus } #endif + + +#if defined(__cplusplus) + + + /** + * @notyetdoced + * @cpponly + * + * @see + * - saveQuregToFile() + */ + void saveQuregToFile(Qureg qureg, std::string); + + + /** + * @notyetdoced + * @cpponly + * + * @see + * - createQuregFromFile() + */ + Qureg createQuregFromFile(std::string fn); + + +#endif // __cplusplus + + #endif // EXPERIMENTAL_H /** @} */ // (end file-wide doxygen defgroup) diff --git a/quest/include/qureg.h b/quest/include/qureg.h index 4ff4c5627..bd2bc7129 100644 --- a/quest/include/qureg.h +++ b/quest/include/qureg.h @@ -135,6 +135,7 @@ typedef struct { * - createDensityQureg() to create a density matrix which can additionally undergo decoherence. * - createForcedQureg() to create a statevector which is forced to make use of all available deployments. * - createCustomQureg() to explicitly set the used deployments. + * - createQuregFromFile() to create a Qureg from a checkpoint file. * @author Tyson Jones */ Qureg createQureg(int numQubits); @@ -487,7 +488,6 @@ void getDensityQuregAmps(qcomp** outAmps, Qureg qureg, qindex startRow, qindex s /** @} */ - // end de-mangler #ifdef __cplusplus } diff --git a/quest/src/api/environment.cpp b/quest/src/api/environment.cpp index c59334b55..e6fd2903a 100644 --- a/quest/src/api/environment.cpp +++ b/quest/src/api/environment.cpp @@ -5,6 +5,7 @@ * @author Tyson Jones */ +#include "quest/include/config.h" #include "quest/include/environment.h" #include "quest/include/precision.h" #include "quest/include/modes.h" @@ -208,12 +209,13 @@ void printCompilationInfo() { print_table( "compilation", { - {"isOmpCompiled", cpu_isOpenmpCompiled()}, - {"isMpiCompiled", comm_isMpiCompiled()}, - {"isMpiSubCommCompiled", comm_isMpiSubCommCompiled()}, - {"isGpuCompiled", gpu_isGpuCompiled()}, - {"isHipCompiled", gpu_isHipCompiled()}, - {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, + {"isOmpCompiled", cpu_isOpenmpCompiled()}, + {"isMpiCompiled", comm_isMpiCompiled()}, + {"isMpiSubCommCompiled", comm_isMpiSubCommCompiled()}, + {"isGpuCompiled", gpu_isGpuCompiled()}, + {"isHipCompiled", gpu_isHipCompiled()}, + {"isCuQuantumCompiled", gpu_isCuQuantumCompiled()}, + {"isCheckpointingCompiled", QUEST_COMPILE_ADIOS2}, }); } diff --git a/quest/src/api/experimental.cpp b/quest/src/api/experimental.cpp index a6f883656..bb260f17c 100644 --- a/quest/src/api/experimental.cpp +++ b/quest/src/api/experimental.cpp @@ -5,16 +5,22 @@ * file against MPI, despite being outside of /comm/, * and so require opt-in macros (QUEST_COMPILE_SUBCOMM) * - * @author Oliver Brown + * @author Oliver Brown (custom QuESTEnv) + * @author Ashmit JaiSarita Gupta (checkpointing) + * @author Tyson Jones (structure, validation) */ #include "quest/include/config.h" #include "quest/include/environment.h" +#include "quest/include/qureg.h" +#include "quest/include/modes.h" #include "quest/src/core/validation.hpp" #include "quest/src/comm/comm_config.hpp" #include "quest/src/gpu/gpu_config.hpp" +#include + #if QUEST_COMPILE_SUBCOMM && ! QUEST_COMPILE_MPI #error "Macro QUEST_COMPILE_SUBCOMM was true, but QUEST_COMPILE_MPI was illegally false." #endif @@ -23,6 +29,14 @@ #include #endif +#if QUEST_COMPILE_ADIOS2 + #include + + #if QUEST_COMPILE_MPI + #include + #endif +#endif + /* @@ -39,14 +53,50 @@ extern void validateAndInitCustomQuESTEnv( int useDistrib, bool userOwnsMpi, int useGpuAccel, int useMultithread, const char* caller); +extern Qureg validateAndCreateCustomQureg( + int numQubits, int isDensMatr, int useDistrib, int useGpuAccel, int useMultithread, const char* caller); + + #if QUEST_COMPILE_SUBCOMM // hide MPI_Comm extern bool comm_setMpiComm(MPI_Comm newComm, bool userOwnsMpi); #endif +#if (QUEST_COMPILE_ADIOS2 && QUEST_COMPILE_MPI) // hide MPI_Comm + extern MPI_Comm comm_getMpiComm(); +#endif + + /* - * API FUNCTIONS + * INTERNAL FUNCTIONS + */ + + +#if QUEST_COMPILE_ADIOS2 +auto createAdios(bool useMpi) { + + // suppress unused warning when MPI not compiled (implies useMpi=false) + (void) useMpi; + + // When the Qureg is distributed, ADIOS2 must be given QuEST's communicator so that each + // node writes/reads its own slice of the shared file + #if QUEST_COMPILE_MPI + return useMpi? + adios2::ADIOS(comm_getMpiComm()) : + adios2::ADIOS(); + #else + return adios2::ADIOS(); // implies useMpi=0 + #endif + + // caller need not call destructor; ADIOS2 uses RAII +} +#endif + + + +/* + * C API FUNCTIONS */ @@ -60,7 +110,6 @@ void initCustomMpiQuESTEnv(int useDistrib, bool userOwnsMpi, int useGpuAccel, in #if QUEST_COMPILE_SUBCOMM // hide MPI_Comm - void initCustomMpiCommQuESTEnv(MPI_Comm userQuestComm, int useGpuAccel, int useMultithread) { // useDistrib and userOwnsMpi are implied by the user of this initialiser @@ -103,5 +152,204 @@ void setQuESTNumGpuThreadsPerBlock(int numTPB) { } +void saveQuregToFile(Qureg qureg, const char* fn) { + validate_adios2IsCompiled(__func__); + validate_quregFields(qureg, __func__); + + (void) fn; // suppress unused warning + +#if QUEST_COMPILE_ADIOS2 + + // When the QuEST env is distributed, but the given Qureg is not (and is instead + // duplicated upon every node), we should permit only a single process (the root) + // to use ADIOS2 to write to the (assumably, shared) filesystem. Note that non-root + // nodes must not exit; they need to participate in validation syncs + bool shouldSkipAdios = (! qureg.isDistributed) && (comm_getRank() > ROOT_RANK); + + // pedantic but safe - don't let ADIOS2 start reading amps prematurely + if (qureg.isDistributed) + comm_sync(); + + // TODO: + // We can optimise in GPU settings by giving ADIOS2 the device memory + // pointers; but for now, we simply stage into CPU memory first + if (qureg.isGpuAccelerated) + gpu_copyGpuToCpu(qureg); + + // gratuitously re-create ADIOS2 at every call, for simplicity (occluded by IO) + adios2::ADIOS adios = createAdios(qureg.isDistributed); + adios2::IO io = adios.DeclareIO("QuESTQuregSave"); + + // use BP5 specifically to avoid non-root-hangs upon rank exceptions + io.SetEngine("BP5"); + + // attempt to open the file + adios2::Engine engine; // default ctor + bool success = false; + try { + if (!shouldSkipAdios) + engine = io.Open(fn, adios2::Mode::Write); + success = true; + } catch (...) {} + validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); + + // global single-value metadata; we deliberately record only the dimension + // and precision, never incidental deployment fields (the loader chooses its + // own deployment) nor derivable fields (like numAmps) + adios2::Variable vNumQubits = io.DefineVariable("numQubits"); + adios2::Variable vNumNodes = io.DefineVariable("numNodes"); + adios2::Variable vIsDensMatr = io.DefineVariable("isDensityMatrix"); + adios2::Variable vQrealBytes = io.DefineVariable("qrealBytes"); // also encodes precision + + // amplitudes are stored as interleaved (real, imag) reals to stay agnostic + // to precision and to ADIOS2's complex-type support; each node writes only + // its local slice into the global array, avoiding excessive memory use + // (these scalars are guaranteed not to overflow by createQureg validation) + qindex globalReals = 2 * qureg.numAmps; + qindex localReals = 2 * qureg.numAmpsPerNode; + qindex startReal = localReals * qureg.rank; + adios2::Variable vAmpComponents = io.DefineVariable( + "ampComponents", + { (size_t) globalReals }, + { (size_t) startReal }, + { (size_t) localReals }); + + // attempt to write to file + success = false; + try { + if (!shouldSkipAdios) { + engine.Put(vNumQubits, qureg.numQubits); + engine.Put(vNumNodes, qureg.numNodes); + engine.Put(vIsDensMatr, qureg.isDensityMatrix); + engine.Put(vQrealBytes, sizeof(qreal)); + engine.Put(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); + engine.Close(); + } + success = true; + } catch (...) {} + validate_adiosCanWriteToFileOnAllNodes(success, fn, __func__); + + // prevent any process from continuing until ADIOS2 is fully finished + if (qureg.isDistributed) + comm_sync(); + +#endif +} + + +Qureg createQuregFromFile(const char* fn) { + validate_adios2IsCompiled(__func__); + +#if QUEST_COMPILE_ADIOS2 + + // pedantic but safe - don't let ADIOS2 start reading while other processes are working + if (comm_isActive()) + comm_sync(); + + // make ADIOS2 MPI-aware even when the subsequently-loaded Qureg is + // auto-deployed to be non-distributed; every process will safely + // parse the file and independently update its Qureg copy + bool giveAdiosMpi = comm_isActive(); + + // gratuitously re-create ADIOS2 at every call, for simplicity (occluded by IO) + adios2::ADIOS adios = createAdios(giveAdiosMpi); + adios2::IO io = adios.DeclareIO("QuESTQuregLoad"); + + // use BP5 specifically to avoid non-root-hangs upon rank exceptions + io.SetEngine("BP5"); + + // attempt to open the file, and prepare to parse + adios2::Engine engine; // default ctor + bool success = false; + try { + engine = io.Open(fn, adios2::Mode::ReadRandomAccess); + success = true; + } catch (...) {} + validate_adiosCanOpenFileOnAllNodes(success, fn, __func__); + + // check that the file contains the expected variables + auto vNumQubits = io.InquireVariable("numQubits"); + auto vNumNodes = io.InquireVariable("numNodes"); + auto vIsDensMatr = io.InquireVariable("isDensityMatrix"); + auto vQrealBytes = io.InquireVariable("qrealBytes"); + auto vAmpComponents = io.InquireVariable("ampComponents"); + bool areAllVarsPresent = vNumQubits && vNumNodes && vIsDensMatr && vQrealBytes && vAmpComponents; + validate_adiosFileContainsFieldsOnAllNodes(areAllVarsPresent, __func__); + + // read dimension + precision metadata first, so we can size the new Qureg + int numQubits = 0; + int numNodes = 0; + int isDensMatr = 0; + size_t fileQrealBytes = 0; + success = false; + try { + engine.Get(vNumQubits, numQubits); + engine.Get(vNumNodes, numNodes); + engine.Get(vIsDensMatr, isDensMatr); + engine.Get(vQrealBytes, fileQrealBytes); + engine.PerformGets(); + success = true; + } catch (...) {} + validate_adiosCanReadFileOnAllNodes(success, fn, __func__); + + // check the amps are of the expected precision, and so are parsable + validate_newQuregFileMatchesPrecision(fileQrealBytes, __func__); + + // attempt to create a matching-dimension Qureg with automatically chosen deployments + Qureg qureg = validateAndCreateCustomQureg(numQubits, isDensMatr, + modeflag::USE_AUTO, modeflag::USE_AUTO, modeflag::USE_AUTO, __func__); + + // auto-distribution MUST match checkpointed distribution (pre-free to avoid leak) + if (qureg.numNodes != numNodes) + destroyQureg(qureg); + validate_newQuregNumNodesMatchesSavedFile(numNodes, qureg.numNodes, comm_getNumNodes(), numQubits, isDensMatr, __func__); + + // read only this node's slice of the global amplitude array into its buffer + // (guaranteed not to overflow by above validateAndCreateCustomQureg validation) + qindex localReals = 2 * qureg.numAmpsPerNode; + qindex startReal = 2 * ((qindex) qureg.rank) * qureg.numAmpsPerNode; + vAmpComponents.SetSelection({ { (size_t) startReal }, { (size_t) localReals } }); + success = false; + try { + engine.Get(vAmpComponents, reinterpret_cast(qureg.cpuAmps)); // immediate; PerformGets redundant + success = true; + } catch (...) {} + validate_adiosCanReadFileOnAllNodes(success, fn, __func__); + + // complete ADIOS2 work + success = false; + try { + engine.Close(); + success = true; + } catch (...) {} + validate_adiosCanReadFileOnAllNodes(success, fn, __func__); + + // propagate the restored CPU amplitudes to the GPU, if deployed + if (qureg.isGpuAccelerated) + gpu_copyCpuToGpu(qureg); + + return qureg; +#else + // unreachable: the validation above always throws in non-checkpointing builds + return Qureg{}; +#endif +} + + // end de-mangler } + + +/* + * C++ API FUNCTIONS + */ + +void saveQuregToFile(Qureg qureg, std::string fn) { + + saveQuregToFile(qureg, fn.c_str()); +} + +Qureg createQuregFromFile(std::string fn) { + + return createQuregFromFile(fn.c_str()); +} diff --git a/quest/src/core/validation.cpp b/quest/src/core/validation.cpp index 62ff93166..beeae12f1 100644 --- a/quest/src/core/validation.cpp +++ b/quest/src/core/validation.cpp @@ -7,6 +7,7 @@ * @author Kshitij Chhabra (patched v3 overflow bug) */ +#include "quest/include/config.h" #include "quest/include/modes.h" #include "quest/include/types.h" #include "quest/include/precision.h" @@ -277,6 +278,15 @@ namespace report { string QUREG_NOT_STATE_VECTOR = "Expected a statevector Qureg but received a density matrix."; + string QUREG_FILE_PRECISION_MISMATCH = + "The checkpoint file was written with a qreal precision of ${FILE_BYTES} bytes, but this QuEST build uses ${EXEC_BYTES} bytes. A Qureg can only be restored by a QuEST build using the same floating-point precision (QUEST_FLOAT_PRECISION) as the build which saved it."; + + string QUREG_FILE_NUM_NODES_MISMATCH = + "The autodeployer chose to distribute the ${NUM_QUBITS}-qubit Qureg (isDensityMatrix=${IS_DENS_MATR}) over ${NUM_AUTODEPLOYED_NODES} nodes (of the ${NUM_AVAILABLE_NODES} available to QuEST), but the saved Qureg was distributed over ${NUM_SAVED_NODES} nodes. The distributions must match."; + + string ADIOS2_NOT_COMPILED = + "Qureg checkpointing (saveQuregToFile and createQuregFromFile) requires QuEST to be compiled with ADIOS2. Reconfigure with the CMake option -DQUEST_ENABLE_ADIOS2=ON."; + /* * MUTABLE OBJECT FLAGS @@ -1148,6 +1158,18 @@ namespace report { string CANNOT_READ_FILE = "Could not load and read the given file. Make sure the file exists and is readable as plaintext."; + string ADIOS2_CANNOT_OPEN_FILE = + "The specified file (or folder) could not be opened by ADIOS2."; + + string ADIOS2_CANNOT_READ_FILE = + "The specified file (or folder) was opened by ADIOS2, but the contents could not be read or loaded."; + + string ADIOS2_CANNOT_WRITE_TO_FILE = + "ADIOS2 failed to write to the specified file (or folder)."; + + string ADIOS2_FILE_INVALID = + "The specified file (or folder) did not contain the expected AIODS2 variables, suggesting it was not created with saveQuregToFile()."; + /* * TEMPORARY ALLOCATIONS @@ -1936,6 +1958,32 @@ void validate_newQuregAllocs(Qureg qureg, const char* caller) { assertAllNodesAgreeThat(mem_isAllocated(qureg.gpuCommBuffer), report::NEW_QUREG_GPU_COMM_BUFFER_ALLOC_FAILED, caller); } +void validate_newQuregFileMatchesPrecision(size_t fileQrealBytes, const char* caller) { + + if (!global_isValidationEnabled) + return; + + tokenSubs vars = { + {"${FILE_BYTES}", (int) fileQrealBytes}, + {"${EXEC_BYTES}", (int) sizeof(qreal)}}; + + assertThat(fileQrealBytes == sizeof(qreal), report::QUREG_FILE_PRECISION_MISMATCH, vars, caller); +} + +void validate_newQuregNumNodesMatchesSavedFile(int numSavedNodes, int numAutoDeployedNodes, int numAvailableNodes, int numQubits, bool isDensMatr, const char* caller) { + + if (!global_isValidationEnabled) + return; + + tokenSubs vars = { + {"${NUM_QUBITS}", numQubits}, + {"${IS_DENS_MATR}", isDensMatr}, + {"${NUM_SAVED_NODES}", numSavedNodes}, + {"${NUM_AUTODEPLOYED_NODES}", numAutoDeployedNodes}, + {"${NUM_AVAILABLE_NODES}", numAvailableNodes}}; + assertThat(numSavedNodes == numAutoDeployedNodes, report::QUREG_FILE_NUM_NODES_MISMATCH, vars, caller); +} + /* @@ -1990,6 +2038,14 @@ void validate_quregIsDensityMatrix(Qureg qureg, const char* caller) { assertThat(qureg.isDensityMatrix, report::QUREG_NOT_DENSITY_MATRIX, caller); } +void validate_adios2IsCompiled(const char* caller) { + + if (!global_isValidationEnabled) + return; + + assertThat(QUEST_COMPILE_ADIOS2, report::ADIOS2_NOT_COMPILED, caller); +} + /* @@ -5040,6 +5096,47 @@ void validate_canReadFile(string fn, const char* caller) { assertThat(parser_canReadFile(fn), report::CANNOT_READ_FILE, caller); } +void validate_adiosCanOpenFileOnAllNodes(bool canOpenInThisNode, string fn, const char* caller) { + + if (!global_isValidationEnabled) + return; + + /// @todo embed filename into error message when tokenSubs is updated to permit strings + (void) fn; + + assertAllNodesAgreeThat(canOpenInThisNode, report::ADIOS2_CANNOT_OPEN_FILE, caller); +} + +void validate_adiosCanReadFileOnAllNodes(bool canReadInThisNode, string fn, const char* caller) { + + if (!global_isValidationEnabled) + return; + + /// @todo embed filename into error message when tokenSubs is updated to permit strings + (void) fn; + + assertAllNodesAgreeThat(canReadInThisNode, report::ADIOS2_CANNOT_READ_FILE, caller); +} + +void validate_adiosCanWriteToFileOnAllNodes(bool canWriteInThisNode, string fn, const char* caller) { + + if (!global_isValidationEnabled) + return; + + /// @todo embed filename into error message when tokenSubs is updated to permit strings + (void) fn; + + assertAllNodesAgreeThat(canWriteInThisNode, report::ADIOS2_CANNOT_WRITE_TO_FILE, caller); +} + +void validate_adiosFileContainsFieldsOnAllNodes(bool areAllVarsPresentInThisNode, const char* caller) { + + if (!global_isValidationEnabled) + return; + + assertAllNodesAgreeThat(areAllVarsPresentInThisNode, report::ADIOS2_FILE_INVALID, caller); +} + /* diff --git a/quest/src/core/validation.hpp b/quest/src/core/validation.hpp index 87f81a0d6..64d3541f1 100644 --- a/quest/src/core/validation.hpp +++ b/quest/src/core/validation.hpp @@ -125,6 +125,10 @@ void validate_newQuregParams(int numQubits, int isDensMatr, int isDistrib, int i void validate_newQuregAllocs(Qureg qureg, const char* caller); +void validate_newQuregFileMatchesPrecision(size_t fileQrealBytes, const char* caller); + +void validate_newQuregNumNodesMatchesSavedFile(int numSavedNodes, int numAutoDeployedNodes, int numAvailableNodes, int numQubits, bool isDensMatr, const char* caller); + /* @@ -137,6 +141,8 @@ void validate_quregIsStateVector(Qureg qureg, const char* caller); void validate_quregIsDensityMatrix(Qureg qureg, const char* caller); +void validate_adios2IsCompiled(const char* caller); + /* @@ -536,6 +542,14 @@ void validate_quregCanBeSetToReducedDensMatr(Qureg out, Qureg in, int numTraceQu void validate_canReadFile(string fn, const char* caller); +void validate_adiosCanOpenFileOnAllNodes(bool canOpen, string fn, const char* caller); + +void validate_adiosCanReadFileOnAllNodes(bool canRead, string fn, const char* caller); + +void validate_adiosCanWriteToFileOnAllNodes(bool canWrite, string fn, const char* caller); + +void validate_adiosFileContainsFieldsOnAllNodes(bool areAllVarsPresent, const char* caller); + /* diff --git a/tests/unit/experimental.cpp b/tests/unit/experimental.cpp index 943645831..df4ebbaa8 100644 --- a/tests/unit/experimental.cpp +++ b/tests/unit/experimental.cpp @@ -3,6 +3,7 @@ * * @author Oliver Brown * @author Tyson Jones + * @author Ashmit JaiSarita Gupta (checkpoint test prototype) * * @defgroup unitexperi Experimental * @ingroup unittests @@ -16,6 +17,10 @@ #include "tests/utils/macros.hpp" #include "tests/utils/config.hpp" +#include "tests/utils/cache.hpp" +#include "tests/utils/compare.hpp" + +#include using Catch::Matchers::ContainsSubstring; @@ -25,10 +30,23 @@ using Catch::Matchers::ContainsSubstring; * UTILITIES */ + #define TEST_CATEGORY \ LABEL_UNIT_TAG "[experimental]" +void TEST_ON_CACHED_QUREGS(quregCache quregs, auto testFunc) { + + for (auto& [label, qureg]: quregs) { + + DYNAMIC_SECTION( label ) { + + testFunc(qureg); + } + } +} + + /** * TESTS @@ -121,6 +139,172 @@ TEST_CASE( "getQuESTNumGpuThreadsPerBlock", TEST_CATEGORY ) { } +TEST_CASE( "saveQuregToFile", TEST_CATEGORY ) { + + SECTION( LABEL_CORRECTNESS ) { + + const char* outFn = "test_checkpoint.bp"; + + auto testFunc = [&](Qureg qureg) { + initRandomPureState(qureg); + REQUIRE_NOTHROW( saveQuregToFile(qureg, outFn) ); + + // note that we are NOT validating the contents was correct; + // that will be performed by the createQuregFromFile() test + }; + + // skip correctness tests if ADIOS2 not compiled + SECTION( LABEL_STATEVEC ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedStatevecs(), testFunc); SUCCEED( ); } + SECTION( LABEL_DENSMATR ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedDensmatrs(), testFunc); SUCCEED( ); } + + // Single process deletes checkpoint file (assumes a shared filesystem; if not, who cares about the scraps?) + // Note these syncs are ESSENTIAL for correct behaviour, else root can begin deletion while a subsequent node + // proceeds to the below validation and re-creates some files within the same direc, causing MPI hangs. Ouch! + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(outFn); + syncQuESTEnv(); + } + + SECTION( LABEL_VALIDATION ) { + + Qureg qureg = getArbitraryCachedStatevec(); + + SECTION( "adios2 not compiled" ) { + + if (!QUEST_COMPILE_ADIOS2) + REQUIRE_THROWS_WITH( saveQuregToFile(qureg, "dummy.bp"), ContainsSubstring("compiled with ADIOS2") ); + + SUCCEED( ); + } + + SECTION( "qureg uninitialised" ) { + + if (QUEST_COMPILE_ADIOS2) { + Qureg badQureg; + badQureg.numQubits = -123; + REQUIRE_THROWS_WITH( saveQuregToFile(badQureg, "dummy.bp"), ContainsSubstring("Received an invalid Qureg") ); + } + + SUCCEED( ); + } + + SECTION( "bad name" ) { + + if (QUEST_COMPILE_ADIOS2) { + // surprisingly hard to find cross-OS illegal names! + #if defined(_MSC_VER) + auto badFn = GENERATE( ":", "?", "*" ); + #else + auto badFn = GENERATE( "", "\0" ); + #endif + REQUIRE_THROWS_WITH( saveQuregToFile(qureg, badFn), ContainsSubstring("could not be opened") ); + } + + SUCCEED( ); + } + } +} + + +TEST_CASE( "createQuregFromFile", TEST_CATEGORY ) { + + SECTION( LABEL_CORRECTNESS ) { + + const char* checkpointFn = "test_checkpoint.bp"; + + // We will iterate the cached Quregs so the save path is exercised under every + // deployment combination (serial, OMP, MPI, GPU and their mixtures). However, + // the restored Qureg uses a distribution chosen by the auto-deployer, which is + // not permitted to differ from the checkpointed distribution. We know, given + // the unit test Quregs are so small, that distribution is NEVER automatically + // enabled; so we will forbid testing with distributed Quregs + int legalNumNodes = 1; + + auto testFunc = [&](Qureg qureg) { + + initRandomPureState(qureg); + REQUIRE_NOTHROW( saveQuregToFile(qureg, checkpointFn) ); + + // skip restoration when new Qureg distribution would disagree with old + if (qureg.numNodes != legalNumNodes) + return; + + Qureg newQureg = createQuregFromFile(checkpointFn); + REQUIRE_AGREE(qureg, newQureg); + + destroyQureg(newQureg); + }; + + // skip correctness tests if ADIOS2 not compiled + SECTION( LABEL_STATEVEC ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedStatevecs(), testFunc); SUCCEED( ); } + SECTION( LABEL_DENSMATR ) { if (QUEST_COMPILE_ADIOS2) TEST_ON_CACHED_QUREGS(getCachedDensmatrs(), testFunc); SUCCEED( ); } + + CAPTURE( checkpointFn ); + + // Single process deletes checkpoint file (assumes a shared filesystem; if not, who cares about the scraps?). + // Note these syncs are ESSENTIAL for correct behaviour, else root can begin deletion while a subsequent node + // proceeds to the below validation and re-creates some files within the same direc, causing MPI hangs. Ouch! + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(checkpointFn); + syncQuESTEnv(); + } + + SECTION( LABEL_VALIDATION ) { + + SECTION( "adios2 not compiled" ) { + + if (!QUEST_COMPILE_ADIOS2) + REQUIRE_THROWS_WITH( createQuregFromFile("dummy.bp"), ContainsSubstring("compiled with ADIOS2") ); + + SUCCEED( ); + } + + SECTION( "bad name" ) { + + if (QUEST_COMPILE_ADIOS2) + REQUIRE_THROWS_WITH( createQuregFromFile("BAD_FILENAME"), ContainsSubstring("could not be opened") ); + + SUCCEED( ); + } + + SECTION( "differing distributions" ) { + + // Distributions can only differ when QuEST is distributed over more than 1 node + if (QUEST_COMPILE_ADIOS2 && getQuESTEnv().numNodes > 1) { + + // Create a new distributed qureg; we know createQuregFromFile() will create + // non-distributed, since unit-test-size Quregs auto-deploy to non-distributed + Qureg quregDistrib = createCustomQureg(getNumCachedQubits(), 0, /*useDistrib=*/1, 0, 0); + + CAPTURE( quregDistrib.numNodes ); + + // Write qureg to file, then deliberately fail to restore it + const char* fn = "test_checkpoint.bp"; + saveQuregToFile(quregDistrib, fn); + REQUIRE_THROWS_WITH( createQuregFromFile(fn), ContainsSubstring("distributions must match") ); + + // cleanup + destroyQureg(quregDistrib); + syncQuESTEnv(); + if (getQuESTEnv().rank == 0) + std::filesystem::remove_all(fn); + syncQuESTEnv(); + } + + SUCCEED( ); + } + + // We do not presently test the below validations, since it will require + // externally generating and saving ADIOS2 files; quite a pain! + // SECTION( "differing precision" ) { } + // SECTION( "overflow" ) { } + // SECTION( "insufficient RAM" ) { } + } +} + + /** @} (end defgroup) */