diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml index 696398046..e277af5a3 100644 --- a/.github/workflows/compile.yml +++ b/.github/workflows/compile.yml @@ -39,7 +39,11 @@ jobs: # test only compilation succeeds (no execution) build-test: name: > - ${{ matrix.os == 'ubuntu-latest' && 'Linux' || matrix.os == 'macos-latest' && 'MacOS' || 'Windows' }} + ${{ startsWith(matrix.os, 'ubuntu' ) && 'Linux' || + startsWith(matrix.os, 'macos' ) && 'MacOS' || + startsWith(matrix.os, 'windows') && 'Windows' || + 'Unknown' }} + ${{ endsWith(matrix.os, 'intel') && '(Intel)' || '' }} [${{ matrix.precision }}] ${{ matrix.omp == 'ON' && 'OMP' || '' }} ${{ matrix.mpi == 'ON' && 'MPI' || '' }} @@ -48,6 +52,7 @@ jobs: ${{ matrix.hip == 'ON' && 'HIP' || '' }} ${{ matrix.cuquantum == 'ON' && 'CUQ' || '' }} ${{ matrix.adios2 == 'ON' && 'CKPT' || '' }} + ${{ matrix.bmi2 == 'ON' && 'BMI' || '' }} runs-on: ${{ matrix.os }} @@ -59,9 +64,10 @@ jobs: # (causes CUDA and MPI installation to fail on Windows) max-parallel: 8 - # compile QuEST with all combinations of below flags + # compile QuEST with all combinations of below flags (Intel runners for BMI2 instrinsics); + # incredibly, this (with exclusions below) achieves 256 combos, which is the Github limit! matrix: - os: [windows-2022, ubuntu-latest, macos-latest] + os: [windows-2022, ubuntu-latest, macos-latest, macos-15-intel, macos-26-intel] precision: [1, 2, 4] omp: [ON, OFF] mpi: [ON, OFF] @@ -69,6 +75,7 @@ jobs: hip: [ON, OFF] cuquantum: [ON, OFF] adios2: [ON, OFF] + bmi2: [ON, OFF] mpilib: ['', 'mpich', 'ompi', 'impi', 'msmpi'] # disable deprecated API on MSVC, and assign unique compilers, @@ -82,6 +89,12 @@ jobs: - os: macos-latest compiler: clang++ deprecated: ON + - os: macos-15-intel + compiler: clang++ + deprecated: ON + - os: macos-26-intel + compiler: clang++ + deprecated: ON - os: windows-2022 compiler: cl deprecated: OFF @@ -104,14 +117,42 @@ jobs: # cannot use GPU on MacOS - cuda: ON os: macos-latest + - cuda: ON + os: macos-15-intel + - cuda: ON + os: macos-26-intel - hip: ON os: macos-latest + - hip: ON + os: macos-15-intel + - hip: ON + os: macos-26-intel # cannot use cuquantum on Windows or MacOS - cuquantum: ON os: windows-2022 - cuquantum: ON os: macos-latest + - cuquantum: ON + os: macos-15-intel + - cuquantum: ON + os: macos-26-intel + + # cannot use BMI2 on non-Intel MacOS + - bmi2: ON + os: macos-latest + + # use ONLY BMI2 on Intel MacOS, just to shrink matrix (Github imposes 256 max) + - bmi2: OFF + os: macos-15-intel + - bmi2: OFF + os: macos-26-intel + + # do not combine BMI2 with MPI or ADIOS2 (they don't interact), just to shrink matrix + - bmi2: ON + mpi: ON + - bmi2: ON + adios2: ON # don't enumerate MPI libraries when not using MPI - mpi: OFF @@ -132,6 +173,14 @@ jobs: mpilib: 'msmpi' # MacOS: [MPICH, OpenMPI] - os: macos-latest mpilib: 'impi' + - os: macos-15-intel + mpilib: 'msmpi' + - os: macos-15-intel + mpilib: 'impi' + - os: macos-26-intel + mpilib: 'msmpi' + - os: macos-26-intel + mpilib: 'impi' - os: windows-2022 mpilib: 'mpich' # Windows: [Intel MPI, MS MPI] - os: windows-2022 @@ -252,6 +301,7 @@ jobs: -DQUEST_ENABLE_HIP=${{ matrix.hip }} -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }} -DQUEST_ENABLE_ADIOS2=${{ matrix.adios2 }} + -DQUEST_ENABLE_BMI2=${{ matrix.bmi2 }} -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }} -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }} -DCMAKE_CXX_COMPILER=${{ matrix.compiler }} diff --git a/.github/workflows/test_free.yml b/.github/workflows/test_free.yml index 90c022958..6e7974338 100644 --- a/.github/workflows/test_free.yml +++ b/.github/workflows/test_free.yml @@ -27,9 +27,14 @@ jobs: # excluding the v4 integration tests, for free serial-unit-test: name: > - ${{ matrix.os == 'ubuntu-latest' && 'Linux' || matrix.os == 'macos-latest' && 'MacOS' || 'Windows' }} + ${{ startsWith(matrix.os, 'ubuntu' ) && 'Linux' || + startsWith(matrix.os, 'macos' ) && 'MacOS' || + startsWith(matrix.os, 'windows') && 'Windows' || + 'Unknown' }} + ${{ endsWith(matrix.os, 'intel') && '(Intel)' || '' }} [${{ matrix.precision }}] serial + ${{ matrix.bmi2 == 'ON' && '(BMI)' || '' }} unit v${{ matrix.version }} runs-on: ${{ matrix.os }} @@ -38,17 +43,22 @@ jobs: # continue other jobs if any fail fail-fast: false - # we will compile QuEST with all precisions but no parallelisation + # we will compile QuEST with all precisions but no parallelisation (though with Intel BMI2) matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - version: [3, 4] + os: [ubuntu-latest, macos-latest, windows-latest, macos-15-intel, macos-26-intel] + version: [3, 4] precision: [1, 2, 4] + bmi2: [ON, OFF] - # MSVC cannot compile deprecated v3 tests exclude: + # MSVC cannot compile deprecated v3 tests - os: windows-latest version: 3 - + + # cannot use BMI2 on non-Intel MacOS + - bmi2: ON + os: macos-latest + # constants env: build_dir: "build" @@ -69,6 +79,7 @@ jobs: -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }} -DQUEST_FLOAT_PRECISION=${{ matrix.precision }} -DQUEST_ENABLE_ADIOS2=ON + -DQUEST_ENABLE_BMI2=${{ matrix.bmi2 }} # force 'Release' build (needed by MSVC to enable optimisations), and force serial (to avoid ADIOS2 OOM) - name: Compile diff --git a/CMakeLists.txt b/CMakeLists.txt index ac3c000cb..7b9f3c045 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -146,6 +146,15 @@ option( message(STATUS "NUMA awareness is turned ${QUEST_ENABLE_NUMA}. Set QUEST_ENABLE_NUMA to modify.") +# BMI2 +option( + QUEST_ENABLE_BMI2 + "Whether QuEST will accelerate CPU bit gather/scatter with x86 BMI2 (PEXT/PDEP) intrinsics." + OFF +) +message(STATUS "BMI2 bitwise acceleration is turned ${QUEST_ENABLE_BMI2}. Set QUEST_ENABLE_BMI2 to modify.") + + # Distribution option( QUEST_ENABLE_MPI @@ -297,6 +306,46 @@ if ((NOT (quest_tpb_remainder EQUAL 0)) OR NOT (QUEST_DEFAULT_NUM_GPU_THREADS_PE endif() +# probe whether BMI2 intrinsics are recognised by compiler +if (QUEST_ENABLE_BMI2) + + # save current CMAKE_REQUIRED_FLAGS for later restoration + set(_quest_saved_req_flags "${CMAKE_REQUIRED_FLAGS}") + + # give probe compilation the bmi flag if exists (does not exist on MSVC) + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag("-mbmi2" _quest_cxx_recognises_bmi2) + if (_quest_cxx_recognises_bmi2) + string(APPEND CMAKE_REQUIRED_FLAGS " -mbmi2") + endif() + + # probe whether intrinsics compile + include(CheckCXXSourceCompiles) + check_cxx_source_compiles( + " + #include + int main() { + _pext_u64(0ULL, 0ULL); + _pdep_u64(0ULL, 0ULL); + return 0; + } + " + _quest_cxx_compiles_bmi2) + + # restore CMAKE_REQUIRED_FLAS + set(CMAKE_REQUIRED_FLAGS "${_quest_saved_req_flags}") + unset(_quest_saved_req_flags) + + # error if probe failed + if (NOT _quest_cxx_compiles_bmi2) + message(FATAL_ERROR "QUEST_ENABLE_BMI2 was ${QUEST_ENABLE_BMI2} but BMI2 intrinsics were not recognised by the compiler. ") + endif() + + # var _quest_cxx_recognises_bmi2 used later during BMI2 management + +endif() + + # warn when numTPB will be later overridden by the current environment variable if( DEFINED ENV{QUEST_DEFAULT_NUM_GPU_THREADS_PER_BLOCK} @@ -342,7 +391,7 @@ if (QUEST_APPEND_CONFIG_TO_LIB_NAME) string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "-fp${QUEST_FLOAT_PRECISION}") if (QUEST_ENABLE_OMP) - string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+mt") + string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+omp") endif() if (QUEST_ENABLE_MPI) @@ -365,6 +414,18 @@ if (QUEST_APPEND_CONFIG_TO_LIB_NAME) string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+depr") endif() + if (QUEST_ENABLE_BMI2) + string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+bmi2") + endif() + + if (QUEST_ENABLE_ADIOS2) + string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+adios2") + endif() + + if (QUEST_ENABLE_SUBCOMM) + string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+subcomm") + endif() + endif() @@ -556,7 +617,6 @@ if (QUEST_ENABLE_CUQUANTUM) endif() - # Checkpointing (ADIOS2) if (QUEST_ENABLE_ADIOS2) @@ -631,6 +691,13 @@ if (QUEST_ENABLE_ADIOS2) endif() +# BMI2 (flag not necessary when unrecognised) +if (QUEST_ENABLE_BMI2 AND _quest_cxx_recognises_bmi2) + target_compile_options(QuEST PRIVATE + $<$:-mbmi2>) +endif() + + # =============================== # Set options to save in config.h @@ -643,6 +710,7 @@ set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI}) set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM}) set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM}) set(QUEST_COMPILE_ADIOS2 ${QUEST_ENABLE_ADIOS2}) +set(QUEST_COMPILE_BMI2 ${QUEST_ENABLE_BMI2}) set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API}) diff --git a/docs/cmake.md b/docs/cmake.md index 7f03d1055..223cce797 100644 --- a/docs/cmake.md +++ b/docs/cmake.md @@ -44,6 +44,7 @@ make | `QUEST_ENABLE_CUDA` | (`OFF`), `ON` | Determines whether QuEST will be built with support for NVIDIA GPU acceleration. If turned on, `CMAKE_CUDA_ARCHITECTURES` should probably also be set. | | `QUEST_ENABLE_CUQUANTUM` | (`OFF`), `ON` | Determines whether QuEST will make use of the NVIDIA CuQuantum library. Cannot be turned on if `QUEST_ENABLE_CUDA` is off. | | `QUEST_ENABLE_HIP` | (`OFF`), `ON` | Determines whether QuEST will be built with support for AMD GPU acceleration. If turned on, `CMAKE_HIP_ARCHITECTURES` should probably also be set. | +| `QUEST_ENABLE_BMI2` | (`OFF`), `ON` | Determines whether QuEST will be built with BMI2 intrinsics to accelerate CPU simulation of few-qubit Quregs. This is not compatible with all compilers and CPUs. **Beware** that if enabled, and the compiled QuEST executable is later run upon a different machine which lacks the BMI2 instructions, execution will crash. | | `QUEST_ENABLE_ADIOS2` | (`OFF`), `ON` | Determines whether QuEST will be built with ADIOS2 to enable checkpointing, via functions `saveQuregToFile()` and `createQuregFromFile()`. | | `QUEST_DOWNLOAD_ADIOS2` | (`ON`), `OFF` | Determines whether to download ADIOS2 from Github, when ADIOS2 is enabled but not found. | | `QUEST_ENABLE_DEPRECATED_API` | (`OFF`), `ON` | Determines whether QuEST will be built with support for the deprecated (v3) API. ***Note**: this will generate compiler warnings and is not supported by MSVC.* | diff --git a/quest/include/config.h.in b/quest/include/config.h.in index d89df4bfc..a62fb4100 100644 --- a/quest/include/config.h.in +++ b/quest/include/config.h.in @@ -42,6 +42,7 @@ defined(QUEST_COMPILE_HIP) || \ defined(QUEST_COMPILE_CUQUANTUM) || \ defined(QUEST_COMPILE_ADIOS2) || \ + defined(QUEST_COMPILE_BMI2) || \ defined(QUEST_ENABLE_NUMA) || \ defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -86,6 +87,7 @@ #cmakedefine01 QUEST_COMPILE_CUQUANTUM #cmakedefine01 QUEST_COMPILE_HIP #cmakedefine01 QUEST_COMPILE_ADIOS2 +#cmakedefine01 QUEST_COMPILE_BMI2 // crucial to QuEST source (informs optional NUMA usage) @@ -128,6 +130,7 @@ ! defined(QUEST_COMPILE_HIP) || \ ! defined(QUEST_COMPILE_CUQUANTUM) || \ ! defined(QUEST_COMPILE_ADIOS2) || \ + ! defined(QUEST_COMPILE_BMI2) || \ ! defined(QUEST_ENABLE_NUMA) || \ ! defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \ ! defined(QUEST_DISABLE_DEPRECATION_WARNINGS) @@ -156,6 +159,7 @@ ! (QUEST_COMPILE_HIP == 0 || QUEST_COMPILE_HIP == 1) || \ ! (QUEST_COMPILE_CUQUANTUM == 0 || QUEST_COMPILE_CUQUANTUM == 1) || \ ! (QUEST_COMPILE_ADIOS2 == 0 || QUEST_COMPILE_ADIOS2 == 1) || \ + ! (QUEST_COMPILE_BMI2 == 0 || QUEST_COMPILE_BMI2 == 1) || \ ! (QUEST_ENABLE_NUMA == 0 || QUEST_ENABLE_NUMA == 1) || \ ! (QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 0 || QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 1) || \ ! (QUEST_DISABLE_DEPRECATION_WARNINGS == 0 || QUEST_DISABLE_DEPRECATION_WARNINGS == 1) diff --git a/quest/src/core/bitwise.hpp b/quest/src/core/bitwise.hpp index f5266afa4..1c079bdfe 100644 --- a/quest/src/core/bitwise.hpp +++ b/quest/src/core/bitwise.hpp @@ -5,19 +5,25 @@ * @author Tyson Jones * @author Erich Essmann (improved OS agnosticism) * @author James Richings (patched setBit) + * @author PoJen Wang (added BMI2 intrinsics) */ #ifndef BITWISE_HPP #define BITWISE_HPP -#ifdef _MSC_VER - #include -#endif - +#include "quest/include/config.h" #include "quest/include/types.h" #include "quest/src/core/inliner.hpp" +#if QUEST_COMPILE_BMI2 + #include +#endif + +#ifdef _MSC_VER + #include +#endif + /* @@ -187,7 +193,10 @@ INLINE qindex setBits(qindex number, const int* bitIndices, int numIndices, qind INLINE qindex getValueOfBits(qindex number, const int* bitIndices, int numIndices) { - // bits are arbitrarily ordered, which affects value + // indices are arbitrarily ordered, which affects value; if the indices are + // known to be sorted, callers should instead use getValueOfPossiblySortedBits() + // which may (if available) use an optimised intrinsic, eliminating the below + // loop (though which will anyway be unrolled when numIndices is compile-time) qindex value = 0; for (int i=0; i list[i]) + return false; + + return true; +} + /* diff --git a/quest/src/core/utilities.hpp b/quest/src/core/utilities.hpp index 8e9509853..ee91a65de 100644 --- a/quest/src/core/utilities.hpp +++ b/quest/src/core/utilities.hpp @@ -80,6 +80,8 @@ qindex util_getBitMask(ConstList64 ctrls, ConstList64 ctrlStates, std::initializ List64 util_getList64OrAllOnes(const int* elemsOrNullptr, size_t length); +bool util_isSorted(ConstList64 list); + /* diff --git a/quest/src/cpu/cpu_subroutines.cpp b/quest/src/cpu/cpu_subroutines.cpp index 9cab85d1f..315b41717 100644 --- a/quest/src/cpu/cpu_subroutines.cpp +++ b/quest/src/cpu/cpu_subroutines.cpp @@ -17,6 +17,7 @@ * @author Luc Jaulmes (optimised initUniformState) * @author Richard Meister (helped patch on LLVM) * @author Amon K. (optimised small-qureg multiQubitProjector) + * @author PoJen Wang (added use of BMI2 intrinsics) * @author Kshitij Chhabra (patched v3 clauses with gcc9) * @author Ania (Anna) Brown (developed QuEST v1 logic) */ @@ -232,6 +233,7 @@ qindex cpu_statevec_packAmpsIntoBuffer(Qureg qureg, ConstList64 qubitInds, Const qindex offset = getSubBufferSendInd(qureg); auto sortedQubitInds = util_getSorted(qubitInds); + auto qubitIndMask = util_getBitMask(qubitInds); auto qubitStateMask = util_getBitMask(qubitInds, qubitStates); // use template param to compile-time unroll loop in insertBits() @@ -241,7 +243,7 @@ qindex cpu_statevec_packAmpsIntoBuffer(Qureg qureg, ConstList64 qubitInds, Const for (qindex n=0; n=1 since all qubits are in suffix, so qubits.size() <= suffix size) qindex numIts = qureg.numAmpsPerNode / powerOf2(qubits.size()); - auto sortedQubits = util_getSorted(qubits); // all in suffix + auto sortedQubits = util_getSorted(qubits); // all in suffix + auto qubitIndMask = util_getBitMask(qubits); auto qubitStateMask = util_getBitMask(qubits, outcomes); // use template param to compile-time unroll loop in insertBits() @@ -2014,7 +2033,7 @@ qreal cpu_statevec_calcProbOfMultiQubitOutcome_sub(Qureg qureg, ConstList64 qubi for (qindex n=0; n MIN_NUM_LOCAL_QUBITS_FOR_AUTO_QUREG_MULTITHREADING; - (void)parallelise; // suppress unused warning when not-compiling openmp) + (void) parallelise; // suppress unused warning when not-compiling openmp // clear amps (may be compile-time unrolled, or parallelised) #pragma omp parallel for if(parallelise) for (int i=0; i MIN_NUM_LOCAL_QUBITS_FOR_AUTO_QUREG_MULTITHREADING; - (void)parallelise; // suppress unused warning when not-compiling openmp) + (void) parallelise; // suppress unused warning when not-compiling openmp // clear amps; be compile-time unrolled, and/or parallelised (independent of qureg) #pragma omp parallel for if(parallelise) @@ -2141,7 +2169,7 @@ void cpu_densmatr_calcProbsOfAllMultiQubitOutcomes_sub(qreal* outProbs, Qureg qu qindex j = concatenateBits(qureg.rank, i, qureg.logNumAmpsPerNode); // k = outcome index corresponding to basis state j - qindex k = getValueOfBits(j, qubits.data(), numBits); // loop therein may be unrolled + qindex k = getValueOfPossiblySortedBits(j, areQubitsSorted, qubitIndMask, qubits.data(), numBits); #pragma omp atomic outProbs[k] += prob;