Optimise getValueOfBits and insertBits with BMI2 PEXT/PDEP (#717)

.github/workflows/compile.yml

@@ -39,7 +39,11 @@ jobs:
  # test only compilation succeeds (no execution)
  build-test:
  name: >
- ${{ matrix.os == 'ubuntu-latest' && 'Linux' || matrix.os == 'macos-latest' && 'MacOS' || 'Windows' }}
+ ${{ startsWith(matrix.os, 'ubuntu' ) && 'Linux' || 
+ startsWith(matrix.os, 'macos' ) && 'MacOS' ||
+ startsWith(matrix.os, 'windows') && 'Windows' ||
+ 'Unknown' }}
+ ${{ endsWith(matrix.os, 'intel') && '(Intel)' || '' }}
  [${{ matrix.precision }}]
  ${{ matrix.omp == 'ON' && 'OMP' || '' }}
  ${{ matrix.mpi == 'ON' && 'MPI' || '' }}
@@ -48,6 +52,7 @@ jobs:
  ${{ matrix.hip == 'ON' && 'HIP' || '' }}
  ${{ matrix.cuquantum == 'ON' && 'CUQ' || '' }}
  ${{ matrix.adios2 == 'ON' && 'CKPT' || '' }}
+ ${{ matrix.bmi2 == 'ON' && 'BMI' || '' }}
 
  runs-on: ${{ matrix.os }}
 
@@ -59,16 +64,18 @@ jobs:
  # (causes CUDA and MPI installation to fail on Windows)
  max-parallel: 8
 
- # compile QuEST with all combinations of below flags
+ # compile QuEST with all combinations of below flags (Intel runners for BMI2 instrinsics);
+ # incredibly, this (with exclusions below) achieves 256 combos, which is the Github limit!
  matrix:
- os: [windows-2022, ubuntu-latest, macos-latest]
+ os: [windows-2022, ubuntu-latest, macos-latest, macos-15-intel, macos-26-intel]
  precision: [1, 2, 4]
  omp: [ON, OFF]
  mpi: [ON, OFF]
  cuda: [ON, OFF]
  hip: [ON, OFF]
  cuquantum: [ON, OFF]
  adios2: [ON, OFF]
+ bmi2: [ON, OFF]
  mpilib: ['', 'mpich', 'ompi', 'impi', 'msmpi']
 
  # disable deprecated API on MSVC, and assign unique compilers,
@@ -82,6 +89,12 @@ jobs:
  - os: macos-latest
  compiler: clang++
  deprecated: ON
+ - os: macos-15-intel
+ compiler: clang++
+ deprecated: ON
+ - os: macos-26-intel
+ compiler: clang++
+ deprecated: ON
  - os: windows-2022
  compiler: cl
  deprecated: OFF
@@ -104,14 +117,42 @@ jobs:
  # cannot use GPU on MacOS
  - cuda: ON
  os: macos-latest
+ - cuda: ON
+ os: macos-15-intel
+ - cuda: ON
+ os: macos-26-intel
  - hip: ON
  os: macos-latest
+ - hip: ON
+ os: macos-15-intel
+ - hip: ON
+ os: macos-26-intel
 
  # cannot use cuquantum on Windows or MacOS
  - cuquantum: ON
  os: windows-2022
  - cuquantum: ON
  os: macos-latest
+ - cuquantum: ON
+ os: macos-15-intel
+ - cuquantum: ON
+ os: macos-26-intel
+
+ # cannot use BMI2 on non-Intel MacOS
+ - bmi2: ON
+ os: macos-latest
+
+ # use ONLY BMI2 on Intel MacOS, just to shrink matrix (Github imposes 256 max)
+ - bmi2: OFF
+ os: macos-15-intel
+ - bmi2: OFF
+ os: macos-26-intel
+
+ # do not combine BMI2 with MPI or ADIOS2 (they don't interact), just to shrink matrix
+ - bmi2: ON
+ mpi: ON
+ - bmi2: ON
+ adios2: ON
 
  # don't enumerate MPI libraries when not using MPI
  - mpi: OFF
@@ -132,6 +173,14 @@ jobs:
  mpilib: 'msmpi' # MacOS: [MPICH, OpenMPI]
  - os: macos-latest
  mpilib: 'impi'
+ - os: macos-15-intel
+ mpilib: 'msmpi'
+ - os: macos-15-intel
+ mpilib: 'impi'
+ - os: macos-26-intel
+ mpilib: 'msmpi'
+ - os: macos-26-intel
+ mpilib: 'impi'
  - os: windows-2022
  mpilib: 'mpich' # Windows: [Intel MPI, MS MPI]
  - os: windows-2022
@@ -252,6 +301,7 @@ jobs:
  -DQUEST_ENABLE_HIP=${{ matrix.hip }}
  -DQUEST_ENABLE_CUQUANTUM=${{ matrix.cuquantum }}
  -DQUEST_ENABLE_ADIOS2=${{ matrix.adios2 }}
+ -DQUEST_ENABLE_BMI2=${{ matrix.bmi2 }}
  -DCMAKE_CUDA_ARCHITECTURES=${{ env.cuda_arch }}
  -DCMAKE_HIP_ARCHITECTURES=${{ env.hip_arch }}
  -DCMAKE_CXX_COMPILER=${{ matrix.compiler }}

.github/workflows/test_free.yml

@@ -27,9 +27,14 @@ jobs:
  # excluding the v4 integration tests, for free
  serial-unit-test:
  name: >
- ${{ matrix.os == 'ubuntu-latest' && 'Linux' || matrix.os == 'macos-latest' && 'MacOS' || 'Windows' }}
+ ${{ startsWith(matrix.os, 'ubuntu' ) && 'Linux' || 
+ startsWith(matrix.os, 'macos' ) && 'MacOS' ||
+ startsWith(matrix.os, 'windows') && 'Windows' ||
+ 'Unknown' }}
+ ${{ endsWith(matrix.os, 'intel') && '(Intel)' || '' }}
  [${{ matrix.precision }}]
  serial
+ ${{ matrix.bmi2 == 'ON' && '(BMI)' || '' }}
  unit v${{ matrix.version }}
 
  runs-on: ${{ matrix.os }}
@@ -38,17 +43,22 @@ jobs:
  # continue other jobs if any fail
  fail-fast: false
 
- # we will compile QuEST with all precisions but no parallelisation
+ # we will compile QuEST with all precisions but no parallelisation (though with Intel BMI2)
  matrix:
- os: [ubuntu-latest, macos-latest, windows-latest]
- version: [3, 4]
+ os: [ubuntu-latest, macos-latest, windows-latest, macos-15-intel, macos-26-intel]
+ version: [3, 4]
  precision: [1, 2, 4]
+ bmi2: [ON, OFF]
 
- # MSVC cannot compile deprecated v3 tests
  exclude:
+ # MSVC cannot compile deprecated v3 tests
  - os: windows-latest
  version: 3
-
+
+ # cannot use BMI2 on non-Intel MacOS
+ - bmi2: ON
+ os: macos-latest
+
  # constants
  env:
  build_dir: "build"
@@ -69,6 +79,7 @@ jobs:
  -DQUEST_DISABLE_DEPRECATION_WARNINGS=${{ matrix.version == 3 && 'ON' || 'OFF' }}
  -DQUEST_FLOAT_PRECISION=${{ matrix.precision }}
  -DQUEST_ENABLE_ADIOS2=ON
+ -DQUEST_ENABLE_BMI2=${{ matrix.bmi2 }}
 
  # force 'Release' build (needed by MSVC to enable optimisations), and force serial (to avoid ADIOS2 OOM)
  - name: Compile

CMakeLists.txt

@@ -146,6 +146,15 @@ option(
 message(STATUS "NUMA awareness is turned ${QUEST_ENABLE_NUMA}. Set QUEST_ENABLE_NUMA to modify.")
 
 
+# BMI2
+option(
+ QUEST_ENABLE_BMI2
+ "Whether QuEST will accelerate CPU bit gather/scatter with x86 BMI2 (PEXT/PDEP) intrinsics."
+ OFF
+)
+message(STATUS "BMI2 bitwise acceleration is turned ${QUEST_ENABLE_BMI2}. Set QUEST_ENABLE_BMI2 to modify.")
+
+
 # Distribution
 option(
  QUEST_ENABLE_MPI
@@ -297,6 +306,46 @@ if ((NOT (quest_tpb_remainder EQUAL 0)) OR NOT (QUEST_DEFAULT_NUM_GPU_THREADS_PE
 endif()
 
 
+# probe whether BMI2 intrinsics are recognised by compiler
+if (QUEST_ENABLE_BMI2)
+
+ # save current CMAKE_REQUIRED_FLAGS for later restoration
+ set(_quest_saved_req_flags "${CMAKE_REQUIRED_FLAGS}")
+
+ # give probe compilation the bmi flag if exists (does not exist on MSVC)
+ include(CheckCXXCompilerFlag)
+ check_cxx_compiler_flag("-mbmi2" _quest_cxx_recognises_bmi2)
+ if (_quest_cxx_recognises_bmi2)
+ string(APPEND CMAKE_REQUIRED_FLAGS " -mbmi2")
+ endif()
+
+ # probe whether intrinsics compile
+ include(CheckCXXSourceCompiles)
+ check_cxx_source_compiles(
+ "
+ #include <immintrin.h>
+ int main() {
+ _pext_u64(0ULL, 0ULL);
+ _pdep_u64(0ULL, 0ULL);
+ return 0;
+ }
+ "
+ _quest_cxx_compiles_bmi2)
+
+ # restore CMAKE_REQUIRED_FLAS
+ set(CMAKE_REQUIRED_FLAGS "${_quest_saved_req_flags}")
+ unset(_quest_saved_req_flags)
+
+ # error if probe failed
+ if (NOT _quest_cxx_compiles_bmi2)
+ message(FATAL_ERROR "QUEST_ENABLE_BMI2 was ${QUEST_ENABLE_BMI2} but BMI2 intrinsics were not recognised by the compiler. ")
+ endif()
+
+ # var _quest_cxx_recognises_bmi2 used later during BMI2 management 
+
+endif()
+
+
 # warn when numTPB will be later overridden by the current environment variable
 if(
  DEFINED ENV{QUEST_DEFAULT_NUM_GPU_THREADS_PER_BLOCK} 
@@ -342,7 +391,7 @@ if (QUEST_APPEND_CONFIG_TO_LIB_NAME)
  string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "-fp${QUEST_FLOAT_PRECISION}")
 
  if (QUEST_ENABLE_OMP)
- string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+mt")
+ string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+omp")
  endif()
 
  if (QUEST_ENABLE_MPI)
@@ -365,6 +414,18 @@ if (QUEST_APPEND_CONFIG_TO_LIB_NAME)
  string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+depr")
  endif()
 
+ if (QUEST_ENABLE_BMI2)
+ string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+bmi2")
+ endif()
+
+ if (QUEST_ENABLE_ADIOS2)
+ string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+adios2")
+ endif()
+
+ if (QUEST_ENABLE_SUBCOMM)
+ string(CONCAT QUEST_OUTPUT_LIB_NAME ${QUEST_OUTPUT_LIB_NAME} "+subcomm")
+ endif()
+
 endif()
 
 
@@ -556,7 +617,6 @@ if (QUEST_ENABLE_CUQUANTUM)
 endif()
 
 
-
 # Checkpointing (ADIOS2)
 if (QUEST_ENABLE_ADIOS2)
 
@@ -631,6 +691,13 @@ if (QUEST_ENABLE_ADIOS2)
 endif()
 
 
+# BMI2 (flag not necessary when unrecognised)
+if (QUEST_ENABLE_BMI2 AND _quest_cxx_recognises_bmi2)
+ target_compile_options(QuEST PRIVATE
+ $<$<COMPILE_LANGUAGE:CXX>:-mbmi2>)
+endif()
+
+
 
 # ===============================
 # Set options to save in config.h
@@ -643,6 +710,7 @@ set(QUEST_COMPILE_MPI ${QUEST_ENABLE_MPI})
 set(QUEST_COMPILE_SUBCOMM ${QUEST_ENABLE_SUBCOMM})
 set(QUEST_COMPILE_CUQUANTUM ${QUEST_ENABLE_CUQUANTUM})
 set(QUEST_COMPILE_ADIOS2 ${QUEST_ENABLE_ADIOS2})
+set(QUEST_COMPILE_BMI2 ${QUEST_ENABLE_BMI2})
 set(QUEST_INCLUDE_DEPRECATED_FUNCTIONS ${QUEST_ENABLE_DEPRECATED_API})
 
 

docs/cmake.md

@@ -44,6 +44,7 @@ make
 | `QUEST_ENABLE_CUDA` | (`OFF`), `ON` | Determines whether QuEST will be built with support for NVIDIA GPU acceleration. If turned on, `CMAKE_CUDA_ARCHITECTURES` should probably also be set. |
 | `QUEST_ENABLE_CUQUANTUM` | (`OFF`), `ON` | Determines whether QuEST will make use of the NVIDIA CuQuantum library. Cannot be turned on if `QUEST_ENABLE_CUDA` is off. |
 | `QUEST_ENABLE_HIP` | (`OFF`), `ON` | Determines whether QuEST will be built with support for AMD GPU acceleration. If turned on, `CMAKE_HIP_ARCHITECTURES` should probably also be set. |
+| `QUEST_ENABLE_BMI2` | (`OFF`), `ON` | Determines whether QuEST will be built with BMI2 intrinsics to accelerate CPU simulation of few-qubit Quregs. This is not compatible with all compilers and CPUs. **Beware** that if enabled, and the compiled QuEST executable is later run upon a different machine which lacks the BMI2 instructions, execution will crash. |
 | `QUEST_ENABLE_ADIOS2` | (`OFF`), `ON` | Determines whether QuEST will be built with ADIOS2 to enable checkpointing, via functions `saveQuregToFile()` and `createQuregFromFile()`. |
 | `QUEST_DOWNLOAD_ADIOS2` | (`ON`), `OFF` | Determines whether to download ADIOS2 from Github, when ADIOS2 is enabled but not found. |
 | `QUEST_ENABLE_DEPRECATED_API` | (`OFF`), `ON` | Determines whether QuEST will be built with support for the deprecated (v3) API. ***Note**: this will generate compiler warnings and is not supported by MSVC.* |

quest/include/config.h.in

@@ -42,6 +42,7 @@
  defined(QUEST_COMPILE_HIP) || \
  defined(QUEST_COMPILE_CUQUANTUM) || \
  defined(QUEST_COMPILE_ADIOS2) || \
+ defined(QUEST_COMPILE_BMI2) || \
  defined(QUEST_ENABLE_NUMA) || \
  defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \
  defined(QUEST_DISABLE_DEPRECATION_WARNINGS)
@@ -86,6 +87,7 @@
 #cmakedefine01 QUEST_COMPILE_CUQUANTUM
 #cmakedefine01 QUEST_COMPILE_HIP
 #cmakedefine01 QUEST_COMPILE_ADIOS2
+#cmakedefine01 QUEST_COMPILE_BMI2
 
 
 // crucial to QuEST source (informs optional NUMA usage)
@@ -128,6 +130,7 @@
  ! defined(QUEST_COMPILE_HIP) || \
  ! defined(QUEST_COMPILE_CUQUANTUM) || \
  ! defined(QUEST_COMPILE_ADIOS2) || \
+ ! defined(QUEST_COMPILE_BMI2) || \
  ! defined(QUEST_ENABLE_NUMA) || \
  ! defined(QUEST_INCLUDE_DEPRECATED_FUNCTIONS) || \
  ! defined(QUEST_DISABLE_DEPRECATION_WARNINGS)
@@ -156,6 +159,7 @@
  ! (QUEST_COMPILE_HIP == 0 || QUEST_COMPILE_HIP == 1) || \
  ! (QUEST_COMPILE_CUQUANTUM == 0 || QUEST_COMPILE_CUQUANTUM == 1) || \
  ! (QUEST_COMPILE_ADIOS2 == 0 || QUEST_COMPILE_ADIOS2 == 1) || \
+ ! (QUEST_COMPILE_BMI2 == 0 || QUEST_COMPILE_BMI2 == 1) || \
  ! (QUEST_ENABLE_NUMA == 0 || QUEST_ENABLE_NUMA == 1) || \
  ! (QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 0 || QUEST_INCLUDE_DEPRECATED_FUNCTIONS == 1) || \
  ! (QUEST_DISABLE_DEPRECATION_WARNINGS == 0 || QUEST_DISABLE_DEPRECATION_WARNINGS == 1)