From 63786ba5324c76ea78b02310ce01e788496cc2e5 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Fri, 16 May 2025 20:15:36 -0700 Subject: [PATCH 01/16] Toolset update: VS 2022 17.14 Preview 7 (#5512) --- CMakeLists.txt | 6 +++--- README.md | 8 ++++---- azure-devops/config.yml | 2 +- azure-devops/provision-image.ps1 | 2 +- benchmarks/CMakeLists.txt | 2 +- tools/CMakeLists.txt | 2 +- tools/format/CMakeLists.txt | 2 +- tools/validate/CMakeLists.txt | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f66dc08858f..f0c0f2d91c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,13 +1,13 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 3.30.0) +cmake_minimum_required(VERSION 3.31.0) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) project(msvc_standard_libraries LANGUAGES CXX) -if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.44.35109.1") - message(FATAL_ERROR "The STL must be built with VS 2022 17.14 Preview 3 or later.") +if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.44.35207.1") + message(FATAL_ERROR "The STL must be built with VS 2022 17.14 Preview 7 or later.") endif() include(CheckCXXSourceCompiles) diff --git a/README.md b/README.md index 6e1567565ac..8b085358593 100644 --- a/README.md +++ b/README.md @@ -141,7 +141,7 @@ Just try to follow these rules, so we can spend more time fixing bugs and implem # How To Build With The Visual Studio IDE -1. Install Visual Studio 2022 17.14 Preview 3 or later. +1. Install Visual Studio 2022 17.14 Preview 7 or later. * Select "Windows 11 SDK (10.0.22621.0)" in the VS Installer. * Select "MSVC v143 - VS 2022 C++ ARM64/ARM64EC build tools (Latest)" in the VS Installer if you would like to build the ARM64/ARM64EC target. @@ -149,7 +149,7 @@ Just try to follow these rules, so we can spend more time fixing bugs and implem if you would like to build the ARM target. * We recommend selecting "C++ CMake tools for Windows" in the VS Installer. This will ensure that you're using supported versions of CMake and Ninja. - * Otherwise, install [CMake][] 3.30.0 or later, and [Ninja][] 1.12.1 or later. + * Otherwise, install [CMake][] 3.31.0 or later, and [Ninja][] 1.12.1 or later. * Make sure [Python][] 3.13 or later is available to CMake. 2. Open Visual Studio, and choose the "Clone or check out code" option. Enter the URL of this repository, `https://github.com/microsoft/STL`. @@ -160,7 +160,7 @@ Just try to follow these rules, so we can spend more time fixing bugs and implem # How To Build With A Native Tools Command Prompt -1. Install Visual Studio 2022 17.14 Preview 3 or later. +1. Install Visual Studio 2022 17.14 Preview 7 or later. * Select "Windows 11 SDK (10.0.22621.0)" in the VS Installer. * Select "MSVC v143 - VS 2022 C++ ARM64/ARM64EC build tools (Latest)" in the VS Installer if you would like to build the ARM64/ARM64EC target. @@ -168,7 +168,7 @@ Just try to follow these rules, so we can spend more time fixing bugs and implem if you would like to build the ARM target. * We recommend selecting "C++ CMake tools for Windows" in the VS Installer. This will ensure that you're using supported versions of CMake and Ninja. - * Otherwise, install [CMake][] 3.30.0 or later, and [Ninja][] 1.12.1 or later. + * Otherwise, install [CMake][] 3.31.0 or later, and [Ninja][] 1.12.1 or later. * Make sure [Python][] 3.13 or later is available to CMake. 2. Open a command prompt. 3. Change directories to a location where you'd like a clone of this STL repository. diff --git a/azure-devops/config.yml b/azure-devops/config.yml index 66b26be36c0..b0833824a78 100644 --- a/azure-devops/config.yml +++ b/azure-devops/config.yml @@ -5,7 +5,7 @@ variables: - name: poolName - value: 'StlBuild-2025-04-16T1328-Pool' + value: 'StlBuild-2025-05-16T0735-Pool' readonly: true - name: poolDemands value: 'EnableSpotVM -equals false' diff --git a/azure-devops/provision-image.ps1 b/azure-devops/provision-image.ps1 index 897855be6d7..901299799b3 100644 --- a/azure-devops/provision-image.ps1 +++ b/azure-devops/provision-image.ps1 @@ -40,7 +40,7 @@ foreach ($workload in $VisualStudioWorkloads) { } # https://github.com/PowerShell/PowerShell/releases/latest -$PowerShellUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.5.0/PowerShell-7.5.0-win-x64.msi' +$PowerShellUrl = 'https://github.com/PowerShell/PowerShell/releases/download/v7.5.1/PowerShell-7.5.1-win-x64.msi' $PowerShellArgs = @('/quiet', '/norestart') $PythonUrl = 'https://www.python.org/ftp/python/3.13.3/python-3.13.3-amd64.exe' diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 04e47486df5..50f9787b6c4 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 3.30.0) +cmake_minimum_required(VERSION 3.31.0) project(msvc_standard_libraries_benchmarks LANGUAGES CXX) if(DEFINED STL_BINARY_DIR) diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 91d829a60c5..25f5a60d672 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 3.30.0) +cmake_minimum_required(VERSION 3.31.0) project(msvc_standard_libraries_tools LANGUAGES CXX) add_subdirectory(format) diff --git a/tools/format/CMakeLists.txt b/tools/format/CMakeLists.txt index dbd31237141..d0ecc26c9d0 100644 --- a/tools/format/CMakeLists.txt +++ b/tools/format/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 3.30.0) +cmake_minimum_required(VERSION 3.31.0) project(msvc_standard_libraries_format NONE) find_program(CLANG_FORMAT diff --git a/tools/validate/CMakeLists.txt b/tools/validate/CMakeLists.txt index 175103401f0..51475a2f67c 100644 --- a/tools/validate/CMakeLists.txt +++ b/tools/validate/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 3.30.0) +cmake_minimum_required(VERSION 3.31.0) project(msvc_standard_libraries_validate LANGUAGES CXX) add_executable(validate-binary validate.cpp) From c86725a032819b4a701200da370f1d2e5b45e576 Mon Sep 17 00:00:00 2001 From: "S. B. Tam" Date: Sat, 17 May 2025 11:18:12 +0800 Subject: [PATCH 02/16] Add arm64EC pipeline (#5492) Co-authored-by: Stephan T. Lavavej --- CMakeLists.txt | 10 ++++++++-- azure-devops/asan-pipeline.yml | 2 ++ azure-devops/build-and-test.yml | 6 +++++- azure-devops/cmake-configure-build.yml | 4 ++++ azure-devops/run-tests.yml | 6 ++++-- azure-pipelines.yml | 27 +++++++++++++++++++++++++- benchmarks/CMakeLists.txt | 2 +- stl/CMakeLists.txt | 3 +++ 8 files changed, 53 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f0c0f2d91c6..5ec7d70b8f2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,7 +27,7 @@ endif() string(TOLOWER "${VCLIBS_TARGET_ARCHITECTURE}" VCLIBS_TARGET_ARCHITECTURE) -if(VCLIBS_TARGET_ARCHITECTURE MATCHES "^x64$|^x86$") +if(VCLIBS_TARGET_ARCHITECTURE MATCHES "^(x86|x64)$") enable_language(ASM_MASM) endif() @@ -67,7 +67,7 @@ elseif(VCLIBS_TARGET_ARCHITECTURE STREQUAL "x64") set(VCLIBS_I386_OR_AMD64 "amd64") set(VCLIBS_X86_OR_X64 "x64") add_compile_definitions(_AMD64_ _VCRT_WIN32_WINNT=0x0501 _STL_WIN32_WINNT=0x0501) -elseif(VCLIBS_TARGET_ARCHITECTURE STREQUAL "armv7") +elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^(arm|armv7)$") set(VCLIBS_TARGET_ARCHITECTURE "arm") set(VCLIBS_I386_OR_AMD64 "arm") set(VCLIBS_X86_OR_X64 "arm") @@ -108,6 +108,12 @@ if(STL_USE_ANALYZE) # LPARAM is a LONG_PTR (intptr_t), and it's invalid to apply _In_opt_ to a non-pointer. # As of the Windows 11 SDK (10.0.22621.0), there are 5 total occurrences of warning C6553 affecting the STL's build. add_compile_options("$<$:/analyze:autolog-;/wd6553>") + + if(VCLIBS_TARGET_ARCHITECTURE STREQUAL "arm64ec") + # TRANSITION, the Windows SDK emits "warning C28301: No annotations for first declaration of 'meow'" + # for various intrinsics when building for ARM64EC. + add_compile_options("$<$:/wd28301>") + endif() endif() set(VCLIBS_DEBUG_OPTIONS "$<$:/Od>") diff --git a/azure-devops/asan-pipeline.yml b/azure-devops/asan-pipeline.yml index f98953de548..bf3a2148c57 100644 --- a/azure-devops/asan-pipeline.yml +++ b/azure-devops/asan-pipeline.yml @@ -26,6 +26,7 @@ stages: parameters: hostArch: x64 targetArch: x64 + targetPlatform: x64 asanBuild: true testTargets: STL-ASan-CI @@ -40,6 +41,7 @@ stages: parameters: hostArch: x86 targetArch: x86 + targetPlatform: x86 asanBuild: true testTargets: STL-ASan-CI diff --git a/azure-devops/build-and-test.yml b/azure-devops/build-and-test.yml index edc9134cd93..a90c7c2f382 100644 --- a/azure-devops/build-and-test.yml +++ b/azure-devops/build-and-test.yml @@ -6,6 +6,8 @@ parameters: type: string - name: targetArch type: string +- name: targetPlatform + type: string - name: analyzeBuild type: boolean default: false @@ -28,7 +30,7 @@ parameters: type: boolean default: false jobs: -- job: '${{ parameters.targetArch }}' +- job: '${{ parameters.targetPlatform }}' strategy: parallel: ${{ parameters.numShards }} timeoutInMinutes: 30 @@ -53,6 +55,7 @@ jobs: parameters: hostArch: ${{ parameters.hostArch }} targetArch: ${{ parameters.targetArch }} + targetPlatform: ${{ parameters.targetPlatform }} analyzeBuild: ${{ parameters.analyzeBuild }} asanBuild: ${{ parameters.asanBuild }} buildBenchmarks: ${{ parameters.buildBenchmarks }} @@ -61,5 +64,6 @@ jobs: parameters: hostArch: ${{ parameters.hostArch }} targetArch: ${{ parameters.targetArch }} + targetPlatform: ${{ parameters.targetPlatform }} testTargets: ${{ parameters.testTargets }} skipTesting: ${{ parameters.skipTesting }} diff --git a/azure-devops/cmake-configure-build.yml b/azure-devops/cmake-configure-build.yml index dec0d987f71..39d27169841 100644 --- a/azure-devops/cmake-configure-build.yml +++ b/azure-devops/cmake-configure-build.yml @@ -6,6 +6,8 @@ parameters: type: string - name: targetArch type: string +- name: targetPlatform + type: string - name: analyzeBuild type: boolean - name: asanBuild @@ -36,6 +38,7 @@ steps: -DSTL_USE_ANALYZE=${{ parameters.analyzeBuild }} ^ -DSTL_ASAN_BUILD=${{ parameters.asanBuild }} ^ -DTESTS_BUILD_ONLY=${{ parameters.testsBuildOnly }} ^ + -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ^ -S $(Build.SourcesDirectory) -B "$(buildOutputLocation)" displayName: 'Configure the STL' timeoutInMinutes: 2 @@ -57,6 +60,7 @@ steps: -DCMAKE_CXX_COMPILER=cl ^ -DCMAKE_BUILD_TYPE=Release ^ -DSTL_BINARY_DIR="$(buildOutputLocation)" ^ + -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ^ -S $(Build.SourcesDirectory)/benchmarks -B "$(benchmarkBuildOutputLocation)" displayName: 'Configure the benchmarks' timeoutInMinutes: 2 diff --git a/azure-devops/run-tests.yml b/azure-devops/run-tests.yml index a6e2c1363d3..aba67bc3e87 100644 --- a/azure-devops/run-tests.yml +++ b/azure-devops/run-tests.yml @@ -6,6 +6,8 @@ parameters: type: string - name: targetArch type: string +- name: targetPlatform + type: string - name: testTargets type: string - name: skipTesting @@ -28,8 +30,8 @@ steps: searchFolder: $(buildOutputLocation) testResultsFormat: JUnit testResultsFiles: '**/test-results.xml' - testRunTitle: 'test-${{ parameters.targetArch }}-$(System.JobPositionInPhase)' + testRunTitle: 'test-${{ parameters.targetPlatform }}-$(System.JobPositionInPhase)' - publish: $(buildOutputLocation)/test-results.xml - artifact: '${{ parameters.targetArch }}-$(System.JobPositionInPhase)-xml-$(System.JobId)' + artifact: '${{ parameters.targetPlatform }}-$(System.JobPositionInPhase)-xml-$(System.JobId)' condition: and(failed(), not(${{ parameters.skipTesting }})) displayName: 'Publish XML Artifact' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3941274d57a..1f8c6505343 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -# Build STL targeting x86, x64, arm, arm64 +# Build STL targeting x86, x64, arm, arm64, arm64ec variables: - template: azure-devops/config.yml @@ -29,6 +29,7 @@ stages: parameters: hostArch: x64 targetArch: x64 + targetPlatform: x64 analyzeBuild: true buildBenchmarks: true numShards: 1 @@ -45,6 +46,7 @@ stages: parameters: hostArch: x86 targetArch: x86 + targetPlatform: x86 analyzeBuild: true buildBenchmarks: true numShards: 1 @@ -61,6 +63,7 @@ stages: parameters: hostArch: x64 targetArch: arm + targetPlatform: arm analyzeBuild: true numShards: 1 skipTesting: true @@ -76,11 +79,28 @@ stages: parameters: hostArch: x64 targetArch: arm64 + targetPlatform: arm64 analyzeBuild: true buildBenchmarks: true numShards: 1 skipTesting: true + - stage: Early_Build_ARM64EC + dependsOn: [] + displayName: 'Early Build ARM64EC' + pool: + name: ${{ variables.poolName }} + demands: ${{ variables.poolDemands }} + jobs: + - template: azure-devops/build-and-test.yml + parameters: + hostArch: x64 + targetArch: arm64 + targetPlatform: arm64ec + analyzeBuild: true + numShards: 1 + skipTesting: true + - stage: Build_And_Test_x64 dependsOn: - Code_Format @@ -88,6 +108,7 @@ stages: - Early_Build_x86 - Early_Build_ARM - Early_Build_ARM64 + - Early_Build_ARM64EC displayName: 'Build and Test x64' pool: name: ${{ variables.poolName }} @@ -97,6 +118,7 @@ stages: parameters: hostArch: x64 targetArch: x64 + targetPlatform: x64 - stage: Build_And_Test_x86 dependsOn: Build_And_Test_x64 @@ -109,6 +131,7 @@ stages: parameters: hostArch: x86 targetArch: x86 + targetPlatform: x86 - stage: Build_And_Test_ARM dependsOn: Build_And_Test_x64 @@ -121,6 +144,7 @@ stages: parameters: hostArch: x64 targetArch: arm + targetPlatform: arm testsBuildOnly: true - stage: Build_And_Test_ARM64 @@ -134,4 +158,5 @@ stages: parameters: hostArch: x64 targetArch: arm64 + targetPlatform: arm64 testsBuildOnly: true diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 50f9787b6c4..d9c45e5aceb 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -23,7 +23,7 @@ if(DEFINED STL_BINARY_DIR) set(VCLIBS_I386_OR_AMD64 "i386") elseif(VCLIBS_TARGET_ARCHITECTURE STREQUAL "x64") set(VCLIBS_I386_OR_AMD64 "amd64") - elseif(VCLIBS_TARGET_ARCHITECTURE STREQUAL "armv7") + elseif(VCLIBS_TARGET_ARCHITECTURE MATCHES "^(arm|armv7)$") set(VCLIBS_I386_OR_AMD64 "arm") elseif(VCLIBS_TARGET_ARCHITECTURE STREQUAL "arm64") set(VCLIBS_I386_OR_AMD64 "arm64") diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index 10d21b3173c..e58bb2defbd 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -507,6 +507,9 @@ function(add_stl_dlls D_SUFFIX REL_OR_DBG) add_library(msvcp${D_SUFFIX} SHARED) target_link_libraries(msvcp${D_SUFFIX} PRIVATE msvcp${D_SUFFIX}_eha_objects msvcp${D_SUFFIX}_objects msvcp${D_SUFFIX}_init_objects "${TOOLSET_LIB}/vcruntime${D_SUFFIX}.lib" "${TOOLSET_LIB}/msvcrt${D_SUFFIX}.lib" "ucrt${D_SUFFIX}.lib") + if(VCLIBS_TARGET_ARCHITECTURE MATCHES "^(arm64|arm64ec)$") + target_link_libraries(msvcp${D_SUFFIX} PRIVATE "softintrin.lib") + endif() set_target_properties(msvcp${D_SUFFIX} PROPERTIES ARCHIVE_OUTPUT_NAME "msvcp140_base${D_SUFFIX}${VCLIBS_SUFFIX}") set_target_properties(msvcp${D_SUFFIX} PROPERTIES ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") set_target_properties(msvcp${D_SUFFIX} PROPERTIES OUTPUT_NAME "msvcp140${D_SUFFIX}${VCLIBS_SUFFIX}") From b214da0f26b423d900d636fa38fbaa3af3d5d167 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 17 May 2025 06:23:16 +0300 Subject: [PATCH 03/16] Vectorize `search` for 32-bit and 64-bit elements, also improve 8-bit and 16-bit vectorization (#5484) Co-authored-by: Stephan T. Lavavej --- benchmarks/src/search.cpp | 17 +- stl/inc/__msvc_string_view.hpp | 4 +- stl/inc/xutility | 20 +- stl/src/vector_algorithms.cpp | 820 +++++++++++++----- .../test.cpp | 25 +- .../VSO_0000000_vector_algorithms/test.cpp | 5 +- 6 files changed, 653 insertions(+), 238 deletions(-) diff --git a/benchmarks/src/search.cpp b/benchmarks/src/search.cpp index 4540860c928..47583b8214b 100644 --- a/benchmarks/src/search.cpp +++ b/benchmarks/src/search.cpp @@ -52,8 +52,9 @@ constexpr data_and_pattern patterns[] = { template using not_highly_aligned_basic_string = std::basic_string, not_highly_aligned_allocator>; -using not_highly_aligned_string = not_highly_aligned_basic_string; -using not_highly_aligned_wstring = not_highly_aligned_basic_string; +using not_highly_aligned_string = not_highly_aligned_basic_string; +using not_highly_aligned_wstring = not_highly_aligned_basic_string; +using not_highly_aligned_u32string = not_highly_aligned_basic_string; void c_strstr(benchmark::State& state) { const auto& src_haystack = patterns[static_cast(state.range())].data; @@ -190,23 +191,35 @@ BENCHMARK(c_strstr)->Apply(common_args); BENCHMARK(classic_search)->Apply(common_args); BENCHMARK(classic_search)->Apply(common_args); +BENCHMARK(classic_search)->Apply(common_args); +BENCHMARK(classic_search)->Apply(common_args); BENCHMARK(ranges_search)->Apply(common_args); BENCHMARK(ranges_search)->Apply(common_args); +BENCHMARK(ranges_search)->Apply(common_args); +BENCHMARK(ranges_search)->Apply(common_args); BENCHMARK(search_default_searcher)->Apply(common_args); BENCHMARK(search_default_searcher)->Apply(common_args); +BENCHMARK(search_default_searcher)->Apply(common_args); +BENCHMARK(search_default_searcher)->Apply(common_args); BENCHMARK(member_find)->Apply(common_args); BENCHMARK(member_find)->Apply(common_args); +BENCHMARK(member_find)->Apply(common_args); BENCHMARK(classic_find_end)->Apply(common_args); BENCHMARK(classic_find_end)->Apply(common_args); +BENCHMARK(classic_find_end)->Apply(common_args); +BENCHMARK(classic_find_end)->Apply(common_args); BENCHMARK(ranges_find_end)->Apply(common_args); BENCHMARK(ranges_find_end)->Apply(common_args); +BENCHMARK(ranges_find_end)->Apply(common_args); +BENCHMARK(ranges_find_end)->Apply(common_args); BENCHMARK(member_rfind)->Apply(common_args); BENCHMARK(member_rfind)->Apply(common_args); +BENCHMARK(member_rfind)->Apply(common_args); BENCHMARK_MAIN(); diff --git a/stl/inc/__msvc_string_view.hpp b/stl/inc/__msvc_string_view.hpp index d6e37a6d46f..701ea459a5a 100644 --- a/stl/inc/__msvc_string_view.hpp +++ b/stl/inc/__msvc_string_view.hpp @@ -734,7 +734,7 @@ constexpr size_t _Traits_find(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits> } #if _USE_STD_VECTOR_ALGORITHMS - if constexpr (_Is_implementation_handled_char_traits<_Traits> && sizeof(typename _Traits::char_type) <= 2) { + if constexpr (_Is_implementation_handled_char_traits<_Traits>) { if (!_STD _Is_constant_evaluated()) { const auto _End = _Haystack + _Hay_size; const auto _Ptr = _STD _Search_vectorized(_Haystack + _Start_at, _End, _Needle, _Needle_size); @@ -808,7 +808,7 @@ constexpr size_t _Traits_rfind(_In_reads_(_Hay_size) const _Traits_ptr_t<_Traits const size_t _Actual_start_at = (_STD min)(_Start_at, _Hay_size - _Needle_size); #if _USE_STD_VECTOR_ALGORITHMS - if constexpr (_Is_implementation_handled_char_traits<_Traits> && sizeof(typename _Traits::char_type) <= 2) { + if constexpr (_Is_implementation_handled_char_traits<_Traits>) { if (!_STD _Is_constant_evaluated()) { // _Find_end_vectorized takes into account the needle length when locating the search start. // As a potentially earlier start position can be specified, we need to take it into account, diff --git a/stl/inc/xutility b/stl/inc/xutility index f3660728b5f..6ceda19327e 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -107,11 +107,20 @@ const void* __stdcall __std_search_1( const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; const void* __stdcall __std_search_2( const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; +const void* __stdcall __std_search_4( + const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; +const void* __stdcall __std_search_8( + const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; const void* __stdcall __std_find_end_1( const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; const void* __stdcall __std_find_end_2( const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; +const void* __stdcall __std_find_end_4( + const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; +const void* __stdcall __std_find_end_8( + const void* _First1, const void* _Last1, const void* _First2, size_t _Count2) noexcept; + const void* __stdcall __std_min_element_1(const void* _First, const void* _Last, bool _Signed) noexcept; const void* __stdcall __std_min_element_2(const void* _First, const void* _Last, bool _Signed) noexcept; @@ -270,6 +279,10 @@ _Ty1* _Search_vectorized(_Ty1* const _First1, _Ty1* const _Last1, _Ty2* const _F return const_cast<_Ty1*>(static_cast(::__std_search_1(_First1, _Last1, _First2, _Count2))); } else if constexpr (sizeof(_Ty1) == 2) { return const_cast<_Ty1*>(static_cast(::__std_search_2(_First1, _Last1, _First2, _Count2))); + } else if constexpr (sizeof(_Ty1) == 4) { + return const_cast<_Ty1*>(static_cast(::__std_search_4(_First1, _Last1, _First2, _Count2))); + } else if constexpr (sizeof(_Ty1) == 8) { + return const_cast<_Ty1*>(static_cast(::__std_search_8(_First1, _Last1, _First2, _Count2))); } else { _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size } @@ -283,6 +296,10 @@ _Ty1* _Find_end_vectorized( return const_cast<_Ty1*>(static_cast(::__std_find_end_1(_First1, _Last1, _First2, _Count2))); } else if constexpr (sizeof(_Ty1) == 2) { return const_cast<_Ty1*>(static_cast(::__std_find_end_2(_First1, _Last1, _First2, _Count2))); + } else if constexpr (sizeof(_Ty1) == 4) { + return const_cast<_Ty1*>(static_cast(::__std_find_end_4(_First1, _Last1, _First2, _Count2))); + } else if constexpr (sizeof(_Ty1) == 8) { + return const_cast<_Ty1*>(static_cast(::__std_find_end_8(_First1, _Last1, _First2, _Count2))); } else { _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size } @@ -5502,8 +5519,7 @@ constexpr bool _Equal_memcmp_is_safe = // Can we activate the vector algorithms for std::search? template -constexpr bool _Vector_alg_in_search_is_safe = _Equal_memcmp_is_safe<_It1, _It2, _Pr> // can search bitwise - && sizeof(_Iter_value_t<_It1>) <= 2; // pcmpestri compatible element size +constexpr bool _Vector_alg_in_search_is_safe = _Equal_memcmp_is_safe<_It1, _It2, _Pr>; template _NODISCARD int _Memcmp_count(_CtgIt1 _First1, _CtgIt2 _First2, const size_t _Count) { diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0fb916c1318..f1f34fbd25b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -4927,7 +4927,77 @@ __declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_2(const namespace { namespace _Find_seq { - template +#ifdef _M_ARM64EC + using _Find_seq_traits_1 = void; + using _Find_seq_traits_2 = void; + using _Find_seq_traits_4 = void; + using _Find_seq_traits_8 = void; +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv + struct _Find_seq_traits_1 { + static __m256i _Broadcast_avx(const __m128i _Data) noexcept { + return _mm256_broadcastb_epi8(_Data); + } + + static unsigned long _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_movemask_epi8(_mm256_cmpeq_epi8(_Lhs, _Rhs)); + } + }; + + struct _Find_seq_traits_2 { + static __m256i _Broadcast_avx(const __m128i _Data) noexcept { + return _mm256_broadcastw_epi16(_Data); + } + + static unsigned long _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_movemask_epi8(_mm256_cmpeq_epi16(_Lhs, _Rhs)) & 0x55555555; + } + }; + + struct _Find_seq_traits_4 { + static __m256i _Broadcast_avx(const __m128i _Data) noexcept { + return _mm256_broadcastd_epi32(_Data); + } + + static unsigned long _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_movemask_epi8(_mm256_cmpeq_epi32(_Lhs, _Rhs)) & 0x11111111; + } + }; + + struct _Find_seq_traits_8 { + static __m256i _Broadcast_avx(const __m128i _Data) noexcept { + return _mm256_broadcastq_epi64(_Data); + } + + static unsigned long _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { + return _mm256_movemask_epi8(_mm256_cmpeq_epi64(_Lhs, _Rhs)) & 0x01010101; + } + }; + + template + __m256i _Avx2_load_tail(const void* const _Src, const size_t _Size_bytes, const __m256i _Mask) noexcept { + if constexpr (sizeof(_Ty) >= 4) { + return _mm256_maskload_epi32(reinterpret_cast(_Src), _Mask); + } else { + unsigned char _Tmp[32]; + memcpy(_Tmp, _Src, _Size_bytes); + return _mm256_loadu_si256(reinterpret_cast<__m256i*>(_Tmp)); + } + } + + template + __m256i _Avx2_load_tail(const void* const _Src, const size_t _Size_bytes) noexcept { + if constexpr (sizeof(_Ty) >= 4) { + const __m256i _Mask = _Avx2_tail_mask_32(_Size_bytes); + return _mm256_maskload_epi32(reinterpret_cast(_Src), _Mask); + } else { + unsigned char _Tmp[32]; + memcpy(_Tmp, _Src, _Size_bytes); + return _mm256_loadu_si256(reinterpret_cast<__m256i*>(_Tmp)); + } + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + + template const void* __stdcall _Search_impl( const void* _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { if (_Count2 == 0) { @@ -4935,7 +5005,7 @@ namespace { } if (_Count2 == 1) { - return _Finding::_Find_impl<_Traits, _Finding::_Predicate::_Equal>( + return _Finding::_Find_impl<_FindTraits, _Finding::_Predicate::_Equal>( _First1, _Last1, *static_cast(_First2)); } @@ -4947,137 +5017,259 @@ namespace { } #ifndef _M_ARM64EC - if (_Use_sse42() && _Size_bytes_1 >= 16) { - constexpr int _Op = (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ORDERED; - constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - - if (_Size_bytes_2 <= 16) { - const int _Size_el_2 = static_cast(_Size_bytes_2 / sizeof(_Ty)); + if (_Use_avx2() && _Size_bytes_1 >= 32) { + _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 - const int _Max_full_match_pos = _Part_size_el - _Size_el_2; - - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First2, _Size_bytes_2); - const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp2)); + if (_Size_bytes_2 <= 32) { + const __m256i _Mask2 = _Avx2_tail_mask_32(_Size_bytes_2); + const __m256i _Data2 = _Avx2_load_tail<_Ty>(_First2, _Size_bytes_2, _Mask2); + const __m256i _Start2 = _Traits::_Broadcast_avx(_mm256_castsi256_si128(_Data2)); const void* _Stop1 = _First1; - _Advance_bytes(_Stop1, _Size_bytes_1 - 16); - + _Advance_bytes(_Stop1, _Size_bytes_1 & ~size_t{0x1F}); do { - const __m128i _Data1 = _mm_loadu_si128(static_cast(_First1)); + const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); + unsigned long _Bingo = _Traits::_Cmp_avx(_Data1, _Start2); + + while (_Bingo != 0) { + const unsigned int _Pos = _tzcnt_u32(_Bingo); + + const void* _Match = _First1; + _Advance_bytes(_Match, _Pos); + + __m256i _Cmp; + if (const size_t _Left_match = _Byte_length(_Match, _Last1); _Left_match >= 32) { + const __m256i _Match_val = _mm256_loadu_si256(reinterpret_cast(_Match)); + _Cmp = _mm256_xor_si256(_Data2, _Match_val); + } else if (_Left_match >= _Size_bytes_2) { + const __m256i _Match_val = _Avx2_load_tail<_Ty>(_Match, _Left_match); + _Cmp = _mm256_xor_si256(_Data2, _Match_val); + } else { + break; + } - if (!_mm_cmpestrc(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op)) { - _Advance_bytes(_First1, 16); // No matches, next. - } else { - const int _Pos = _mm_cmpestri(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op); - _Advance_bytes(_First1, _Pos * sizeof(_Ty)); - if (_Pos <= _Max_full_match_pos) { - // Full match. Return this match. - return _First1; + if (_mm256_testz_si256(_Cmp, _Mask2)) { + return _Match; } - // Partial match. Search again from the match start. Will return it if it is full. + + _Bingo ^= 1 << _Pos; } - } while (_First1 <= _Stop1); - const size_t _Size_bytes_1_tail = _Byte_length(_First1, _Last1); - if (_Size_bytes_1_tail != 0) { - const int _Size_el_1_tail = static_cast(_Size_bytes_1_tail / sizeof(_Ty)); + _Advance_bytes(_First1, 32); - alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _First1, _Size_bytes_1_tail); - const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp1)); + } while (_First1 != _Stop1); - if (_mm_cmpestrc(_Data2, _Size_el_2, _Data1, _Size_el_1_tail, _Op)) { - const int _Pos = _mm_cmpestri(_Data2, _Size_el_2, _Data1, _Size_el_1_tail, _Op); - _Advance_bytes(_First1, _Pos * sizeof(_Ty)); - // Full match because size is less than 16. Return this match. - return _First1; + if (const size_t _Left1 = _Byte_length(_First1, _Last1); _Left1 >= _Size_bytes_2) { + const __m256i _Data1 = _Avx2_load_tail<_Ty>(_First1, _Left1); + unsigned long _Bingo = _Traits::_Cmp_avx(_Data1, _Start2); + + while (_Bingo != 0) { + const unsigned int _Pos = _tzcnt_u32(_Bingo); + + if (_Pos > _Left1 - _Size_bytes_2) { + break; + } + + const void* _Match = _First1; + _Advance_bytes(_Match, _Pos); + + const size_t _Left_match = _Byte_length(_Match, _Last1); + const __m256i _Match_val = _Avx2_load_tail<_Ty>(_Match, _Left_match); + const __m256i _Cmp = _mm256_xor_si256(_Data2, _Match_val); + + if (_mm256_testz_si256(_Cmp, _Mask2)) { + return _Match; + } + + _Bingo ^= 1 << _Pos; } } - } else { - const __m128i _Data2 = _mm_loadu_si128(reinterpret_cast(_First2)); + + return _Last1; + } else { // _Size_bytes_2 is greater than 32 bytes + const __m256i _Data2 = _mm256_loadu_si256(reinterpret_cast(_First2)); + const __m256i _Start2 = _Traits::_Broadcast_avx(_mm256_castsi256_si128(_Data2)); + const size_t _Max_pos = _Size_bytes_1 - _Size_bytes_2; const void* _Stop1 = _First1; _Advance_bytes(_Stop1, _Max_pos); const void* _Tail2 = _First2; - _Advance_bytes(_Tail2, 16); + _Advance_bytes(_Tail2, 32); do { - const __m128i _Data1 = _mm_loadu_si128(static_cast(_First1)); - if (!_mm_cmpestrc(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op)) { - _Advance_bytes(_First1, 16); // No matches, next. - } else { - const int _Pos = _mm_cmpestri(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); - - bool _Match_1st_16 = true; + const __m256i _Data1 = _mm256_loadu_si256(static_cast(_First1)); + unsigned long _Bingo = _Traits::_Cmp_avx(_Data1, _Start2); - if (_Pos != 0) { - _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + while (_Bingo != 0) { + const unsigned int _Pos = _tzcnt_u32(_Bingo); - if (_First1 > _Stop1) { - break; // Oops, doesn't fit - } + const void* _Match = _First1; + _Advance_bytes(_Match, _Pos); - // Match not from the first byte, check 16 symbols - const __m128i _Match1 = _mm_loadu_si128(static_cast(_First1)); - const __m128i _Cmp = _mm_xor_si128(_Data2, _Match1); - if (!_mm_testz_si128(_Cmp, _Cmp)) { - _Match_1st_16 = false; - } + if (_Match > _Stop1) { + break; // Oops, doesn't fit } - if (_Match_1st_16) { - const void* _Tail1 = _First1; - _Advance_bytes(_Tail1, 16); + const __m256i _Match_val = _mm256_loadu_si256(reinterpret_cast(_Match)); + const __m256i _Cmp = _mm256_xor_si256(_Data2, _Match_val); - if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 16) == 0) { - return _First1; + if (_mm256_testz_si256(_Cmp, _Cmp)) { + const void* _Tail1 = _Match; + _Advance_bytes(_Tail1, 32); + + if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 32) == 0) { + return _Match; } } - // Start from the next element - _Advance_bytes(_First1, sizeof(_Ty)); + _Bingo ^= 1 << _Pos; } + + _Advance_bytes(_First1, 32); + } while (_First1 <= _Stop1); + + return _Last1; } + } - return _Last1; - } else -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - { - const size_t _Max_pos = _Size_bytes_1 - _Size_bytes_2 + sizeof(_Ty); + if constexpr (sizeof(_Ty) <= 2) { + if (_Use_sse42() && _Size_bytes_1 >= 16) { + constexpr int _Op = + (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ORDERED; + constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - auto _Ptr1 = static_cast(_First1); - const auto _Ptr2 = static_cast(_First2); - const void* _Stop1 = _Ptr1; - _Advance_bytes(_Stop1, _Max_pos); + if (_Size_bytes_2 <= 16) { + const int _Size_el_2 = static_cast(_Size_bytes_2 / sizeof(_Ty)); - for (; _Ptr1 != _Stop1; ++_Ptr1) { - if (*_Ptr1 != *_Ptr2) { - continue; - } + const int _Max_full_match_pos = _Part_size_el - _Size_el_2; - bool _Equal = true; + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First2, _Size_bytes_2); + const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp2)); - for (size_t _Idx = 1; _Idx != _Count2; ++_Idx) { - if (_Ptr1[_Idx] != _Ptr2[_Idx]) { - _Equal = false; - break; + const void* _Stop1 = _First1; + _Advance_bytes(_Stop1, _Size_bytes_1 - 16); + + do { + const __m128i _Data1 = _mm_loadu_si128(static_cast(_First1)); + + if (!_mm_cmpestrc(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op)) { + _Advance_bytes(_First1, 16); // No matches, next. + } else { + const int _Pos = _mm_cmpestri(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + if (_Pos <= _Max_full_match_pos) { + // Full match. Return this match. + return _First1; + } + // Partial match. Search again from the match start. Will return it if it is full. + } + } while (_First1 <= _Stop1); + + const size_t _Size_bytes_1_tail = _Byte_length(_First1, _Last1); + if (_Size_bytes_1_tail != 0) { + const int _Size_el_1_tail = static_cast(_Size_bytes_1_tail / sizeof(_Ty)); + + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _First1, _Size_bytes_1_tail); + const __m128i _Data1 = _mm_load_si128(reinterpret_cast(_Tmp1)); + + if (_mm_cmpestrc(_Data2, _Size_el_2, _Data1, _Size_el_1_tail, _Op)) { + const int _Pos = _mm_cmpestri(_Data2, _Size_el_2, _Data1, _Size_el_1_tail, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + // Full match because size is less than 16. Return this match. + return _First1; + } } + + return _Last1; + } else { // _Size_bytes_2 is greater than 16 bytes + const __m128i _Data2 = _mm_loadu_si128(reinterpret_cast(_First2)); + const size_t _Max_pos = _Size_bytes_1 - _Size_bytes_2; + + const void* _Stop1 = _First1; + _Advance_bytes(_Stop1, _Max_pos); + + const void* _Tail2 = _First2; + _Advance_bytes(_Tail2, 16); + + do { + const __m128i _Data1 = _mm_loadu_si128(static_cast(_First1)); + if (!_mm_cmpestrc(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op)) { + _Advance_bytes(_First1, 16); // No matches, next. + } else { + const int _Pos = _mm_cmpestri(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); + + bool _Match_1st_16 = true; + + if (_Pos != 0) { + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + + if (_First1 > _Stop1) { + break; // Oops, doesn't fit + } + + // Match not from the first byte, check 16 symbols + const __m128i _Match1 = _mm_loadu_si128(static_cast(_First1)); + const __m128i _Cmp = _mm_xor_si128(_Data2, _Match1); + if (!_mm_testz_si128(_Cmp, _Cmp)) { + _Match_1st_16 = false; + } + } + + if (_Match_1st_16) { + const void* _Tail1 = _First1; + _Advance_bytes(_Tail1, 16); + + if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 16) == 0) { + return _First1; + } + } + + // Start from the next element + _Advance_bytes(_First1, sizeof(_Ty)); + } + } while (_First1 <= _Stop1); + + return _Last1; } + } + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ - if (_Equal) { - return _Ptr1; + const size_t _Max_pos = _Size_bytes_1 - _Size_bytes_2 + sizeof(_Ty); + + auto _Ptr1 = static_cast(_First1); + const auto _Ptr2 = static_cast(_First2); + const void* _Stop1 = _Ptr1; + _Advance_bytes(_Stop1, _Max_pos); + + for (; _Ptr1 != _Stop1; ++_Ptr1) { + if (*_Ptr1 != *_Ptr2) { + continue; + } + + bool _Equal = true; + + for (size_t _Idx = 1; _Idx != _Count2; ++_Idx) { + if (_Ptr1[_Idx] != _Ptr2[_Idx]) { + _Equal = false; + break; } } - return _Last1; + if (_Equal) { + return _Ptr1; + } } + + return _Last1; } - template + template const void* __stdcall _Find_end_impl(const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { if (_Count2 == 0) { @@ -5085,7 +5277,7 @@ namespace { } if (_Count2 == 1) { - return _Finding::_Find_last_impl<_Traits, _Finding::_Predicate::_Equal>( + return _Finding::_Find_last_impl<_FindTraits, _Finding::_Predicate::_Equal>( _First1, _Last1, *static_cast(_First2)); } @@ -5097,230 +5289,403 @@ namespace { } #ifndef _M_ARM64EC - if (_Use_sse42() && _Size_bytes_1 >= 16) { - constexpr int _Op = (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ORDERED; - constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - - static constexpr int8_t _Low_part_mask[] = {// - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + if (_Use_avx2() && _Size_bytes_1 >= 32) { + _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 - if (_Size_bytes_2 <= 16) { - const int _Size_el_2 = static_cast(_Count2); - constexpr unsigned int _Whole_mask = (1 << _Part_size_el) - 1; - const unsigned int _Needle_fit_mask = (1 << (_Part_size_el - _Size_el_2 + 1)) - 1; - const unsigned int _Needle_unfit_mask = _Whole_mask ^ _Needle_fit_mask; + if (_Size_bytes_2 <= 32) { + const unsigned int _Needle_fit_mask = (1 << (32 - _Size_bytes_2 + sizeof(_Ty))) - 1; const void* _Stop1 = _First1; - _Advance_bytes(_Stop1, _Size_bytes_1 & 0xF); + _Advance_bytes(_Stop1, _Size_bytes_1 & 0x1F); - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First2, _Size_bytes_2); - const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp2)); + const __m256i _Mask2 = _Avx2_tail_mask_32(_Size_bytes_2); + const __m256i _Data2 = _Avx2_load_tail<_Ty>(_First2, _Size_bytes_2, _Mask2); + const __m256i _Start2 = _Traits::_Broadcast_avx(_mm256_castsi256_si128(_Data2)); const void* _Mid1 = _Last1; - _Rewind_bytes(_Mid1, 16); + _Rewind_bytes(_Mid1, 32); + +#pragma warning(push) +#pragma warning(disable : 4324) // structure was padded due to alignment specifier + const auto _Check_first = [=, &_Mid1](long _Match) noexcept { + while (_Match != 0) { + const unsigned int _Pos = 31 - _lzcnt_u32(_Match); + + const void* _Tmp1 = _Mid1; + _Advance_bytes(_Tmp1, _Pos); - const auto _Check_fit = [&_Mid1, _Needle_fit_mask](const unsigned int _Match) noexcept { - const unsigned int _Fit_match = _Match & _Needle_fit_mask; - if (_Fit_match != 0) { - unsigned long _Match_last_pos; + const __m256i _Match_data = _Avx2_load_tail<_Ty>(_Tmp1, _Byte_length(_Tmp1, _Last1)); + const __m256i _Cmp_result = _mm256_xor_si256(_Data2, _Match_data); - // CodeQL [SM02313] Result is always initialized: we just tested that _Fit_match != 0. - _BitScanReverse(&_Match_last_pos, _Fit_match); + if (_mm256_testz_si256(_Cmp_result, _Mask2)) { + _Mid1 = _Tmp1; + return true; + } - _Advance_bytes(_Mid1, _Match_last_pos * sizeof(_Ty)); - return true; + _Match ^= 1 << _Pos; } return false; }; -#pragma warning(push) -#pragma warning(disable : 4324) // structure was padded due to alignment specifier - const auto _Check_unfit = [=, &_Mid1](const unsigned int _Match) noexcept { - long _Unfit_match = _Match & _Needle_unfit_mask; - while (_Unfit_match != 0) { - const void* _Tmp1 = _Mid1; - unsigned long _Match_last_pos; - - // CodeQL [SM02313] Result is always initialized: we just tested that _Unfit_match != 0. - _BitScanReverse(&_Match_last_pos, _Unfit_match); + const auto _Check = [=, &_Mid1](long _Match) noexcept { + while (_Match != 0) { + const unsigned int _Pos = 31 - _lzcnt_u32(_Match); - _Advance_bytes(_Tmp1, _Match_last_pos * sizeof(_Ty)); + const void* _Tmp1 = _Mid1; + _Advance_bytes(_Tmp1, _Pos); - const __m128i _Match_data = _mm_loadu_si128(reinterpret_cast(_Tmp1)); - const __m128i _Cmp_result = _mm_xor_si128(_Data2, _Match_data); - const __m128i _Data_mask = - _mm_loadu_si128(reinterpret_cast(_Low_part_mask + 16 - _Size_bytes_2)); + const __m256i _Match_data = _mm256_loadu_si256(reinterpret_cast(_Tmp1)); + const __m256i _Cmp_result = _mm256_xor_si256(_Data2, _Match_data); - if (_mm_testz_si128(_Cmp_result, _Data_mask)) { + if (_mm256_testz_si256(_Cmp_result, _Mask2)) { _Mid1 = _Tmp1; return true; } - _bittestandreset(&_Unfit_match, _Match_last_pos); + _Match ^= 1 << _Pos; } return false; }; #pragma warning(pop) - // TRANSITION, DevCom-10689455, the code below could test with _mm_cmpestrc, - // if it has been fused with _mm_cmpestrm. - // The very last part, for any match needle should fit, otherwise false match - const __m128i _Data1_last = _mm_loadu_si128(reinterpret_cast(_Mid1)); - const auto _Match_last = _mm_cmpestrm(_Data2, _Size_el_2, _Data1_last, _Part_size_el, _Op); - const unsigned int _Match_last_val = _mm_cvtsi128_si32(_Match_last); - if (_Check_fit(_Match_last_val)) { + const __m256i _Data1_last = _mm256_loadu_si256(reinterpret_cast(_Mid1)); + const unsigned long _Match_last_val = _Traits::_Cmp_avx(_Data1_last, _Start2); + if (_Check_first(_Match_last_val & _Needle_fit_mask)) { return _Mid1; } // The middle part, fit and unfit needle while (_Mid1 != _Stop1) { - _Rewind_bytes(_Mid1, 16); - const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); - const auto _Match = _mm_cmpestrm(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op); - const unsigned int _Match_val = _mm_cvtsi128_si32(_Match); - if (_Match_val != 0 && (_Check_unfit(_Match_val) || _Check_fit(_Match_val))) { + _Rewind_bytes(_Mid1, 32); + const __m256i _Data1 = _mm256_loadu_si256(reinterpret_cast(_Mid1)); + const unsigned long _Match_val = _Traits::_Cmp_avx(_Data1, _Start2); + if (_Check(_Match_val)) { return _Mid1; } } // The first part, fit and unfit needle, mask out already processed positions - if (const size_t _Tail_bytes_1 = _Size_bytes_1 & 0xF; _Tail_bytes_1 != 0) { - _Mid1 = _First1; - const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); - const auto _Match = _mm_cmpestrm(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op); - const unsigned int _Match_val = _mm_cvtsi128_si32(_Match) & ((1 << _Tail_bytes_1) - 1); - if (_Match_val != 0 && (_Check_unfit(_Match_val) || _Check_fit(_Match_val))) { + if (const size_t _Tail_bytes_1 = _Size_bytes_1 & 0x1F; _Tail_bytes_1 != 0) { + _Mid1 = _First1; + const __m256i _Data1 = _mm256_loadu_si256(reinterpret_cast(_Mid1)); + const unsigned long _Match_val = _Traits::_Cmp_avx(_Data1, _Start2); + if (_Match_val != 0 && _Check(_Match_val & ((1 << _Tail_bytes_1) - 1))) { return _Mid1; } } return _Last1; - } else { - const __m128i _Data2 = _mm_loadu_si128(reinterpret_cast(_First2)); + } else { // _Size_bytes_2 is greater than 32 bytes + const __m256i _Data2 = _mm256_loadu_si256(reinterpret_cast(_First2)); + const __m256i _Start2 = _Traits::_Broadcast_avx(_mm256_castsi256_si128(_Data2)); const void* _Tail2 = _First2; - _Advance_bytes(_Tail2, 16); + _Advance_bytes(_Tail2, 32); const void* _Mid1 = _Last1; _Rewind_bytes(_Mid1, _Size_bytes_2); const size_t _Size_diff_bytes = _Size_bytes_1 - _Size_bytes_2; const void* _Stop1 = _First1; - _Advance_bytes(_Stop1, _Size_diff_bytes & 0xF); + _Advance_bytes(_Stop1, _Size_diff_bytes & 0x1F); #pragma warning(push) #pragma warning(disable : 4324) // structure was padded due to alignment specifier const auto _Check = [=, &_Mid1](long _Match) noexcept { while (_Match != 0) { - const void* _Tmp1 = _Mid1; - unsigned long _Match_last_pos; - - // CodeQL [SM02313] Result is always initialized: we just tested that _Match != 0. - _BitScanReverse(&_Match_last_pos, _Match); - - bool _Match_1st_16 = true; - - if (_Match_last_pos != 0) { - _Advance_bytes(_Tmp1, _Match_last_pos * sizeof(_Ty)); + const unsigned int _Pos = 31 - _lzcnt_u32(_Match); - const __m128i _Match_data = _mm_loadu_si128(reinterpret_cast(_Tmp1)); - const __m128i _Cmp_result = _mm_xor_si128(_Data2, _Match_data); + const void* _Tmp1 = _Mid1; + _Advance_bytes(_Tmp1, _Pos); - if (!_mm_testz_si128(_Cmp_result, _Cmp_result)) { - _Match_1st_16 = false; - } - } + const __m256i _Match_data = _mm256_loadu_si256(reinterpret_cast(_Tmp1)); + const __m256i _Cmp_result = _mm256_xor_si256(_Data2, _Match_data); - if (_Match_1st_16) { + if (_mm256_testz_si256(_Cmp_result, _Cmp_result)) { const void* _Tail1 = _Tmp1; - _Advance_bytes(_Tail1, 16); + _Advance_bytes(_Tail1, 32); - if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 16) == 0) { + if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 32) == 0) { _Mid1 = _Tmp1; return true; } } - _bittestandreset(&_Match, _Match_last_pos); + _Match ^= 1 << _Pos; } return false; }; #pragma warning(pop) // The very last part, just compare, as true match must start with first symbol - const __m128i _Data1_last = _mm_loadu_si128(reinterpret_cast(_Mid1)); - const __m128i _Match_last = _mm_xor_si128(_Data2, _Data1_last); - if (_mm_testz_si128(_Match_last, _Match_last)) { - // Matched 16 bytes, check the rest + const __m256i _Data1_last = _mm256_loadu_si256(reinterpret_cast(_Mid1)); + const __m256i _Match_last = _mm256_xor_si256(_Data2, _Data1_last); + if (_mm256_testz_si256(_Match_last, _Match_last)) { + // Matched 32 bytes, check the rest const void* _Tail1 = _Mid1; - _Advance_bytes(_Tail1, 16); + _Advance_bytes(_Tail1, 32); - if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 16) == 0) { + if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 32) == 0) { return _Mid1; } } - // TRANSITION, DevCom-10689455, the code below could test with _mm_cmpestrc, - // if it has been fused with _mm_cmpestrm. - // The main part, match all characters while (_Mid1 != _Stop1) { - _Rewind_bytes(_Mid1, 16); + _Rewind_bytes(_Mid1, 32); - const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); - const auto _Match = _mm_cmpestrm(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); - const unsigned int _Match_val = _mm_cvtsi128_si32(_Match); - if (_Match_val != 0 && _Check(_Match_val)) { + const __m256i _Data1 = _mm256_loadu_si256(reinterpret_cast(_Mid1)); + const unsigned long _Match_val = _Traits::_Cmp_avx(_Data1, _Start2); + if (_Check(_Match_val)) { return _Mid1; } } // The first part, mask out already processed positions - if (const size_t _Tail_bytes_1 = _Size_diff_bytes & 0xF; _Tail_bytes_1 != 0) { - _Mid1 = _First1; - const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); - const auto _Match = _mm_cmpestrm(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); - const unsigned int _Match_val = _mm_cvtsi128_si32(_Match) & ((1 << _Tail_bytes_1) - 1); - if (_Match_val != 0 && _Check(_Match_val)) { + if (const size_t _Tail_bytes_1 = _Size_diff_bytes & 0x1F; _Tail_bytes_1 != 0) { + _Mid1 = _First1; + const __m256i _Data1 = _mm256_loadu_si256(reinterpret_cast(_Mid1)); + const unsigned long _Match_val = _Traits::_Cmp_avx(_Data1, _Start2); + if (_Match_val != 0 && _Check(_Match_val & ((1 << _Tail_bytes_1) - 1))) { return _Mid1; } } return _Last1; } - } else -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - { - auto _Ptr1 = static_cast(_Last1) - _Count2; - const auto _Ptr2 = static_cast(_First2); + } - for (;;) { - if (*_Ptr1 == *_Ptr2) { - bool _Equal = true; + if constexpr (sizeof(_Ty) <= 2) { + if (_Use_sse42() && _Size_bytes_1 >= 16) { + constexpr int _Op = + (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ORDERED; + constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - for (size_t _Idx = 1; _Idx != _Count2; ++_Idx) { - if (_Ptr1[_Idx] != _Ptr2[_Idx]) { - _Equal = false; - break; + static constexpr int8_t _Low_part_mask[] = {// + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; + + if (_Size_bytes_2 <= 16) { + const int _Size_el_2 = static_cast(_Count2); + constexpr unsigned int _Whole_mask = (1 << _Part_size_el) - 1; + const unsigned int _Needle_fit_mask = (1 << (_Part_size_el - _Size_el_2 + 1)) - 1; + const unsigned int _Needle_unfit_mask = _Whole_mask ^ _Needle_fit_mask; + + const void* _Stop1 = _First1; + _Advance_bytes(_Stop1, _Size_bytes_1 & 0xF); + + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First2, _Size_bytes_2); + const __m128i _Data2 = _mm_load_si128(reinterpret_cast(_Tmp2)); + + const void* _Mid1 = _Last1; + _Rewind_bytes(_Mid1, 16); + + const auto _Check_fit = [&_Mid1, _Needle_fit_mask](const unsigned int _Match) noexcept { + const unsigned int _Fit_match = _Match & _Needle_fit_mask; + if (_Fit_match != 0) { + unsigned long _Match_last_pos; + + // CodeQL [SM02313] Result is always initialized: we just tested that _Fit_match != 0. + _BitScanReverse(&_Match_last_pos, _Fit_match); + + _Advance_bytes(_Mid1, _Match_last_pos * sizeof(_Ty)); + return true; + } + + return false; + }; + +#pragma warning(push) +#pragma warning(disable : 4324) // structure was padded due to alignment specifier + const auto _Check_unfit = [=, &_Mid1](const unsigned int _Match) noexcept { + long _Unfit_match = _Match & _Needle_unfit_mask; + while (_Unfit_match != 0) { + const void* _Tmp1 = _Mid1; + unsigned long _Match_last_pos; + + // CodeQL [SM02313] Result is always initialized: we just tested that _Unfit_match != 0. + _BitScanReverse(&_Match_last_pos, _Unfit_match); + + _Advance_bytes(_Tmp1, _Match_last_pos * sizeof(_Ty)); + + const __m128i _Match_data = _mm_loadu_si128(reinterpret_cast(_Tmp1)); + const __m128i _Cmp_result = _mm_xor_si128(_Data2, _Match_data); + const __m128i _Data_mask = _mm_loadu_si128( + reinterpret_cast(_Low_part_mask + 16 - _Size_bytes_2)); + + if (_mm_testz_si128(_Cmp_result, _Data_mask)) { + _Mid1 = _Tmp1; + return true; + } + + _bittestandreset(&_Unfit_match, _Match_last_pos); + } + + return false; + }; +#pragma warning(pop) + + // TRANSITION, DevCom-10689455, the code below could test with _mm_cmpestrc, + // if it has been fused with _mm_cmpestrm. + + // The very last part, for any match needle should fit, otherwise false match + const __m128i _Data1_last = _mm_loadu_si128(reinterpret_cast(_Mid1)); + const auto _Match_last = _mm_cmpestrm(_Data2, _Size_el_2, _Data1_last, _Part_size_el, _Op); + const unsigned int _Match_last_val = _mm_cvtsi128_si32(_Match_last); + if (_Check_fit(_Match_last_val)) { + return _Mid1; + } + + // The middle part, fit and unfit needle + while (_Mid1 != _Stop1) { + _Rewind_bytes(_Mid1, 16); + const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); + const auto _Match = _mm_cmpestrm(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op); + const unsigned int _Match_val = _mm_cvtsi128_si32(_Match); + if (_Match_val != 0 && (_Check_unfit(_Match_val) || _Check_fit(_Match_val))) { + return _Mid1; } } - if (_Equal) { - return _Ptr1; + // The first part, fit and unfit needle, mask out already processed positions + if (const size_t _Tail_bytes_1 = _Size_bytes_1 & 0xF; _Tail_bytes_1 != 0) { + _Mid1 = _First1; + const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); + const auto _Match = _mm_cmpestrm(_Data2, _Size_el_2, _Data1, _Part_size_el, _Op); + const size_t _Tail_el_1 = _Tail_bytes_1 / sizeof(_Ty); + const unsigned int _Match_val = _mm_cvtsi128_si32(_Match) & ((1 << _Tail_el_1) - 1); + if (_Match_val != 0 && (_Check_unfit(_Match_val) || _Check_fit(_Match_val))) { + return _Mid1; + } + } + + return _Last1; + } else { // _Size_bytes_2 is greater than 16 bytes + const __m128i _Data2 = _mm_loadu_si128(reinterpret_cast(_First2)); + + const void* _Tail2 = _First2; + _Advance_bytes(_Tail2, 16); + + const void* _Mid1 = _Last1; + _Rewind_bytes(_Mid1, _Size_bytes_2); + + const size_t _Size_diff_bytes = _Size_bytes_1 - _Size_bytes_2; + const void* _Stop1 = _First1; + _Advance_bytes(_Stop1, _Size_diff_bytes & 0xF); + +#pragma warning(push) +#pragma warning(disable : 4324) // structure was padded due to alignment specifier + const auto _Check = [=, &_Mid1](long _Match) noexcept { + while (_Match != 0) { + const void* _Tmp1 = _Mid1; + unsigned long _Match_last_pos; + + // CodeQL [SM02313] Result is always initialized: we just tested that _Match != 0. + _BitScanReverse(&_Match_last_pos, _Match); + + bool _Match_1st_16 = true; + + if (_Match_last_pos != 0) { + _Advance_bytes(_Tmp1, _Match_last_pos * sizeof(_Ty)); + + const __m128i _Match_data = + _mm_loadu_si128(reinterpret_cast(_Tmp1)); + const __m128i _Cmp_result = _mm_xor_si128(_Data2, _Match_data); + + if (!_mm_testz_si128(_Cmp_result, _Cmp_result)) { + _Match_1st_16 = false; + } + } + + if (_Match_1st_16) { + const void* _Tail1 = _Tmp1; + _Advance_bytes(_Tail1, 16); + + if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 16) == 0) { + _Mid1 = _Tmp1; + return true; + } + } + + _bittestandreset(&_Match, _Match_last_pos); + } + + return false; + }; +#pragma warning(pop) + // The very last part, just compare, as true match must start with first symbol + const __m128i _Data1_last = _mm_loadu_si128(reinterpret_cast(_Mid1)); + const __m128i _Match_last = _mm_xor_si128(_Data2, _Data1_last); + if (_mm_testz_si128(_Match_last, _Match_last)) { + // Matched 16 bytes, check the rest + const void* _Tail1 = _Mid1; + _Advance_bytes(_Tail1, 16); + + if (memcmp(_Tail1, _Tail2, _Size_bytes_2 - 16) == 0) { + return _Mid1; + } + } + + // TRANSITION, DevCom-10689455, the code below could test with _mm_cmpestrc, + // if it has been fused with _mm_cmpestrm. + + // The main part, match all characters + while (_Mid1 != _Stop1) { + _Rewind_bytes(_Mid1, 16); + + const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); + const auto _Match = _mm_cmpestrm(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); + const unsigned int _Match_val = _mm_cvtsi128_si32(_Match); + if (_Match_val != 0 && _Check(_Match_val)) { + return _Mid1; + } + } + + // The first part, mask out already processed positions + if (const size_t _Tail_bytes_1 = _Size_diff_bytes & 0xF; _Tail_bytes_1 != 0) { + _Mid1 = _First1; + const __m128i _Data1 = _mm_loadu_si128(reinterpret_cast(_Mid1)); + const auto _Match = _mm_cmpestrm(_Data2, _Part_size_el, _Data1, _Part_size_el, _Op); + const size_t _Tail_el_1 = _Tail_bytes_1 / sizeof(_Ty); + const unsigned int _Match_val = _mm_cvtsi128_si32(_Match) & ((1 << _Tail_el_1) - 1); + if (_Match_val != 0 && _Check(_Match_val)) { + return _Mid1; + } } - } - if (_Ptr1 == _First1) { return _Last1; } + } + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + auto _Ptr1 = static_cast(_Last1) - _Count2; + const auto _Ptr2 = static_cast(_First2); + + for (;;) { + if (*_Ptr1 == *_Ptr2) { + bool _Equal = true; + + for (size_t _Idx = 1; _Idx != _Count2; ++_Idx) { + if (_Ptr1[_Idx] != _Ptr2[_Idx]) { + _Equal = false; + break; + } + } + + if (_Equal) { + return _Ptr1; + } + } - --_Ptr1; + if (_Ptr1 == _First1) { + return _Last1; } + + --_Ptr1; } } } // namespace _Find_seq @@ -5330,22 +5695,51 @@ extern "C" { const void* __stdcall __std_search_1( const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { - return _Find_seq::_Search_impl<_Finding::_Find_traits_1, uint8_t>(_First1, _Last1, _First2, _Count2); + return _Find_seq::_Search_impl<_Finding::_Find_traits_1, _Find_seq::_Find_seq_traits_1, uint8_t>( + _First1, _Last1, _First2, _Count2); } const void* __stdcall __std_search_2( const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { - return _Find_seq::_Search_impl<_Finding::_Find_traits_2, uint16_t>(_First1, _Last1, _First2, _Count2); + return _Find_seq::_Search_impl<_Finding::_Find_traits_2, _Find_seq::_Find_seq_traits_2, uint16_t>( + _First1, _Last1, _First2, _Count2); +} + +const void* __stdcall __std_search_4( + const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { + return _Find_seq::_Search_impl<_Finding::_Find_traits_4, _Find_seq::_Find_seq_traits_4, uint32_t>( + _First1, _Last1, _First2, _Count2); +} + +const void* __stdcall __std_search_8( + const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { + return _Find_seq::_Search_impl<_Finding::_Find_traits_8, _Find_seq::_Find_seq_traits_8, uint64_t>( + _First1, _Last1, _First2, _Count2); } + const void* __stdcall __std_find_end_1( const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { - return _Find_seq::_Find_end_impl<_Finding::_Find_traits_1, uint8_t>(_First1, _Last1, _First2, _Count2); + return _Find_seq::_Find_end_impl<_Finding::_Find_traits_1, _Find_seq::_Find_seq_traits_1, uint8_t>( + _First1, _Last1, _First2, _Count2); } const void* __stdcall __std_find_end_2( const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { - return _Find_seq::_Find_end_impl<_Finding::_Find_traits_2, uint16_t>(_First1, _Last1, _First2, _Count2); + return _Find_seq::_Find_end_impl<_Finding::_Find_traits_2, _Find_seq::_Find_seq_traits_2, uint16_t>( + _First1, _Last1, _First2, _Count2); +} + +const void* __stdcall __std_find_end_4( + const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { + return _Find_seq::_Find_end_impl<_Finding::_Find_traits_4, _Find_seq::_Find_seq_traits_4, uint32_t>( + _First1, _Last1, _First2, _Count2); +} + +const void* __stdcall __std_find_end_8( + const void* const _First1, const void* const _Last1, const void* const _First2, const size_t _Count2) noexcept { + return _Find_seq::_Find_end_impl<_Finding::_Find_traits_8, _Find_seq::_Find_seq_traits_8, uint64_t>( + _First1, _Last1, _First2, _Count2); } } // extern "C" diff --git a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp index fcd796d73a1..dd00180318d 100644 --- a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp +++ b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp @@ -46,27 +46,16 @@ int main() { assert(ranges::adjacent_find(arr_begin, arr_end) == arr_begin + _Signed128{5}); { - // As of 2025-05-09, 'search' and 'find_end' are manually vectorized for 8-bit and 16-bit elements only. - short short_arr[arr_size]; - picky_contiguous_iterator short_arr_begin(begin(short_arr)); - picky_contiguous_iterator short_arr_end(end(short_arr)); + const int needle[] = {300, 310, 320}; - transform(arr_begin, arr_end, short_arr_begin, [](int v) { return static_cast(v); }); - - const short short_needle[] = {300, 310, 320}; - - picky_contiguous_iterator short_needle_begin(begin(short_needle)); - picky_contiguous_iterator short_needle_end(end(short_needle)); + picky_contiguous_iterator needle_begin(begin(needle)); + picky_contiguous_iterator needle_end(end(needle)); - assert(search(short_arr_begin, short_arr_end, short_needle_begin, short_needle_end) - == short_arr_begin + _Signed128{10}); - assert(begin(ranges::search(short_arr_begin, short_arr_end, short_needle_begin, short_needle_end)) - == short_arr_begin + _Signed128{10}); + assert(search(arr_begin, arr_end, needle_begin, needle_end) == arr_begin + _Signed128{10}); + assert(begin(ranges::search(arr_begin, arr_end, needle_begin, needle_end)) == arr_begin + _Signed128{10}); - assert(find_end(short_arr_begin, short_arr_end, short_needle_begin, short_needle_end) - == short_arr_begin + _Signed128{10}); - assert(begin(ranges::find_end(short_arr_begin, short_arr_end, short_needle_begin, short_needle_end)) - == short_arr_begin + _Signed128{10}); + assert(find_end(arr_begin, arr_end, needle_begin, needle_end) == arr_begin + _Signed128{10}); + assert(begin(ranges::find_end(arr_begin, arr_end, needle_begin, needle_end)) == arr_begin + _Signed128{10}); } assert(count(arr_begin, arr_end, 250) == 6); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index d2f241334fb..35dac73c833 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -1112,7 +1112,10 @@ void test_vector_algorithms(mt19937_64& gen) { test_search(gen); test_search(gen); test_search(gen); - // search() and find_end() are vectorized for 1 and 2 bytes only. + test_search(gen); + test_search(gen); + test_search(gen); + test_search(gen); test_min_max_element(gen); test_min_max_element(gen); From d6155d4764cd4ac4fa6e5635ee0c2014392d89e4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 17 May 2025 06:33:08 +0300 Subject: [PATCH 04/16] `vector_algorithms.cpp`: further cleanup (#5485) --- stl/src/vector_algorithms.cpp | 223 ++++++++++++++-------------------- 1 file changed, 88 insertions(+), 135 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f1f34fbd25b..6261e16de87 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -503,50 +503,6 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( namespace { namespace _Sorting { - template - const void* _Min_tail(const void* const _First, const void* const _Last, const void* _Res, _Ty _Cur) noexcept { - for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { - if (*_Ptr < _Cur) { - _Res = _Ptr; - _Cur = *_Ptr; - } - } - - return _Res; - } - - template - const void* _Max_tail(const void* const _First, const void* const _Last, const void* _Res, _Ty _Cur) noexcept { - for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { - if (_Cur < *_Ptr) { - _Res = _Ptr; - _Cur = *_Ptr; - } - } - - return _Res; - } - - template - _Min_max_element_t _Both_tail(const void* const _First, const void* const _Last, _Min_max_element_t& _Res, - _Ty _Cur_min, _Ty _Cur_max) noexcept { - for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { - if (*_Ptr < _Cur_min) { - _Res._Min = _Ptr; - _Cur_min = *_Ptr; - } - // Not else! - // * Needed for correctness if start with maximum, as we don't handle specially the first element. - // * Promote branchless code generation. - if (_Cur_max <= *_Ptr) { - _Res._Max = _Ptr; - _Cur_max = *_Ptr; - } - } - - return _Res; - } - enum _Min_max_mode { _Mode_min = 1 << 0, _Mode_max = 1 << 1, @@ -1837,6 +1793,50 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; + template + const void* _Min_tail(const void* const _First, const void* const _Last, const void* _Res, _Ty _Cur) noexcept { + for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { + if (*_Ptr < _Cur) { + _Res = _Ptr; + _Cur = *_Ptr; + } + } + + return _Res; + } + + template + const void* _Max_tail(const void* const _First, const void* const _Last, const void* _Res, _Ty _Cur) noexcept { + for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { + if (_Cur < *_Ptr) { + _Res = _Ptr; + _Cur = *_Ptr; + } + } + + return _Res; + } + + template + _Min_max_element_t _Both_tail(const void* const _First, const void* const _Last, _Min_max_element_t& _Res, + _Ty _Cur_min, _Ty _Cur_max) noexcept { + for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { + if (*_Ptr < _Cur_min) { + _Res._Min = _Ptr; + _Cur_min = *_Ptr; + } + // Not else! + // * Needed for correctness if start with maximum, as we don't handle specially the first element. + // * Promote branchless code generation. + if (_Cur_max <= *_Ptr) { + _Res._Max = _Ptr; + _Cur_max = *_Ptr; + } + } + + return _Res; + } + template <_Min_max_mode _Mode, class _Traits> auto _Minmax_element_impl(const void* _First, const void* const _Last, const bool _Sign) noexcept { _Min_max_element_t _Res = {_First, _First}; @@ -3104,31 +3104,20 @@ namespace { uint64_t _MskX = uint64_t{_Carry} | (uint64_t{_Mask} << 32); + _MskX = (_MskX >> sizeof(_Ty)) & _MskX; + if constexpr (sizeof(_Ty) == 1) { - _MskX = (_MskX >> 1) & _MskX; _MskX = __ull_rshift(_MskX, _Sh1) & _MskX; } - if constexpr (sizeof(_Ty) == 2) { - _MskX = (_MskX >> 2) & _MskX; - } - if constexpr (sizeof(_Ty) < 4) { _MskX = __ull_rshift(_MskX, _Sh2) & _MskX; } - if constexpr (sizeof(_Ty) == 4) { - _MskX = (_MskX >> 4) & _MskX; - } - if constexpr (sizeof(_Ty) < 8) { _MskX = __ull_rshift(_MskX, _Sh3) & _MskX; } - if constexpr (sizeof(_Ty) == 8) { - _MskX = (_MskX >> 8) & _MskX; - } - if (_MskX != 0) { #ifdef _M_IX86 const uint32_t _MskLow = static_cast(_MskX); @@ -5746,54 +5735,7 @@ const void* __stdcall __std_find_end_8( namespace { namespace _Mismatching { -#ifdef _M_ARM64EC - using _Traits_1 = void; - using _Traits_2 = void; - using _Traits_4 = void; - using _Traits_8 = void; -#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv - struct _Traits_1 { - static __m256i _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_cmpeq_epi8(_Lhs, _Rhs); - } - - static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_cmpeq_epi8(_Lhs, _Rhs); - } - }; - - struct _Traits_2 { - static __m256i _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_cmpeq_epi16(_Lhs, _Rhs); - } - - static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_cmpeq_epi16(_Lhs, _Rhs); - } - }; - - struct _Traits_4 { - static __m256i _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_cmpeq_epi32(_Lhs, _Rhs); - } - - static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_cmpeq_epi32(_Lhs, _Rhs); - } - }; - - struct _Traits_8 { - static __m256i _Cmp_avx(const __m256i _Lhs, const __m256i _Rhs) noexcept { - return _mm256_cmpeq_epi64(_Lhs, _Rhs); - } - - static __m128i _Cmp_sse(const __m128i _Lhs, const __m128i _Rhs) noexcept { - return _mm_cmpeq_epi64(_Lhs, _Rhs); - } - }; -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - template + template __declspec(noalias) size_t __stdcall _Mismatch_impl( const void* const _First1, const void* const _First2, const size_t _Count) noexcept { size_t _Result = 0; @@ -5810,7 +5752,7 @@ namespace { for (; _Result != _Count_bytes_avx_full; _Result += 0x20) { const __m256i _Elem1 = _mm256_loadu_si256(reinterpret_cast(_First1_ch + _Result)); const __m256i _Elem2 = _mm256_loadu_si256(reinterpret_cast(_First2_ch + _Result)); - const __m256i _Cmp = _Traits::_Cmp_avx(_Elem1, _Elem2); + const __m256i _Cmp = _mm256_cmpeq_epi8(_Elem1, _Elem2); const auto _Bingo = ~static_cast(_mm256_movemask_epi8(_Cmp)); if (_Bingo != 0) { return (_Result + _tzcnt_u32(_Bingo)) / sizeof(_Ty); @@ -5826,7 +5768,7 @@ namespace { const __m256i _Elem2 = _mm256_maskload_epi32(reinterpret_cast(_First2_ch + _Result), _Tail_mask); - const __m256i _Cmp = _Traits::_Cmp_avx(_Elem1, _Elem2); + const __m256i _Cmp = _mm256_cmpeq_epi8(_Elem1, _Elem2); const auto _Bingo = ~static_cast(_mm256_movemask_epi8(_Cmp)); if (_Bingo != 0) { return (_Result + _tzcnt_u32(_Bingo)) / sizeof(_Ty); @@ -5846,7 +5788,7 @@ namespace { for (; _Result != _Count_bytes_sse; _Result += 0x10) { const __m128i _Elem1 = _mm_loadu_si128(reinterpret_cast(_First1_ch + _Result)); const __m128i _Elem2 = _mm_loadu_si128(reinterpret_cast(_First2_ch + _Result)); - const __m128i _Cmp = _Traits::_Cmp_sse(_Elem1, _Elem2); + const __m128i _Cmp = _mm_cmpeq_epi8(_Elem1, _Elem2); const auto _Bingo = static_cast(_mm_movemask_epi8(_Cmp)) ^ 0xFFFF; if (_Bingo != 0) { unsigned long _Offset; @@ -5877,22 +5819,22 @@ extern "C" { __declspec(noalias) size_t __stdcall __std_mismatch_1( const void* const _First1, const void* const _First2, const size_t _Count) noexcept { - return _Mismatching::_Mismatch_impl<_Mismatching::_Traits_1, uint8_t>(_First1, _First2, _Count); + return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } __declspec(noalias) size_t __stdcall __std_mismatch_2( const void* const _First1, const void* const _First2, const size_t _Count) noexcept { - return _Mismatching::_Mismatch_impl<_Mismatching::_Traits_2, uint16_t>(_First1, _First2, _Count); + return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } __declspec(noalias) size_t __stdcall __std_mismatch_4( const void* const _First1, const void* const _First2, const size_t _Count) noexcept { - return _Mismatching::_Mismatch_impl<_Mismatching::_Traits_4, uint32_t>(_First1, _First2, _Count); + return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } __declspec(noalias) size_t __stdcall __std_mismatch_8( const void* const _First1, const void* const _First2, const size_t _Count) noexcept { - return _Mismatching::_Mismatch_impl<_Mismatching::_Traits_8, uint64_t>(_First1, _First2, _Count); + return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } __declspec(noalias) void __stdcall __std_replace_4( @@ -6705,7 +6647,25 @@ namespace { using _Traits_2_avx = void; using _Traits_2_sse = void; #else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv - struct _Traits_1_avx { + struct _Traits_avx { + static void _Out(void* const _Dest, const __m256i _Elems) noexcept { + _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Elems); + } + + static void _Exit_vectorized() noexcept { + _mm256_zeroupper(); + } + }; + + struct _Traits_sse { + static void _Out(void* const _Dest, const __m128i _Elems) noexcept { + _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Elems); + } + + static void _Exit_vectorized() noexcept {} + }; + + struct _Traits_1_avx : _Traits_avx { using _Value_type = uint32_t; static __m256i _Set(const char _Val) noexcept { @@ -6723,13 +6683,9 @@ namespace { const __m256i _Ex1 = _mm256_blendv_epi8(_Px1, _Px0, _Ex0); return _Ex1; } - - static void _Out(void* const _Dest, const __m256i _Elems) noexcept { - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Elems); - } }; - struct _Traits_1_sse { + struct _Traits_1_sse : _Traits_sse { using _Value_type = uint16_t; static __m128i _Set(const char _Val) noexcept { @@ -6745,13 +6701,9 @@ namespace { const __m128i _Ex1 = _mm_blendv_epi8(_Px1, _Px0, _Ex0); return _Ex1; } - - static void _Out(void* const _Dest, const __m128i _Elems) noexcept { - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Elems); - } }; - struct _Traits_2_avx { + struct _Traits_2_avx : _Traits_avx { using _Value_type = uint16_t; static __m256i _Set(const wchar_t _Val) noexcept { @@ -6770,13 +6722,9 @@ namespace { const __m256i _Ex1 = _mm256_blendv_epi8(_Px1, _Px0, _Ex0); return _Ex1; } - - static void _Out(void* const _Dest, const __m256i _Elems) noexcept { - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Elems); - } }; - struct _Traits_2_sse { + struct _Traits_2_sse : _Traits_sse { using _Value_type = uint8_t; static __m128i _Set(const wchar_t _Val) noexcept { @@ -6790,10 +6738,6 @@ namespace { const __m128i _Ex1 = _mm_blendv_epi8(_Px1, _Px0, _Ex0); return _Ex1; } - - static void _Out(void* const _Dest, const __m128i _Elems) noexcept { - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Elems); - } }; template @@ -6827,6 +6771,8 @@ namespace { const _Elem* const _Tmpd = _Tmp + (_Step_size_bits - _Size_bits); memcpy(_Dest, _Tmpd, _Size_bits * sizeof(_Elem)); } + + _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 } #endif // ^^^ !defined(_M_ARM64EC) ^^^ @@ -6836,7 +6782,6 @@ namespace { #ifndef _M_ARM64EC if (_Use_avx2() && _Size_bits >= 256) { _Impl<_Avx_traits>(_Dest, _Src, _Size_bits, _Elem0, _Elem1); - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 } else if (_Use_sse42()) { _Impl<_Sse_traits>(_Dest, _Src, _Size_bits, _Elem0, _Elem1); } else @@ -6889,6 +6834,10 @@ namespace { static bool _Check(const __m256i _Val, const __m256i _Ex1, const __m256i _Dx0) noexcept { return _mm256_testc_si256(_Ex1, _mm256_xor_si256(_Val, _Dx0)); } + + static void _Exit_vectorized() noexcept { + _mm256_zeroupper(); + } }; struct _Traits_sse { @@ -6905,6 +6854,8 @@ namespace { static bool _Check(const __m128i _Val, const __m128i _Ex1, const __m128i _Dx0) noexcept { return _mm_testc_si128(_Ex1, _mm_xor_si128(_Val, _Dx0)); } + + static void _Exit_vectorized() noexcept {} }; struct _Traits_1_avx : _Traits_avx { @@ -7035,12 +6986,14 @@ namespace { // Convert characters to bits if (!_Loop<_Traits>(_Src, _Src + _Size_convert, _Dx0, _Dx1, _Out)) { + _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 return false; } // Verify remaining characters, if any if (_Size_convert != _Size_chars && !_Loop<_Traits>(_Src + _Size_convert, _Src + _Size_chars, _Dx0, _Dx1, [](_Traits::_Vec) {})) { + _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 return false; } @@ -7049,6 +7002,8 @@ namespace { memset(_Dst_words, 0, _Byte_length(_Dst_words, _Dst_words_end)); } + _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 + return true; } #endif // ^^^ !defined(_M_ARM64EC) ^^^ @@ -7089,8 +7044,6 @@ namespace { _Elem _Elem0, _Elem _Elem1) noexcept { #ifndef _M_ARM64EC if (_Use_avx2() && _Size_bits >= 256) { - _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 - return _Impl<_Avx>(_Dest, _Src, _Size_bytes, _Size_bits, _Size_chars, _Elem0, _Elem1); } else if (_Use_sse42()) { return _Impl<_Sse>(_Dest, _Src, _Size_bytes, _Size_bits, _Size_chars, _Elem0, _Elem1); From 2ce34fdd6a777b18a6ff2f4b712203d2cdefd3c7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 17 May 2025 06:35:34 +0300 Subject: [PATCH 05/16] Benchmark vectorized `reverse` and `reverse_copy`, use traits, optimize `reverse_copy` tail (#5493) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/reverse.cpp | 53 ++++ stl/src/vector_algorithms.cpp | 443 +++++++++++++--------------------- 3 files changed, 216 insertions(+), 281 deletions(-) create mode 100644 benchmarks/src/reverse.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index d9c45e5aceb..908b3640019 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -118,6 +118,7 @@ add_benchmark(priority_queue_push_range src/priority_queue_push_range.cpp) add_benchmark(random_integer_generation src/random_integer_generation.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) +add_benchmark(reverse src/reverse.cpp) add_benchmark(search src/search.cpp) add_benchmark(search_n src/search_n.cpp) add_benchmark(std_copy src/std_copy.cpp) diff --git a/benchmarks/src/reverse.cpp b/benchmarks/src/reverse.cpp new file mode 100644 index 00000000000..b4566a9aaa1 --- /dev/null +++ b/benchmarks/src/reverse.cpp @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include + +#include "skewed_allocator.hpp" +#include "utility.hpp" + +template +void r(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + auto v = random_vector(size); + + for (auto _ : state) { + benchmark::DoNotOptimize(v); + std::reverse(v.begin(), v.end()); + } +} + +template +void rc(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + auto v = random_vector(size); + std::vector> d(size); + + for (auto _ : state) { + benchmark::DoNotOptimize(v); + std::reverse_copy(v.begin(), v.end(), d.begin()); + benchmark::DoNotOptimize(d); + } +} + +void common_args(auto bm) { + bm->Arg(3449); + // AVX tail tests + bm->Arg(63)->Arg(31)->Arg(15)->Arg(7); +} + + +BENCHMARK(r)->Apply(common_args); +BENCHMARK(r)->Apply(common_args); +BENCHMARK(r)->Apply(common_args); +BENCHMARK(r)->Apply(common_args); + +BENCHMARK(rc)->Apply(common_args); +BENCHMARK(rc)->Apply(common_args); +BENCHMARK(rc)->Apply(common_args); +BENCHMARK(rc)->Apply(common_args); + +BENCHMARK_MAIN(); diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 6261e16de87..12558f8f3b4 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -165,6 +165,66 @@ void* __cdecl __std_swap_ranges_trivially_swappable( namespace { namespace _Reversing { +#ifdef _M_ARM64EC + using _Traits_1 = void; + using _Traits_2 = void; + using _Traits_4 = void; + using _Traits_8 = void; +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv + struct _Traits_1 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + + const __m256i _Perm = _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + return _mm256_shuffle_epi8(_Perm, _Reverse_char_lanes_avx); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); + return _mm_shuffle_epi8(_Val, _Reverse_char_sse); + } + }; + + struct _Traits_2 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // + 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // + 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + + const __m256i _Perm = _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + return _mm256_shuffle_epi8(_Perm, _Reverse_short_lanes_avx); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); + return _mm_shuffle_epi8(_Val, _Reverse_short_sse); + } + }; + + struct _Traits_4 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); + return _mm256_permutevar8x32_epi32(_Val, _Shuf); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + return _mm_shuffle_epi32(_Val, _MM_SHUFFLE(0, 1, 2, 3)); + } + }; + + struct _Traits_8 { + static __m256i _Rev_avx(const __m256i _Val) noexcept { + return _mm256_permute4x64_epi64(_Val, _MM_SHUFFLE(0, 1, 2, 3)); + } + + static __m128i _Rev_sse(const __m128i _Val) noexcept { + return _mm_shuffle_epi32(_Val, _MM_SHUFFLE(1, 0, 3, 2)); + } + }; +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + template void _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept { for (; _First != _Last && _First != --_Last; ++_First) { @@ -180,323 +240,144 @@ namespace { *_Dest++ = *--_Last; } } - } // namespace _Reversing -} // unnamed namespace -extern "C" { +#ifndef _M_ARM64EC + __m256i _Avx2_rev_tail_mask_32(const size_t _Count_in_bytes) noexcept { + // _Count_in_bytes must be within [0, 32]. + static constexpr unsigned int _Tail_masks[16] = { + 0, 0, 0, 0, 0, 0, 0, 0, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u, ~0u}; + return _mm256_loadu_si256(reinterpret_cast( + reinterpret_cast(_Tail_masks) + _Count_in_bytes)); + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ -__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { + template + __declspec(noalias) void __cdecl _Reverse_impl(void* _First, void* _Last) noexcept { #ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - // vpermq to load left and right, and transpose the lanes - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Right_perm = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(1, 0, 3, 2)); - // transpose all the chars in the lanes - const __m256i _Left_reversed = _mm256_shuffle_epi8(_Left_perm, _Reverse_char_lanes_avx); - const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_char_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); + if (const size_t _Length = _Byte_length(_First, _Last); _Length >= 64 && _Use_avx2()) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, (_Length >> 1) & ~size_t{0x1F}); + do { + _Advance_bytes(_Last, -32); + const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); + const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); + const __m256i _Left_reversed = _Traits::_Rev_avx(_Left); + const __m256i _Right_reversed = _Traits::_Rev_avx(_Right); + _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); + _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_char_sse); - const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_char_sse); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } + if (const size_t _Length = _Byte_length(_First, _Last); _Length >= 32 && _Use_sse42()) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, (_Length >> 1) & ~size_t{0xF}); + do { + _Advance_bytes(_Last, -16); + const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); + const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); + const __m128i _Left_reversed = _Traits::_Rev_sse(_Left); + const __m128i _Right_reversed = _Traits::_Rev_sse(_Right); + _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); + _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); + _Advance_bytes(_First, 16); + } while (_First != _Stop_at); + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); -} + _Reverse_tail(static_cast<_Ty*>(_First), static_cast<_Ty*>(_Last)); + } -__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { + template + __declspec(noalias) void __cdecl _Reverse_copy_impl( + const void* _First, const void* _Last, void* _Dest) noexcept { #ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Right_perm = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Left_reversed = _mm256_shuffle_epi8(_Left_perm, _Reverse_short_lanes_avx); - const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_short_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); + if (const size_t _Length = _Byte_length(_First, _Last); _Length >= 32 && _Use_avx2()) { + const void* _Stop_at = _Dest; + _Advance_bytes(_Stop_at, _Length & ~size_t{0x1F}); + do { + _Advance_bytes(_Last, -32); + const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); + const __m256i _Block_reversed = _Traits::_Rev_avx(_Block); + _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); + _Advance_bytes(_Dest, 32); + } while (_Dest != _Stop_at); + + if (const size_t _Avx_tail = _Length & 0x1C; _Avx_tail != 0) { + _Advance_bytes(_Last, -32); + const __m256i _Mask = _Avx2_tail_mask_32(_Avx_tail); + const __m256i _Rev_mask = _Avx2_rev_tail_mask_32(_Avx_tail); + const __m256i _Block = _mm256_maskload_epi32(static_cast(_Last), _Rev_mask); + const __m256i _Block_reversed = _Traits::_Rev_avx(_Block); + _mm256_maskstore_epi32(static_cast(_Dest), _Mask, _Block_reversed); + if constexpr (sizeof(_Ty) < 4) { + _Advance_bytes(_Dest, _Avx_tail); + _Advance_bytes(_Last, 32 - _Avx_tail); + } + } - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_short_sse); - const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_short_sse); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } + if constexpr (sizeof(_Ty) >= 4) { + return; + } + } else if (_Length >= 16 && _Use_sse42()) { + const void* _Stop_at = _Dest; + _Advance_bytes(_Stop_at, _Length & ~size_t{0xF}); + do { + _Advance_bytes(_Last, -16); + const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); + const __m128i _Block_reversed = _Traits::_Rev_sse(_Block); + _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); + _Advance_bytes(_Dest, 16); + } while (_Dest != _Stop_at); + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); -} + _Reverse_copy_tail( + static_cast(_First), static_cast(_Last), static_cast<_Ty*>(_Dest)); + } + } // namespace _Reversing +} // unnamed namespace -__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); - do { - _Advance_bytes(_Last, -32); - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_reversed = _mm256_permutevar8x32_epi32(_Left, _Shuf); - const __m256i _Right_reversed = _mm256_permutevar8x32_epi32(_Right, _Shuf); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); +extern "C" { - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } +__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { + _Reversing::_Reverse_impl<_Reversing::_Traits_1, uint8_t>(_First, _Last); +} - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(0, 1, 2, 3)); - const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(0, 1, 2, 3)); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { + _Reversing::_Reverse_impl<_Reversing::_Traits_2, uint16_t>(_First, _Last); +} - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); +__declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { + _Reversing::_Reverse_impl<_Reversing::_Traits_4, uint32_t>(_First, _Last); } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _First, void* _Last) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); - const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); - const __m256i _Left_reversed = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(0, 1, 2, 3)); - const __m256i _Right_reversed = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(0, 1, 2, 3)); - _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); - _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 32); - } while (_First != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, (_Byte_length(_First, _Last) >> 1) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); - const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); - const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(1, 0, 3, 2)); - const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(1, 0, 3, 2)); - _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); - _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); - _Advance_bytes(_First, 16); - } while (_First != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_tail(static_cast(_First), static_cast(_Last)); + _Reversing::_Reverse_impl<_Reversing::_Traits_8, uint64_t>(_First, _Last); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_char_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_char_sse); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), static_cast(_Last), - static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_1, uint8_t>(_First, _Last, _Dest); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // - 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); - const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_short_lanes_avx); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_short_sse); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), - static_cast(_Last), static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_2, uint16_t>(_First, _Last, _Dest); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_reversed = _mm256_permutevar8x32_epi32(_Block, _Shuf); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(0, 1, 2, 3)); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), static_cast(_Last), - static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_4, uint32_t>(_First, _Last, _Dest); } __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( const void* _First, const void* _Last, void* _Dest) noexcept { -#ifndef _M_ARM64EC - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0x1F}); - do { - _Advance_bytes(_Last, -32); - const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); - const __m256i _Block_reversed = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(0, 1, 2, 3)); - _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 32); - } while (_Dest != _Stop_at); - - _mm256_zeroupper(); // TRANSITION, DevCom-10331414 - } - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) & ~size_t{0xF}); - do { - _Advance_bytes(_Last, -16); - const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); - const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(1, 0, 3, 2)); - _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); - _Advance_bytes(_Dest, 16); - } while (_Dest != _Stop_at); - } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ - - _Reversing::_Reverse_copy_tail(static_cast(_First), - static_cast(_Last), static_cast(_Dest)); + _Reversing::_Reverse_copy_impl<_Reversing::_Traits_8, uint64_t>(_First, _Last, _Dest); } } // extern "C" From e9912d893be7b0509593f87ae43d656b416dab8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 17 May 2025 06:23:07 +0200 Subject: [PATCH 06/16] ``: Cache bitmasks of negated character classes during matching (#5487) --- stl/inc/regex | 102 ++++++++++++++++++-------------------------------- 1 file changed, 36 insertions(+), 66 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 3366345c261..aad62172c2a 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1657,6 +1657,17 @@ public: _Longest((_Re->_Flags & _Fl_longest) && !(_Mf & regex_constants::match_any)), _Traits(_Tr) { _Loop_vals.resize(_Re->_Loops); _Adl_verify_range(_Pfirst, _Plast); + if (_Re->_Flags & _Fl_begin_needs_w) { + _Char_class_w = _Lookup_char_class(static_cast<_Elem>('W')); + } + + if (_Re->_Flags & _Fl_begin_needs_s) { + _Char_class_s = _Lookup_char_class(static_cast<_Elem>('S')); + } + + if (_Re->_Flags & _Fl_begin_needs_d) { + _Char_class_d = _Lookup_char_class(static_cast<_Elem>('D')); + } } void _Setf(regex_constants::match_flag_type _Mf) { // set specified flags @@ -1739,7 +1750,7 @@ private: bool _Do_rep(_Node_rep*, bool, int); bool _Do_rep_first(_Node_rep*); bool _Find_first_inner_capture_group(_Node_base*, _Loop_vals_v2_t*); - bool _Do_class(_Node_base*); + _It _Do_class(_Node_base*, _It); bool _Match_pat(_Node_base*); bool _Better_match(); bool _Is_wbound() const; @@ -1758,6 +1769,9 @@ private: bool _Full; long _Max_complexity_count; long _Max_stack_count; + typename _RxTraits::char_class_type _Char_class_w{}; + typename _RxTraits::char_class_type _Char_class_s{}; + typename _RxTraits::char_class_type _Char_class_d{}; public: _Matcher2& operator=(const _Matcher2&) = delete; @@ -3679,9 +3693,10 @@ _BidIt _Lookup_coll2(_Elem _First_ch, _BidIt _First, const _BidIt _Last, const _ } template -bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_class(_Node_base* _Nx) { // apply bracket expression +_It _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_class(_Node_base* _Nx, _It _First) { + // apply bracket expression bool _Found; - _Elem _Ch = *_Tgt_state._Cur; + _Elem _Ch = *_First; if (_Sflags & regex_constants::icase) { _Ch = _Traits.translate_nocase(_Ch); } else if (_Sflags & regex_constants::collate) { @@ -3689,13 +3704,13 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_class(_Node_base* _Nx } const auto _UCh = static_cast(_Ch); - _It _Res0 = _Tgt_state._Cur; + _It _Res0 = _First; ++_Res0; _It _Resx; _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Nx); if (_Node->_Coll - && (_Resx = _STD _Lookup_coll2(_Ch, _Tgt_state._Cur, _End, _Node->_Coll, _Traits, _Sflags)) - != _Tgt_state._Cur) { // check for collation element + && (_Resx = _STD _Lookup_coll2(_Ch, _First, _End, _Node->_Coll, _Traits, _Sflags)) + != _First) { // check for collation element _Res0 = _Resx; _Found = true; } else if (_Node->_Ranges @@ -3715,14 +3730,11 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_class(_Node_base* _Nx _Found = true; } else if (_Node->_Equiv && _STD _Lookup_equiv2(_Ch, _Node->_Equiv, _Traits)) { _Found = true; - } else if ((_Node->_Flags & _Fl_class_negated_w) - && !_Traits.isctype(_Ch, _Lookup_char_class(static_cast<_Elem>('W')))) { + } else if ((_Node->_Flags & _Fl_class_negated_w) && !_Traits.isctype(_Ch, _Char_class_w)) { _Found = true; - } else if ((_Node->_Flags & _Fl_class_negated_s) - && !_Traits.isctype(_Ch, _Lookup_char_class(static_cast<_Elem>('S')))) { + } else if ((_Node->_Flags & _Fl_class_negated_s) && !_Traits.isctype(_Ch, _Char_class_s)) { _Found = true; - } else if ((_Node->_Flags & _Fl_class_negated_d) - && !_Traits.isctype(_Ch, _Lookup_char_class(static_cast<_Elem>('D')))) { + } else if ((_Node->_Flags & _Fl_class_negated_d) && !_Traits.isctype(_Ch, _Char_class_d)) { _Found = true; } else { _Found = false; @@ -3731,10 +3743,9 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_class(_Node_base* _Nx const bool _Negated = (_Node->_Flags & _Fl_negate) != 0; if (_Found == _Negated) { - return false; - } else { // record result - _Tgt_state._Cur = _Res0; - return true; + return _First; + } else { + return _Res0; } } @@ -3868,7 +3879,12 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N case _N_class: { // check for bracket expression match - _Failed = _Tgt_state._Cur == _End || !_Do_class(_Nx); + _It _Res; + if (_Tgt_state._Cur != _End && (_Res = _Do_class(_Nx, _Tgt_state._Cur)) != _Tgt_state._Cur) { + _Tgt_state._Cur = _Res; + } else { + _Failed = true; + } break; } @@ -4046,56 +4062,10 @@ _BidIt _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Skip(_BidIt _First_arg case _N_class: { // check for string match - for (; _First_arg != _Last; ++_First_arg) { // look for starting match - bool _Found; - _Elem _Ch = *_First_arg; - if (_Sflags & regex_constants::icase) { - _Ch = _Traits.translate_nocase(_Ch); - } else if (_Sflags & regex_constants::collate) { - _Ch = _Traits.translate(_Ch); - } - const auto _UCh = static_cast(_Ch); - - _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Nx); - - if (_Node->_Coll - && _STD _Lookup_coll2(_Ch, _First_arg, _Last, _Node->_Coll, _Traits, _Sflags) != _First_arg) { - _Found = true; - } else if (_Node->_Ranges - && (_Sflags & regex_constants::collate - ? _STD _Lookup_collating_range(_Ch, _Node->_Ranges, _Traits) - : _STD _Lookup_range(_UCh, _Node->_Ranges))) { - _Found = true; - } else if (_UCh < _Bmp_max) { - _Found = _Node->_Small && _Node->_Small->_Find(_UCh); - } else if (_Node->_Large - && _STD find(_Node->_Large->_Str(), _Node->_Large->_Str() + _Node->_Large->_Size(), _Ch) - != _Node->_Large->_Str() + _Node->_Large->_Size()) { - _Found = true; - } else if (_Node->_Classes != typename _RxTraits::char_class_type{} - && _Traits.isctype(_Ch, _Node->_Classes)) { - _Found = true; - } else if ((_Node->_Flags & _Fl_class_cl_all_bits) - && _Traits.isctype(_Ch, static_cast(-1))) { - _Found = true; - } else if (_Node->_Equiv && _STD _Lookup_equiv2(_Ch, _Node->_Equiv, _Traits)) { - _Found = true; - } else if ((_Node->_Flags & _Fl_class_negated_w) - && !_Traits.isctype(_Ch, _Lookup_char_class(static_cast<_Elem>('W')))) { - _Found = true; - } else if ((_Node->_Flags & _Fl_class_negated_s) - && !_Traits.isctype(_Ch, _Lookup_char_class(static_cast<_Elem>('S')))) { - _Found = true; - } else if ((_Node->_Flags & _Fl_class_negated_d) - && !_Traits.isctype(_Ch, _Lookup_char_class(static_cast<_Elem>('D')))) { - _Found = true; - } else { - _Found = false; - } + _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Nx); - const bool _Negated = (_Node->_Flags & _Fl_negate) != 0; - - if (_Found != _Negated) { + for (; _First_arg != _Last; ++_First_arg) { // look for starting match + if (_Do_class(_Node, _First_arg) != _First_arg) { return _First_arg; } } From 099280790ccd3622948da03b8863af5a81ea511d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 17 May 2025 06:26:16 +0200 Subject: [PATCH 07/16] ``: Reject empty repetitions when required by regex grammars (#5494) Co-authored-by: Stephan T. Lavavej --- stl/inc/regex | 217 ++++++++++-------- .../std/tests/VSO_0000000_regex_use/test.cpp | 192 ++++++++++++++++ tests/tr1/tests/regex2/test.cpp | 3 +- 3 files changed, 320 insertions(+), 92 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index aad62172c2a..8b359cfa7e8 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -119,6 +119,7 @@ namespace regex_constants { grep = 0x10, egrep = 0x20, _Gmask = 0x3F, + _Any_posix = basic | extended | grep | egrep | awk, icase = 0x0100, nosubs = 0x0200, @@ -3213,10 +3214,14 @@ void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_rep2(int _Min, int _Max, bool _Gre _Node_base* _Pos = _Current; if (_Pos->_Kind == _N_end_group || _Pos->_Kind == _N_end_capture) { _Pos = static_cast<_Node_end_group*>(_Pos)->_Back; - } - - if (_Min == 0 && _Max == 1) { // rewrite zero-or-one quantifiers as alternations to make the - // "simple loop" optimization more likely to engage + } else if (_Min == 0 && _Max == 1) { + // Rewrite zero-or-one quantifiers as alternations to make the + // "simple loop" optimization more likely to engage. + // + // GH-5490: This rewrite becomes observably incorrect + // if the subexpression contains capture groups, + // so we don't apply it if the subexpression is surrounded + // by a capturing or non-capturing group. _Node_endif* _End = new _Node_endif; _Node_if* _If_expr = new _Node_if(_End); _Node_if* _If_empty_str = new _Node_if(_End); @@ -3240,13 +3245,14 @@ void _Builder<_FwdIt, _Elem, _RxTraits>::_Add_rep2(int _Min, int _Max, bool _Gre swap(_If_expr->_Next->_Prev, _If_empty_str->_Next->_Prev); // intentional ADL swap(_If_expr->_Next, _If_empty_str->_Next); // intentional ADL } - } else { - _Node_end_rep* _Node0 = new _Node_end_rep(); - _Node_rep* _Nx = new _Node_rep(_Greedy, _Min, _Max, _Node0, _Root->_Loops++); - _Node0->_Begin_rep = _Nx; - _Link_node(_Node0); - _Insert_node(_Pos, _Nx); + return; } + + _Node_end_rep* _Node0 = new _Node_end_rep(); + _Node_rep* _Nx = new _Node_rep(_Greedy, _Min, _Max, _Node0, _Root->_Loops++); + _Node0->_Begin_rep = _Nx; + _Link_node(_Node0); + _Insert_node(_Pos, _Nx); } template @@ -3325,26 +3331,30 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_rep0(_Node_rep* _Node int _Ix = 0; _Tgt_state_t<_It> _St = _Tgt_state; - for (; _Ix < _Node->_Min; ++_Ix) { // do minimum number of reps - // GH-5365: We have to reset the capture groups from the second iteration on. - // We can avoid the reset for the first iteration - // because we know that a simple repetition was not encountered before. - if (_Ix > 0) { - _Tgt_state._Grp_valid = _St._Grp_valid; - } - - _It _Cur = _Tgt_state._Cur; + if (0 < _Node->_Min) { + // GH-5365: We can avoid resetting capture groups for the first iteration + // because we know that a simple repetition of this loop was not encountered before. if (!_Match_pat(_Node->_Next)) { // didn't match minimum number of reps, fail - _Tgt_state = _St; return false; - } else if (_Cur == _Tgt_state._Cur) { - _Ix = _Node->_Min - 1; // skip matches that don't change state + } else if (_Tgt_state._Cur == _St._Cur) { // matches empty string + // loop is branchless, so it will only ever match empty strings + // -> skip all other matches as they don't change state and immediately try tail + return _Match_pat(_Node->_End_rep->_Next); + } else { // loop never matches the empty string + for (_Ix = 1; _Ix < _Node->_Min; ++_Ix) { // do minimum number of reps + // GH-5365: We have to reset the capture groups from the second iteration on. + _Tgt_state._Grp_valid = _St._Grp_valid; + if (!_Match_pat(_Node->_Next)) { // didn't match minimum number of reps, fail + return false; + } + } } } - _Tgt_state_t<_It> _Final = _Tgt_state; - bool _Matched0 = false; - _It _Saved_pos = _Tgt_state._Cur; + _Tgt_state_t<_It> _Final; + bool _Matched0 = false; + _It _Saved_pos = _Tgt_state._Cur; + bool _Done = false; if (_Match_pat(_Node->_End_rep->_Next)) { if (!_Greedy) { @@ -3356,32 +3366,58 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_rep0(_Node_rep* _Node _Matched0 = true; } - while (_Node->_Max == -1 || _Ix++ < _Node->_Max) { // try another rep/tail match + if (_Ix == 0 && _Node->_Max != 0) { _Tgt_state._Cur = _Saved_pos; _Tgt_state._Grp_valid = _St._Grp_valid; - if (!_Match_pat(_Node->_Next)) { - break; // rep match failed, quit loop - } - _It _Mid = _Tgt_state._Cur; - if (_Match_pat(_Node->_End_rep->_Next)) { - if (!_Greedy) { + if (!_Match_pat(_Node->_Next)) { // rep match failed, we are done + _Done = true; + } else if (_Saved_pos == _Tgt_state._Cur) { // match empty, try no more repetitions + _Done = true; + // we only potentially accept/try tail for POSIX + if ((_Sflags & regex_constants::_Any_posix) && _Match_pat(_Node->_End_rep->_Next)) { return true; // go with current match } + } else { + _Saved_pos = _Tgt_state._Cur; + if (_Match_pat(_Node->_End_rep->_Next)) { + if (!_Greedy) { + return true; // go with current match + } - // record match and continue - _Final = _Tgt_state; - _Matched0 = true; + // record match and continue + _Final = _Tgt_state; + _Matched0 = true; + } } + _Ix = 1; + } - if (_Saved_pos == _Mid) { - break; // rep match ate no additional elements, quit loop - } + if (!_Done) { + while (_Node->_Max == -1 || _Ix++ < _Node->_Max) { // try another rep/tail match + _Tgt_state._Cur = _Saved_pos; + _Tgt_state._Grp_valid = _St._Grp_valid; + if (!_Match_pat(_Node->_Next) || _Tgt_state._Cur == _Saved_pos) { + break; // rep match failed, quit loop + } - _Saved_pos = _Mid; + // since loop is branchless, empty rep match is not possible at this point + _Saved_pos = _Tgt_state._Cur; + if (_Match_pat(_Node->_End_rep->_Next)) { + if (!_Greedy) { + return true; // go with current match + } + + // record match and continue + _Final = _Tgt_state; + _Matched0 = true; + } + } } - _Tgt_state = _Matched0 ? _Final : _St; + if (_Matched0) { // record final match + _Tgt_state = _Final; + } return _Matched0; } @@ -3395,61 +3431,56 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_rep(_Node_rep* _Node, _It* _Loop_iter_sav = static_cast<_It*>(_Psav->_Loop_iter); bool _Progress = _Init_idx == 0 || *_Loop_iter_sav != _St._Cur; - if (0 <= _Node->_Max && _Node->_Max <= _Init_idx) { - _Matched0 = _Match_pat(_Node->_End_rep->_Next); // reps done, try tail - } else if (_Init_idx < _Node->_Min) { // try a required rep - if (!_Progress) { - _Matched0 = _Match_pat(_Node->_End_rep->_Next); // empty, try tail - } else { // try another required match - _Psav->_Loop_idx = _Init_idx + 1; - _Psav->_Loop_iter = _STD addressof(_St._Cur); - _STD fill(_Tgt_state._Grp_valid.begin() + static_cast(_Psav->_Group_first), - _Tgt_state._Grp_valid.end(), false); - _Matched0 = _Match_pat(_Node->_Next); - } - } else if (_Longest) { // longest, try any number of repetitions - - // match with no further repetition - _Matched0 = _Match_pat(_Node->_End_rep->_Next); - // match with at least one more repetition if last repetition made progress - if (_Progress) { + if (_Init_idx < _Node->_Min) { // try another required match + _Psav->_Loop_iter = _STD addressof(_St._Cur); + _Psav->_Loop_idx = _Progress ? _Init_idx + 1 : _Node->_Min; // try only one more match after an empty match + _STD fill(_Tgt_state._Grp_valid.begin() + static_cast(_Psav->_Group_first), + _Tgt_state._Grp_valid.end(), false); + _Matched0 = _Match_pat(_Node->_Next); + } else if (_Init_idx == _Node->_Min || _Progress) { + if (0 <= _Node->_Max && _Node->_Max <= _Init_idx) { + _Matched0 = _Match_pat(_Node->_End_rep->_Next); // reps done, try tail + } else if (_Longest) { // longest, try any number of repetitions + + // match with no further repetition + _Matched0 = _Match_pat(_Node->_End_rep->_Next); + + // try to match with one more repetition _Tgt_state = _St; _Psav->_Loop_idx = _Init_idx + 1; _Psav->_Loop_iter = _STD addressof(_St._Cur); - if (_Match_pat(_Node->_Next)) { // always call _Match_pat, even when _Matched0 is already true _Matched0 = true; } - } - } else if (!_Greedy) { // not greedy, favor minimum number of reps - _Matched0 = _Match_pat(_Node->_End_rep->_Next); - if (!_Matched0 && _Progress) { // tail failed, try another rep - _Tgt_state = _St; - _Psav->_Loop_idx = _Init_idx + 1; - _Psav->_Loop_iter = _STD addressof(_St._Cur); - _STD fill(_Tgt_state._Grp_valid.begin() + static_cast(_Psav->_Group_first), - _Tgt_state._Grp_valid.end(), false); - _Matched0 = _Match_pat(_Node->_Next); - } - } else { // greedy, favor maximum number of reps - if (_Progress) { // try another rep + } else if (!_Greedy) { // not greedy, favor minimum number of reps + _Matched0 = _Match_pat(_Node->_End_rep->_Next); + if (!_Matched0) { // tail failed, try another rep + _Tgt_state = _St; + _Psav->_Loop_idx = _Init_idx + 1; + _Psav->_Loop_iter = _STD addressof(_St._Cur); + _STD fill(_Tgt_state._Grp_valid.begin() + static_cast(_Psav->_Group_first), + _Tgt_state._Grp_valid.end(), false); + _Matched0 = _Match_pat(_Node->_Next); + } + } else { // greedy, favor maximum number of reps, + // so try another rep _Psav->_Loop_idx = _Init_idx + 1; _Psav->_Loop_iter = _STD addressof(_St._Cur); _STD fill(_Tgt_state._Grp_valid.begin() + static_cast(_Psav->_Group_first), _Tgt_state._Grp_valid.end(), false); _Matched0 = _Match_pat(_Node->_Next); - } - if ((_Progress || 1 >= _Init_idx) && !_Matched0) { // rep failed, try tail - _Psav->_Loop_idx = _Loop_idx_sav; - _Psav->_Loop_iter = _Loop_iter_sav; - _Tgt_state = _St; - _Matched0 = _Match_pat(_Node->_End_rep->_Next); + if (!_Matched0) { // rep failed, try tail + _Psav->_Loop_idx = _Loop_idx_sav; + _Psav->_Loop_iter = _Loop_iter_sav; + _Tgt_state = _St; + _Matched0 = _Match_pat(_Node->_End_rep->_Next); + } } - } - - if (!_Matched0) { - _Tgt_state = _St; + } else if (_Init_idx == 1 && (_Sflags & regex_constants::_Any_posix)) { + // POSIX allows an empty repetition if the subexpression is matched only once, + // so try tail + _Matched0 = _Match_pat(_Node->_End_rep->_Next); } _Psav->_Loop_idx = _Loop_idx_sav; @@ -3470,9 +3501,7 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Do_rep_first(_Node_rep* // No capture group reset is performed for POSIX regexes, // so we prevent any reset by setting the first capture group to the number of capture groups _Ncap. if (_Psav->_Group_first == 0) { - constexpr auto _Any_posix = regex_constants::basic | regex_constants::extended | regex_constants::grep - | regex_constants::egrep | regex_constants::awk; - if ((_Sflags & _Any_posix) || !_Find_first_inner_capture_group(_Node->_Next, _Psav)) { + if ((_Sflags & regex_constants::_Any_posix) || !_Find_first_inner_capture_group(_Node->_Next, _Psav)) { _Psav->_Group_first = _Ncap; } } @@ -3844,10 +3873,8 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N if (_Tgt_state._Cur == _End) { _Failed = true; } else { - constexpr auto _Any_posix = regex_constants::basic | regex_constants::extended | regex_constants::grep - | regex_constants::egrep | regex_constants::awk; const _Elem _Ch = *_Tgt_state._Cur; - if (_Sflags & _Any_posix) { + if (_Sflags & regex_constants::_Any_posix) { if (_Ch == _Elem()) { _Failed = true; } @@ -4893,10 +4920,18 @@ void _Parser<_FwdIt, _Elem, _RxTraits>::_Calculate_loop_simplicity( _Calculate_loop_simplicity(static_cast<_Node_assert*>(_Nx)->_Child, nullptr, nullptr); break; case _N_rep: - // _Node_rep inside another _Node_rep makes both not simple + // _Node_rep inside another _Node_rep makes both not simple if _Outer_rep can be repeated more than once + // because _Matcher2::_Do_rep0() does not reset capture group boundaries when control is returned to it. + // If _Outer_rep can repeat at most once, we have to analyze the structure of the inner loop. if (_Outer_rep) { - _Outer_rep->_Simple_loop = 0; - static_cast<_Node_rep*>(_Nx)->_Simple_loop = 0; + _Outer_rep->_Simple_loop = 0; + auto _Inner_rep = static_cast<_Node_rep*>(_Nx); + if (_Outer_rep->_Max >= 0 && _Outer_rep->_Max <= 1) { + _Calculate_loop_simplicity(_Inner_rep->_Next, _Inner_rep->_End_rep->_Next, _Inner_rep); + _Nx = _Inner_rep->_End_rep; + } else { + _Inner_rep->_Simple_loop = 0; + } } else { _Outer_rep = static_cast<_Node_rep*>(_Nx); } diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index 36c8ef1b8ac..59c4f14cc99 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -1698,6 +1698,197 @@ void test_gh_5377() { } } +void test_gh_5490() { + // GH-5490: Optional empty repetitions are illegal + + // ECMA-262 15.10.2.5 "Term": + // "If min is zero and y's endIndex is equal to x's endIndex, then return failure." + // So if no additional repetition is required due to minimum requirements, the match should be rejected. + + // Similarly, POSIX 9.3.6 and 9.4.6 state that a null expression can only be matched if this is the only match or it + // is necessary to satisfy the minimum number of repetitions. + // Note the subtle difference that the empty match is allowed if it is the only match. + for (string pattern : {"()*", "()?", "()*?", "()??", "(){0,}", "(){0,}?", "(){0,1}", "(){0,1}?"}) { + test_regex quantified_empty_regex_ecma(&g_regexTester, pattern, ECMAScript); + quantified_empty_regex_ecma.should_search_match_capture_groups("", "", match_default, {{-1, -1}}); + quantified_empty_regex_ecma.should_search_match_capture_groups("b", "", match_default, {{-1, -1}}); + + // empty groups are not allowed in EREs + } + + for (string pattern : {"()+", "()+?", "(){2,}?", "(){1}"}) { + test_regex quantified_empty_regex_ecma(&g_regexTester, pattern, ECMAScript); + quantified_empty_regex_ecma.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + quantified_empty_regex_ecma.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + + // empty groups are not allowed in EREs + } + + for (auto option : {basic, grep}) { + test_regex quantified_empty_regex_bre(&g_regexTester, R"(\(\)*)", option); + quantified_empty_regex_bre.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + quantified_empty_regex_bre.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + } + + for (auto option : {ECMAScript, extended, egrep, awk}) { + test_regex simple_sequence_regex_ecma_or_ere(&g_regexTester, "(ab)*", option); + simple_sequence_regex_ecma_or_ere.should_search_match_capture_groups("", "", match_default, {{-1, -1}}); + simple_sequence_regex_ecma_or_ere.should_search_match_capture_groups("b", "", match_default, {{-1, -1}}); + simple_sequence_regex_ecma_or_ere.should_search_match_capture_groups("ababcc", "abab", match_default, {{2, 4}}); + } + + for (auto option : {basic, grep}) { + test_regex simple_sequence_regex_bre(&g_regexTester, R"(\(ab\)*)", option); + simple_sequence_regex_bre.should_search_match_capture_groups("", "", match_default, {{-1, -1}}); + simple_sequence_regex_bre.should_search_match_capture_groups("b", "", match_default, {{-1, -1}}); + simple_sequence_regex_bre.should_search_match_capture_groups("ababcc", "abab", match_default, {{2, 4}}); + } + + for (string pattern : {"(ab(?=ab))*", "(ab(?!cc))*"}) { + test_regex nested_assertion_regex_ecma(&g_regexTester, pattern, ECMAScript); + nested_assertion_regex_ecma.should_search_match_capture_groups("", "", match_default, {{-1, -1}}); + nested_assertion_regex_ecma.should_search_match_capture_groups("b", "", match_default, {{-1, -1}}); + nested_assertion_regex_ecma.should_search_match_capture_groups("ababcc", "ab", match_default, {{0, 2}}); + nested_assertion_regex_ecma.should_search_match_capture_groups("abababcc", "abab", match_default, {{2, 4}}); + } + + for (string pattern : {"(a*)*", "(a?)*", "(a?)?"}) { + test_regex nested_quantifier_regex_ecma(&g_regexTester, pattern, ECMAScript); + nested_quantifier_regex_ecma.should_search_match_capture_groups("", "", match_default, {{-1, -1}}); + nested_quantifier_regex_ecma.should_search_match_capture_groups("b", "", match_default, {{-1, -1}}); + nested_quantifier_regex_ecma.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + + for (auto option : {extended, egrep, awk}) { + test_regex nested_quantifier_regex_ere(&g_regexTester, pattern, option); + nested_quantifier_regex_ere.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + nested_quantifier_regex_ere.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + nested_quantifier_regex_ere.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {R"(\(a*\)*)", R"(\(a\{0,1\}\)*)", R"(\(a\{0,1\}\)\{0,1\})"}) { + for (auto option : {basic, grep}) { + test_regex nested_quantifier_regex_bre(&g_regexTester, pattern, option); + nested_quantifier_regex_bre.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + nested_quantifier_regex_bre.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + nested_quantifier_regex_bre.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {"(a*)+", "(a?)+"}) { + for (auto option : {ECMAScript, extended, egrep, awk}) { + test_regex plus_quantifier_regex_ecma_or_ere(&g_regexTester, pattern, option); + plus_quantifier_regex_ecma_or_ere.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + plus_quantifier_regex_ecma_or_ere.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + plus_quantifier_regex_ecma_or_ere.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {R"(\(a*\)\{1,\})", R"(\(a\{0,1\}\)\{1,\})"}) { + for (auto option : {basic, grep}) { + test_regex plus_quantifier_regex_bre(&g_regexTester, pattern, option); + plus_quantifier_regex_bre.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + plus_quantifier_regex_bre.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + plus_quantifier_regex_bre.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {"(a*){1}", "(a?){1}"}) { + for (auto option : {ECMAScript, extended, egrep, awk}) { + test_regex repeat_once_regex_ecma_or_ere(&g_regexTester, pattern, option); + repeat_once_regex_ecma_or_ere.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeat_once_regex_ecma_or_ere.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + repeat_once_regex_ecma_or_ere.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {R"(\(a*\)\{1\})", R"(\(a\{0,1\}\)\{1\})"}) { + for (auto option : {basic, grep}) { + test_regex repeat_once_regex_bre(&g_regexTester, pattern, option); + repeat_once_regex_bre.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeat_once_regex_bre.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + // leftmost-longest rule according to Boost semantics + repeat_once_regex_bre.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {"(a*){2}", "(a?){2}"}) { + test_regex repeat_twice_regex_ecma(&g_regexTester, pattern, ECMAScript); + repeat_twice_regex_ecma.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeat_twice_regex_ecma.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + repeat_twice_regex_ecma.should_search_match_capture_groups("a", "a", match_default, {{1, 1}}); + + for (auto option : {extended, egrep, awk}) { + test_regex repeat_twice_regex_ere(&g_regexTester, pattern, option); + repeat_twice_regex_ere.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeat_twice_regex_ere.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + // leftmost-longest rule according to Boost semantics + repeat_twice_regex_ere.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {R"(\(a*\)\{2\})", R"(\(a\{0,1\}\)\{2\})"}) { + for (auto option : {basic, grep}) { + test_regex repeat_twice_regex_bre(&g_regexTester, pattern, option); + repeat_twice_regex_bre.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeat_twice_regex_bre.should_search_match_capture_groups("b", "", match_default, {{0, 0}}); + // leftmost-longest rule according to Boost semantics + repeat_twice_regex_bre.should_search_match_capture_groups("a", "a", match_default, {{0, 1}}); + } + } + + for (string pattern : {"(a?a?){2}", "(a?a?)+"}) { + test_regex repeated_double_question_regex_ecma(&g_regexTester, pattern, ECMAScript); + repeated_double_question_regex_ecma.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeated_double_question_regex_ecma.should_search_match_capture_groups("bbb", "", match_default, {{0, 0}}); + repeated_double_question_regex_ecma.should_search_match_capture_groups("aaa", "aaa", match_default, {{2, 3}}); + + for (auto option : {extended, egrep, awk}) { + test_regex repeated_double_question_regex_ere(&g_regexTester, pattern, option); + repeated_double_question_regex_ere.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeated_double_question_regex_ere.should_search_match_capture_groups("bbb", "", match_default, {{0, 0}}); + // leftmost-longest rule according to Boost semantics + repeated_double_question_regex_ere.should_search_match_capture_groups( + "aaa", "aaa", match_default, {{1, 3}}); + } + } + + for (string pattern : {R"(\(a\{0,1\}a\{0,1\}\)\{2\})", R"(\(a\{0,1\}a\{0,1\}\)\{1,\})"}) { + for (auto option : {basic, grep}) { + test_regex repeated_double_question_regex_bre(&g_regexTester, pattern, option); + repeated_double_question_regex_bre.should_search_match_capture_groups("", "", match_default, {{0, 0}}); + repeated_double_question_regex_bre.should_search_match_capture_groups("bbb", "", match_default, {{0, 0}}); + // leftmost-longest rule according to Boost semantics + repeated_double_question_regex_bre.should_search_match_capture_groups( + "aaa", "aaa", match_default, {{1, 3}}); + } + } + + { + test_regex backref_ecma(&g_regexTester, R"(a(b?)+c\1d)", ECMAScript); + backref_ecma.should_search_fail("abcd"); + backref_ecma.should_search_match_capture_groups("acd", "acd", match_default, {{1, 1}}); + backref_ecma.should_search_match_capture_groups("abcbd", "abcbd", match_default, {{1, 2}}); + } + + for (auto option : {basic, grep}) { + test_regex backref_bre(&g_regexTester, R"(a\(b\{0,1\}\)\{1,\}c\1d)", option); + backref_bre.should_search_fail("abcd"); + backref_bre.should_search_match_capture_groups("acd", "acd", match_default, {{1, 1}}); + backref_bre.should_search_match_capture_groups("abcbd", "abcbd", match_default, {{1, 2}}); + } + + { + test_regex backref_min_repeat_ecma(&g_regexTester, R"((a?){3,4}b\1c)", ECMAScript); + backref_min_repeat_ecma.should_search_match_capture_groups("aabc", "aabc", match_default, {{2, 2}}); + } + + for (auto option : {basic, grep}) { + test_regex backref_min_repeat_bre(&g_regexTester, R"(\(a\{0,1\}\)\{3,4\}b\1c)", option); + backref_min_repeat_bre.should_search_match_capture_groups("aabc", "aabc", match_default, {{2, 2}}); + } +} + int main() { test_dev10_449367_case_insensitivity_should_work(); test_dev11_462743_regex_collate_should_not_disable_regex_icase(); @@ -1744,6 +1935,7 @@ int main() { test_gh_5371(); test_gh_5374(); test_gh_5377(); + test_gh_5490(); return g_regexTester.result(); } diff --git a/tests/tr1/tests/regex2/test.cpp b/tests/tr1/tests/regex2/test.cpp index d52371192de..83706bb66bc 100644 --- a/tests/tr1/tests/regex2/test.cpp +++ b/tests/tr1/tests/regex2/test.cpp @@ -741,7 +741,8 @@ static const regex_test tests[] = { {__LINE__, T("a(bbb+|bb+|b)b"), T("abbb"), "2 0 4 1 3", NOT_BG}, {__LINE__, T("a(bbb+|bb+|b)bb"), T("abbb"), "2 0 4 1 2", NOT_BG}, {__LINE__, T("(.*).*"), T("abcdef"), "2 0 6 0 6", NOT_BG}, - {__LINE__, T("(a*)*"), T("bc"), "2 0 0 0 0", NOT_BG}, + {__LINE__, T("(a*)*"), T("bc"), "2 0 0 -1 -1", ECMA}, + {__LINE__, T("(a*)*"), T("bc"), "2 0 0 0 0", EEA}, {__LINE__, T("a(b|c)*d"), T("ad"), "2 0 2 -1 -1", NOT_BG}, {__LINE__, T("a(b|c)*d"), T("abcd"), "2 0 4 2 3", NOT_BG}, {__LINE__, T("a(b|c)+d"), T("abd"), "2 0 3 1 2", NOT_BG}, From dfceacee7fa388220139ba1becbee41cddf5bab6 Mon Sep 17 00:00:00 2001 From: "A. Jiang" Date: Sat, 17 May 2025 12:41:13 +0800 Subject: [PATCH 08/16] `<__msvc_string_view.hpp>`: Avoid redundant forwarding in `char_traits::assign` (#5488) --- stl/inc/__msvc_string_view.hpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/stl/inc/__msvc_string_view.hpp b/stl/inc/__msvc_string_view.hpp index 701ea459a5a..e3f7b844bf0 100644 --- a/stl/inc/__msvc_string_view.hpp +++ b/stl/inc/__msvc_string_view.hpp @@ -418,11 +418,6 @@ struct _WChar_traits : private _Char_traits<_Elem, unsigned short> { } static _CONSTEXPR17 void assign(_Elem& _Left, const _Elem& _Right) noexcept { -#if _HAS_CXX20 - if (_STD is_constant_evaluated()) { - return _Primary_char_traits::assign(_Left, _Right); - } -#endif // _HAS_CXX20 _Left = _Right; } @@ -573,11 +568,6 @@ struct _Narrow_char_traits : private _Char_traits<_Elem, _Int_type> { } static _CONSTEXPR17 void assign(_Elem& _Left, const _Elem& _Right) noexcept { -#if _HAS_CXX20 - if (_STD is_constant_evaluated()) { - return _Primary_char_traits::assign(_Left, _Right); - } -#endif // _HAS_CXX20 _Left = _Right; } From c8ae40bc5f661c23c2e84cd4610b8a95ea715dc9 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 17 May 2025 07:43:37 +0300 Subject: [PATCH 09/16] Use `_Verify_ranges_do_not_overlap` in more places (#5495) Co-authored-by: Stephan T. Lavavej --- stl/inc/algorithm | 37 +++++++++++------ tests/std/test.lst | 1 + .../tests/GH_005472_do_not_overlap/env.lst | 4 ++ .../tests/GH_005472_do_not_overlap/test.cpp | 41 +++++++++++++++++++ 4 files changed, 70 insertions(+), 13 deletions(-) create mode 100644 tests/std/tests/GH_005472_do_not_overlap/env.lst create mode 100644 tests/std/tests/GH_005472_do_not_overlap/test.cpp diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 6d7e76ed4af..fe005e72d51 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -4452,6 +4452,7 @@ _EXPORT_STD template _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const _Ty& _Oldval, const _Ty& _Newval) { // copy replacing each matching _Oldval with _Newval _STD _Adl_verify_range(_First, _Last); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Dest); auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); @@ -4500,7 +4501,7 @@ namespace ranges { auto _UFirst = _RANGES _Unwrap_iter<_Se>(_STD move(_First)); auto _ULast = _RANGES _Unwrap_sent<_It>(_STD move(_Last)); const auto _Count = _RANGES _Idl_distance<_It>(_UFirst, _ULast); - auto _UResult = _Replace_copy_unchecked(_STD move(_UFirst), _STD move(_ULast), + auto _UResult = _Replace_copy_common(_STD move(_UFirst), _STD move(_ULast), _STD _Get_unwrapped_n(_STD move(_Output), _Count), _Oldval, _Newval, _STD _Pass_fn(_Proj)); _STD _Seek_wrapped(_First, _STD move(_UResult.in)); @@ -4515,8 +4516,8 @@ namespace ranges { _Out _Output, const _Ty1& _Oldval, const _Ty2& _Newval, _Pj _Proj = {}) _CONST_CALL_OPERATOR { const auto _Count = _RANGES _Idl_distance(_Range); auto _First = _RANGES begin(_Range); - auto _UResult = _Replace_copy_unchecked(_RANGES _Unwrap_range_iter<_Rng>(_STD move(_First)), _Uend(_Range), - _STD _Get_unwrapped_n(_STD move(_Output), _Count), _Oldval, _Newval, _STD _Pass_fn(_Proj)); + auto _UResult = _Replace_copy_common(_RANGES _Unwrap_range_iter<_Rng>(_STD move(_First)), _Uend(_Range), + _STD _Get_unwrapped_n(_STD move(_Output), _Count), _Oldval, _Newval, _STD _Pass_fn(_Proj)); _STD _Seek_wrapped(_First, _STD move(_UResult.in)); _STD _Seek_wrapped(_Output, _STD move(_UResult.out)); @@ -4525,7 +4526,7 @@ namespace ranges { private: template - _NODISCARD static constexpr replace_copy_result<_It, _Out> _Replace_copy_unchecked( + _NODISCARD static constexpr replace_copy_result<_It, _Out> _Replace_copy_common( _It _First, const _Se _Last, _Out _Output, const _Ty1& _Oldval, const _Ty2& _Newval, _Pj _Proj) { // copy [_First, _Last) to _Output while replacing projected _Oldval with _Newval _STL_INTERNAL_STATIC_ASSERT(input_iterator<_It>); @@ -4534,6 +4535,8 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(indirectly_copyable<_It, _Out>); _STL_INTERNAL_STATIC_ASSERT(indirect_binary_predicate, const _Ty1*>); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); + for (; _First != _Last; ++_First, (void) ++_Output) { if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty2>) { *_Output = _STD invoke(_Proj, *_First) == _Oldval ? _Newval : *_First; @@ -4558,6 +4561,7 @@ _EXPORT_STD template _CONSTEXPR20 _OutIt replace_copy_if(_InIt _First, _InIt _Last, _OutIt _Dest, _Pr _Pred, const _Ty& _Val) { // copy replacing each satisfying _Pred with _Val _STD _Adl_verify_range(_First, _Last); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Dest); auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); @@ -4607,7 +4611,7 @@ namespace ranges { auto _ULast = _RANGES _Unwrap_sent<_It>(_STD move(_Last)); const auto _Count = _RANGES _Idl_distance<_It>(_UFirst, _ULast); - auto _UResult = _Replace_copy_if_unchecked(_STD move(_UFirst), _STD move(_ULast), + auto _UResult = _Replace_copy_if_common(_STD move(_UFirst), _STD move(_ULast), _STD _Get_unwrapped_n(_STD move(_Output), _Count), _STD _Pass_fn(_Pred), _Newval, _STD _Pass_fn(_Proj)); _STD _Seek_wrapped(_First, _STD move(_UResult.in)); @@ -4622,9 +4626,8 @@ namespace ranges { _Rng&& _Range, _Out _Output, _Pr _Pred, const _Ty& _Newval, _Pj _Proj = {}) _CONST_CALL_OPERATOR { const auto _Count = _RANGES _Idl_distance(_Range); auto _First = _RANGES begin(_Range); - auto _UResult = _Replace_copy_if_unchecked(_RANGES _Unwrap_range_iter<_Rng>(_STD move(_First)), - _Uend(_Range), _STD _Get_unwrapped_n(_STD move(_Output), _Count), _STD _Pass_fn(_Pred), _Newval, - _STD _Pass_fn(_Proj)); + auto _UResult = _Replace_copy_if_common(_RANGES _Unwrap_range_iter<_Rng>(_STD move(_First)), _Uend(_Range), + _STD _Get_unwrapped_n(_STD move(_Output), _Count), _STD _Pass_fn(_Pred), _Newval, _STD _Pass_fn(_Proj)); _STD _Seek_wrapped(_First, _STD move(_UResult.in)); _STD _Seek_wrapped(_Output, _STD move(_UResult.out)); @@ -4633,7 +4636,7 @@ namespace ranges { private: template - _NODISCARD static constexpr replace_copy_if_result<_It, _Out> _Replace_copy_if_unchecked( + _NODISCARD static constexpr replace_copy_if_result<_It, _Out> _Replace_copy_if_common( _It _First, const _Se _Last, _Out _Output, _Pr _Pred, const _Ty& _Newval, _Pj _Proj) { // copy [_First, _Last) to _Output while replacing _Oldval with _Newval if projected _Oldval fulfills _Pred _STL_INTERNAL_STATIC_ASSERT(input_iterator<_It>); @@ -4642,6 +4645,8 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(indirectly_copyable<_It, _Out>); _STL_INTERNAL_STATIC_ASSERT(indirect_unary_predicate<_Pr, projected<_It, _Pj>>); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); + for (; _First != _Last; ++_First, (void) ++_Output) { if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty>) { *_Output = _STD invoke(_Pred, _STD invoke(_Proj, *_First)) ? _Newval : *_First; @@ -5615,6 +5620,7 @@ _EXPORT_STD template _CONSTEXPR20 _OutIt reverse_copy(_BidIt _First, _BidIt _Last, _OutIt _Dest) { // copy reversing elements in [_First, _Last) _STD _Adl_verify_range(_First, _Last); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Dest); const auto _UFirst = _STD _Get_unwrapped(_First); auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_BidIt>(_UFirst, _ULast)); @@ -5706,6 +5712,8 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(weakly_incrementable<_Out>); _STL_INTERNAL_STATIC_ASSERT(indirectly_copyable<_It, _Out>); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); + #if _USE_STD_VECTOR_ALGORITHMS if constexpr (contiguous_iterator<_It> && contiguous_iterator<_Out>) { using _Elem = remove_reference_t>; @@ -5851,6 +5859,7 @@ _CONSTEXPR20 _OutIt rotate_copy(_FwdIt _First, _FwdIt _Mid, _FwdIt _Last, _OutIt // copy rotating [_First, _Last) _STD _Adl_verify_range(_First, _Mid); _STD _Adl_verify_range(_Mid, _Last); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Dest); const auto _UFirst = _STD _Get_unwrapped(_First); const auto _UMid = _STD _Get_unwrapped(_Mid); const auto _ULast = _STD _Get_unwrapped(_Last); @@ -5886,7 +5895,7 @@ namespace ranges { auto _UFirst = _RANGES _Unwrap_iter<_Se>(_STD move(_First)); auto _ULast = _RANGES _Unwrap_sent<_It>(_STD move(_Last)); const auto _Count = _RANGES _Idl_distance<_It>(_UFirst, _ULast); - auto _UResult = _Rotate_copy_unchecked(_STD move(_UFirst), _RANGES _Unwrap_iter<_Se>(_STD move(_Mid)), + auto _UResult = _Rotate_copy_common(_STD move(_UFirst), _RANGES _Unwrap_iter<_Se>(_STD move(_Mid)), _STD move(_ULast), _STD _Get_unwrapped_n(_STD move(_Output), _Count)); _STD _Seek_wrapped(_First, _STD move(_UResult.in)); @@ -5901,15 +5910,15 @@ namespace ranges { _STD _Adl_verify_range(_RANGES begin(_Range), _Mid); _STD _Adl_verify_range(_Mid, _RANGES end(_Range)); const auto _Count = _RANGES _Idl_distance(_Range); - auto _UResult = _Rotate_copy_unchecked(_Ubegin(_Range), _RANGES _Unwrap_range_iter<_Rng>(_STD move(_Mid)), - _Uend(_Range), _STD _Get_unwrapped_n(_STD move(_Output), _Count)); + auto _UResult = _Rotate_copy_common(_Ubegin(_Range), _RANGES _Unwrap_range_iter<_Rng>(_STD move(_Mid)), + _Uend(_Range), _STD _Get_unwrapped_n(_STD move(_Output), _Count)); _STD _Seek_wrapped(_Output, _STD move(_UResult.out)); return {_RANGES _Rewrap_iterator(_Range, _STD move(_UResult.in)), _STD move(_Output)}; } private: template - _NODISCARD static constexpr rotate_copy_result<_It, _Out> _Rotate_copy_unchecked( + _NODISCARD static constexpr rotate_copy_result<_It, _Out> _Rotate_copy_common( _It _First, _It _Mid, _Se _Last, _Out _Output) { // Copy the content of [_Mid, _Last) and [_First, _Mid) to _Output _STL_INTERNAL_STATIC_ASSERT(forward_iterator<_It>); @@ -5917,6 +5926,8 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(weakly_incrementable<_Out>); _STL_INTERNAL_STATIC_ASSERT(indirectly_copyable<_It, _Out>); + _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); + auto _UResult1 = _RANGES _Copy_unchecked(_Mid, _STD move(_Last), _STD move(_Output)); auto _UResult2 = _RANGES _Copy_unchecked(_STD move(_First), _STD move(_Mid), _STD move(_UResult1.out)); return {_STD move(_UResult1.in), _STD move(_UResult2.out)}; diff --git a/tests/std/test.lst b/tests/std/test.lst index 32e116d37d3..87f84f36107 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -263,6 +263,7 @@ tests\GH_005276_system_error_heap_use_after_free tests\GH_005315_destructor_tombstones tests\GH_005402_string_with_volatile_range tests\GH_005421_vector_algorithms_integer_class_type_iterator +tests\GH_005472_do_not_overlap tests\LWG2381_num_get_floating_point tests\LWG2597_complex_branch_cut tests\LWG3018_shared_ptr_function diff --git a/tests/std/tests/GH_005472_do_not_overlap/env.lst b/tests/std/tests/GH_005472_do_not_overlap/env.lst new file mode 100644 index 00000000000..19f025bd0e6 --- /dev/null +++ b/tests/std/tests/GH_005472_do_not_overlap/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_matrix.lst diff --git a/tests/std/tests/GH_005472_do_not_overlap/test.cpp b/tests/std/tests/GH_005472_do_not_overlap/test.cpp new file mode 100644 index 00000000000..ff246460dda --- /dev/null +++ b/tests/std/tests/GH_005472_do_not_overlap/test.cpp @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include + +#include + +using namespace std; + +int arr[3] = {}; + +int main(int argc, char* argv[]) { +#if _ITERATOR_DEBUG_LEVEL == 2 + std_testing::death_test_executive exec; + + exec.add_death_tests({ + +[] { swap_ranges(arr, arr + 2, arr + 1); }, + +[] { reverse_copy(arr, arr + 2, arr + 1); }, + +[] { replace_copy(arr, arr + 2, arr + 1, 1, 2); }, + +[] { replace_copy_if(arr, arr + 2, arr + 1, [](int) { return false; }, 2); }, + +[] { rotate_copy(arr, arr + 1, arr + 2, arr + 1); }, +#if _HAS_CXX20 + +[] { ranges::reverse_copy(arr, arr + 2, arr + 1); }, + +[] { ranges::reverse_copy(ranges::subrange(arr, arr + 2), arr + 1); }, + +[] { ranges::replace_copy(arr, arr + 2, arr + 1, 1, 2); }, + +[] { ranges::replace_copy(ranges::subrange(arr, arr + 2), arr + 1, 1, 2); }, + +[] { ranges::replace_copy_if(arr, arr + 2, arr + 1, [](int) { return false; }, 2); }, + +[] { ranges::replace_copy_if(ranges::subrange(arr, arr + 2), arr + 1, [](int) { return false; }, 2); }, + +[] { ranges::rotate_copy(arr, arr + 1, arr + 2, arr + 1); }, + +[] { ranges::rotate_copy(ranges::subrange(arr, arr + 2), arr + 1, arr + 1); }, +#endif // _HAS_CXX20 + }); + + return exec.run(argc, argv); +#else // ^^^ _ITERATOR_DEBUG_LEVEL == 2 / _ITERATOR_DEBUG_LEVEL != 2 vvv + (void) argc; + (void) argv; + return 0; // This test is only for iterator debug mode +#endif // ^^^ _ITERATOR_DEBUG_LEVEL != 2 ^^^ +} From c6c64d33359acc12ff3f0d62be27e755952eae13 Mon Sep 17 00:00:00 2001 From: "A. Jiang" Date: Sat, 17 May 2025 12:49:53 +0800 Subject: [PATCH 10/16] Use the `_Transparent` concept and remove `_Is_transparent` (#5498) --- stl/inc/xhash | 3 +-- stl/inc/xmemory | 11 +---------- stl/inc/xtree | 17 ++++++++++++++--- 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/stl/inc/xhash b/stl/inc/xhash index 8ad550329c9..6710782f3a3 100644 --- a/stl/inc/xhash +++ b/stl/inc/xhash @@ -100,8 +100,7 @@ struct _Uhash_choose_transparency { }; #if _HAS_CXX20 -template - requires _Is_transparent_v<_Hasher> && _Is_transparent_v<_Keyeq> +template struct _Uhash_choose_transparency<_Kty, _Hasher, _Keyeq> { // transparency selector for transparent hashed containers static constexpr bool _Has_transparent_overloads = true; diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 36e4102cc78..e4cfa3008b5 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -2734,18 +2734,9 @@ _EXPORT_STD inline void* align(size_t _Bound, size_t _Size, void*& _Ptr, size_t& return _Ptr; } -template -constexpr bool _Is_transparent_v = false; - -template -constexpr bool _Is_transparent_v<_Ty, void_t> = true; - -template -struct _Is_transparent : bool_constant<_Is_transparent_v<_Ty>> {}; - #if _HAS_CXX20 template -concept _Transparent = _Is_transparent_v<_Ty>; +concept _Transparent = requires { typename _Ty::is_transparent; }; #endif // _HAS_CXX20 #if _HAS_CXX23 diff --git a/stl/inc/xtree b/stl/inc/xtree index bfbadffced9..3b03dde8fbc 100644 --- a/stl/inc/xtree +++ b/stl/inc/xtree @@ -21,6 +21,17 @@ _STL_DISABLE_CLANG_WARNINGS #undef new _STD_BEGIN +#if _HAS_CXX20 +template +constexpr bool _Is_transparent_v = _Transparent<_Ty>; +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv +template +constexpr bool _Is_transparent_v = false; + +template +constexpr bool _Is_transparent_v<_Ty, void_t> = true; +#endif // ^^^ !_HAS_CXX20 ^^^ + template class _Tree_unchecked_const_iterator : public _Base { public: @@ -1360,7 +1371,7 @@ public: #if _HAS_CXX23 template - requires _Is_transparent_v && (!is_convertible_v<_Kx, const_iterator>) + requires _Transparent && (!is_convertible_v<_Kx, const_iterator>) && (!is_convertible_v<_Kx, iterator>) size_type erase(_Kx&& _Keyval) noexcept(noexcept(_Eqrange(_Keyval))) /* strengthened */ { return _Erase(_Eqrange(_Keyval)); @@ -1414,7 +1425,7 @@ public: } template - requires _Is_transparent_v + requires _Transparent _NODISCARD bool contains(const _Other& _Keyval) const { return _Lower_bound_duplicate(_Find_lower_bound(_Keyval)._Bound, _Keyval); } @@ -1764,7 +1775,7 @@ public: #if _HAS_CXX23 template - requires _Is_transparent_v && (!is_convertible_v<_Kx, const_iterator>) + requires _Transparent && (!is_convertible_v<_Kx, const_iterator>) && (!is_convertible_v<_Kx, iterator>) node_type extract(_Kx&& _Keyval) { const const_iterator _Where = find(_Keyval); From 2eb069fbe290f0064c80833dea80e36abce6b64e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 17 May 2025 07:52:34 +0300 Subject: [PATCH 11/16] Avoid calling `memcpy` in `swap` (#5500) Co-authored-by: Stephan T. Lavavej --- stl/inc/type_traits | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/stl/inc/type_traits b/stl/inc/type_traits index ab5bdf97b90..9c8e1d740dd 100644 --- a/stl/inc/type_traits +++ b/stl/inc/type_traits @@ -9,7 +9,6 @@ #if _STL_COMPILER_PREPROCESSOR #include #include -#include #include #pragma pack(push, _CRT_PACKING) @@ -2513,10 +2512,15 @@ void _Swap_trivial_arrays(_Ty (&_Left)[_Size], _Ty (&_Right)[_Size]) noexcept { if constexpr (_Size_parts != 0) { const auto _Stop = _Left_ptr + _Size_parts; do { - unsigned char _Buf[_Part_size_bytes]; - _CSTD memcpy(_Buf, _Left_ptr, _Part_size_bytes); - _CSTD memcpy(_Left_ptr, _Right_ptr, _Part_size_bytes); - _CSTD memcpy(_Right_ptr, _Buf, _Part_size_bytes); + struct _Buffer_type { + unsigned char _Data[_Part_size_bytes]; + }; + + _STL_INTERNAL_STATIC_ASSERT(sizeof(_Buffer_type) == _Part_size_bytes); // assume no padding + + const _Buffer_type _Buffer = *reinterpret_cast(_Left_ptr); + *reinterpret_cast<_Buffer_type*>(_Left_ptr) = *reinterpret_cast(_Right_ptr); + *reinterpret_cast<_Buffer_type*>(_Right_ptr) = _Buffer; _Left_ptr += _Part_size_bytes; _Right_ptr += _Part_size_bytes; @@ -2524,10 +2528,15 @@ void _Swap_trivial_arrays(_Ty (&_Left)[_Size], _Ty (&_Right)[_Size]) noexcept { } if constexpr (_Size_tail != 0) { - unsigned char _Buf[_Size_tail]; - _CSTD memcpy(_Buf, _Left_ptr, _Size_tail); - _CSTD memcpy(_Left_ptr, _Right_ptr, _Size_tail); - _CSTD memcpy(_Right_ptr, _Buf, _Size_tail); + struct _Last_buffer_type { + unsigned char _Data[_Size_tail]; + }; + + _STL_INTERNAL_STATIC_ASSERT(sizeof(_Last_buffer_type) == _Size_tail); // assume no padding + + const _Last_buffer_type _Last_buffer = *reinterpret_cast(_Left_ptr); + *reinterpret_cast<_Last_buffer_type*>(_Left_ptr) = *reinterpret_cast(_Right_ptr); + *reinterpret_cast<_Last_buffer_type*>(_Right_ptr) = _Last_buffer; } } From ad4bf41e9395d9a3b0a7cffec31389fea2611ea7 Mon Sep 17 00:00:00 2001 From: Jakub Mazurkiewicz Date: Sat, 17 May 2025 06:55:52 +0200 Subject: [PATCH 12/16] ``: Implement missing precondition in `layout_stride::mapping`'s constructor (#5505) --- stl/inc/mdspan | 20 ++++++-- .../P0009R18_mdspan_layout_stride/test.cpp | 46 +++++++++---------- .../test.cpp | 16 +++++++ 3 files changed, 56 insertions(+), 26 deletions(-) diff --git a/stl/inc/mdspan b/stl/inc/mdspan index f11eda31a47..ea05bb61d5c 100644 --- a/stl/inc/mdspan +++ b/stl/inc/mdspan @@ -826,8 +826,9 @@ public: using layout_type = layout_stride; private: - using _Extents_base = _Maybe_fully_static_extents; - using _Strides_base = _Maybe_empty_array; + using _Extents_base = _Maybe_fully_static_extents; + using _Strides_base = _Maybe_empty_array; + using _Stride_extent_pair = pair; static_assert(_Is_extents, "Extents must be a specialization of std::extents (N4950 [mdspan.layout.stride.overview]/2)."); @@ -847,6 +848,7 @@ private: bool _Found_zero = false; bool _Overflow = false; index_type _Req_span_size = 0; + array<_Stride_extent_pair, extents_type::rank()> _Pairs; for (rank_type _Idx = 0; _Idx < extents_type::_Rank; ++_Idx) { const index_type _Stride = this->_Array[_Idx]; _STL_VERIFY(_Stride > 0, "Value of s[i] must be greater than 0 for all i in the range [0, rank_) " @@ -861,9 +863,22 @@ private: _Overflow = _Mul_overflow(static_cast(_Ext - 1), _Stride, _Prod) || _Add_overflow(_Req_span_size, _Prod, _Req_span_size); } + + _Pairs[_Idx] = {static_cast(_Stride), static_cast(_Ext)}; } _STL_VERIFY(_Found_zero || !_Overflow, "REQUIRED-SPAN-SIZE(e, s) must be representable as a value of type " "index_type (N4950 [mdspan.layout.stride.cons]/4.2)."); + + if (!_Found_zero) { + _RANGES sort(_Pairs); + for (rank_type _Idx = 1; _Idx < extents_type::_Rank; ++_Idx) { + _STL_VERIFY(_Pairs[_Idx].first >= _Pairs[_Idx - 1].first * _Pairs[_Idx - 1].second, + "If rank_ is greater than 0, then there must exist a permutation P of the integers in the " + "range [0, rank_), such that 's[p(i)] >= s[p(i-1)] * e.extent(p(i-1))' is true for all i in " + "the range [1, rank_), where p(i) is the i-th element of P (N5008 " + "[mdspan.layout.stride.cons]/4.3)."); + } + } } #endif // _ITERATOR_DEBUG_LEVEL != 0 } @@ -1111,7 +1126,6 @@ private: } _NODISCARD constexpr bool _Is_exhaustive_special_case() const noexcept { - using _Stride_extent_pair = pair; array<_Stride_extent_pair, extents_type::rank()> _Pairs; for (rank_type _Idx = 0; _Idx < extents_type::_Rank; ++_Idx) { rank_type _Ext = static_cast(this->_Exts.extent(_Idx)); diff --git a/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp b/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp index 8e6f27ff054..eebbd167afe 100644 --- a/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_layout_stride/test.cpp @@ -386,7 +386,7 @@ constexpr void check_required_span_size() { using M1 = layout_stride::mapping>; static_assert(M1{}.required_span_size() == 0); - layout_stride::mapping> m2{dextents{3, 0, 3, 3}, array{1, 3, 1, 1}}; + layout_stride::mapping> m2{dextents{3, 0, 3, 3}, array{1, 3, 1, 3}}; assert(m2.required_span_size() == 0); } @@ -800,37 +800,37 @@ constexpr void check_correctness() { { // 3x2x2 tensor using E = extents; - const array vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}; - layout_stride::mapping m{E{}, array{8, 1, 6}}; // non-exhaustive mapping + const array vals{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18}; + layout_stride::mapping m{E{}, array{7, 1, 3}}; // non-exhaustive mapping assert(!m.is_exhaustive()); mdspan tensor{vals.data(), m}; #ifdef __cpp_multidimensional_subscript // TRANSITION, P2128R6 assert((tensor[0, 0, 0] == 0)); - assert((tensor[0, 0, 1] == 6)); + assert((tensor[0, 0, 1] == 3)); assert((tensor[0, 1, 0] == 1)); - assert((tensor[0, 1, 1] == 7)); - assert((tensor[1, 0, 0] == 8)); - assert((tensor[1, 0, 1] == 14)); - assert((tensor[1, 1, 0] == 9)); - assert((tensor[1, 1, 1] == 15)); - assert((tensor[2, 0, 0] == 16)); - assert((tensor[2, 0, 1] == 22)); - assert((tensor[2, 1, 0] == 17)); - assert((tensor[2, 1, 1] == 23)); + assert((tensor[0, 1, 1] == 4)); + assert((tensor[1, 0, 0] == 7)); + assert((tensor[1, 0, 1] == 10)); + assert((tensor[1, 1, 0] == 8)); + assert((tensor[1, 1, 1] == 11)); + assert((tensor[2, 0, 0] == 14)); + assert((tensor[2, 0, 1] == 17)); + assert((tensor[2, 1, 0] == 15)); + assert((tensor[2, 1, 1] == 18)); #else // ^^^ defined(__cpp_multidimensional_subscript) / !defined(__cpp_multidimensional_subscript) vvv assert((tensor[array{0, 0, 0}] == 0)); - assert((tensor[array{0, 0, 1}] == 6)); + assert((tensor[array{0, 0, 1}] == 3)); assert((tensor[array{0, 1, 0}] == 1)); - assert((tensor[array{0, 1, 1}] == 7)); - assert((tensor[array{1, 0, 0}] == 8)); - assert((tensor[array{1, 0, 1}] == 14)); - assert((tensor[array{1, 1, 0}] == 9)); - assert((tensor[array{1, 1, 1}] == 15)); - assert((tensor[array{2, 0, 0}] == 16)); - assert((tensor[array{2, 0, 1}] == 22)); - assert((tensor[array{2, 1, 0}] == 17)); - assert((tensor[array{2, 1, 1}] == 23)); + assert((tensor[array{0, 1, 1}] == 4)); + assert((tensor[array{1, 0, 0}] == 7)); + assert((tensor[array{1, 0, 1}] == 10)); + assert((tensor[array{1, 1, 0}] == 8)); + assert((tensor[array{1, 1, 1}] == 11)); + assert((tensor[array{2, 0, 0}] == 14)); + assert((tensor[array{2, 0, 1}] == 17)); + assert((tensor[array{2, 1, 0}] == 15)); + assert((tensor[array{2, 1, 1}] == 18)); #endif // ^^^ !defined(__cpp_multidimensional_subscript) ^^^ } diff --git a/tests/std/tests/P0009R18_mdspan_layout_stride_death/test.cpp b/tests/std/tests/P0009R18_mdspan_layout_stride_death/test.cpp index 2c1cb19e7df..f1c3b38a9cd 100644 --- a/tests/std/tests/P0009R18_mdspan_layout_stride_death/test.cpp +++ b/tests/std/tests/P0009R18_mdspan_layout_stride_death/test.cpp @@ -29,6 +29,12 @@ void test_construction_from_extents_and_array_2() { [[maybe_unused]] layout_stride::mapping m{Ext{}, array{2}}; } +void test_construction_from_extents_and_array_3() { + using Ext = extents; + const array a{29, 2, 6}; + // Incorrect strides + [[maybe_unused]] layout_stride::mapping m{Ext{}, a}; +} void test_construction_from_extents_and_span_1() { array a{-1}; @@ -44,6 +50,14 @@ void test_construction_from_extents_and_span_2() { [[maybe_unused]] layout_stride::mapping m{Ext{}, s}; } +void test_construction_from_extents_and_span_3() { + using Ext = extents; + array a{3, 1, 8, 3}; + const span s{a}; + // Incorrect strides + [[maybe_unused]] layout_stride::mapping m{Ext{}, s}; +} + void test_construction_from_strided_layout_mapping() { layout_right::mapping> m1; // Value of other.required_span_size() must be representable as a value of type index_type @@ -73,8 +87,10 @@ int main(int argc, char* argv[]) { test_default_construction, test_construction_from_extents_and_array_1, test_construction_from_extents_and_array_2, + test_construction_from_extents_and_array_3, test_construction_from_extents_and_span_1, test_construction_from_extents_and_span_2, + test_construction_from_extents_and_span_3, test_construction_from_strided_layout_mapping, test_call_operator, test_stride_with_empty_extents, From e2f43c5e145cc323e6fc515a912e4225e8fa1d84 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Fri, 16 May 2025 21:58:59 -0700 Subject: [PATCH 13/16] `stl/CMakeLists.txt`: Remove redundant `Synchronization.lib` (#5496) --- stl/CMakeLists.txt | 3 --- 1 file changed, 3 deletions(-) diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index e58bb2defbd..7a93242a9bd 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -442,9 +442,6 @@ if(VCLIBS_TARGET_ARCHITECTURE MATCHES "^(x86|x64)$") add_library(stl_alias_objects OBJECT ${ALIAS_SOURCES_X86_X64}) else() add_library(stl_alias_objects INTERFACE) - - # on ARM64 and ARM, we can unconditionally expect Synchronization.lib to exist - string(APPEND CMAKE_CXX_STANDARD_LIBRARIES " Synchronization.lib") endif() if(STL_ASAN_BUILD) From 6de3f1cf54f04a25aa81127ea1a3c98ddfb43ba8 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Fri, 16 May 2025 22:00:49 -0700 Subject: [PATCH 14/16] Win10 baseline: Drop support for Win8.1 / Server 2012 R2 (#5510) --- stl/inc/yvals_core.h | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index 022ea5b2cc9..1ed0b04bf1a 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -1974,18 +1974,12 @@ compiler option, or define _ALLOW_RTCc_IN_STL to suppress this error. #endif // defined(MRTDLL) && !defined(_M_CEE_PURE) #define _STL_WIN32_WINNT_VISTA 0x0600 // _WIN32_WINNT_VISTA from sdkddkver.h -#define _STL_WIN32_WINNT_WIN8 0x0602 // _WIN32_WINNT_WIN8 from sdkddkver.h #define _STL_WIN32_WINNT_WIN10 0x0A00 // _WIN32_WINNT_WIN10 from sdkddkver.h // Note that the STL DLL builds will set this to XP for ABI compatibility with VS2015 which supported XP. #ifndef _STL_WIN32_WINNT -#if defined(_M_ARM64) -// The first ARM64 Windows was Windows 10 +// The earliest Windows supported by this implementation is Windows 10 #define _STL_WIN32_WINNT _STL_WIN32_WINNT_WIN10 -#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv -// The earliest Windows supported by this implementation is Windows 8 -#define _STL_WIN32_WINNT _STL_WIN32_WINNT_WIN8 -#endif // ^^^ !defined(_M_ARM64) ^^^ #endif // !defined(_STL_WIN32_WINNT) #ifdef __cpp_noexcept_function_type From 2391e5eb104bc0b027a73c8c98c5b7cdff35dd52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 17 May 2025 07:26:10 +0200 Subject: [PATCH 15/16] ``: Improve search performance for regexes with initial `+` quantifiers (#5509) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/regex_search.cpp | 38 ++++++++++++++ stl/inc/regex | 10 +++- tests/std/include/test_regex_support.hpp | 12 +++-- .../std/tests/VSO_0000000_regex_use/test.cpp | 49 +++++++++++++++++++ 5 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 benchmarks/src/regex_search.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 908b3640019..1ce49241b49 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -116,6 +116,7 @@ add_benchmark(nth_element src/nth_element.cpp) add_benchmark(path_lexically_normal src/path_lexically_normal.cpp) add_benchmark(priority_queue_push_range src/priority_queue_push_range.cpp) add_benchmark(random_integer_generation src/random_integer_generation.cpp) +add_benchmark(regex_search src/regex_search.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) add_benchmark(reverse src/reverse.cpp) diff --git a/benchmarks/src/regex_search.cpp b/benchmarks/src/regex_search.cpp new file mode 100644 index 00000000000..36f4cb9d6a2 --- /dev/null +++ b/benchmarks/src/regex_search.cpp @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include + +#include "lorem.hpp" + +using namespace std; + +void bm_lorem_search(benchmark::State& state, const char* pattern) { + string repeated_lorem{lorem_ipsum}; + for (long long i = 0; i < state.range(); ++i) { + repeated_lorem += repeated_lorem; + } + regex re{pattern}; + + for (auto _ : state) { + benchmark::DoNotOptimize(repeated_lorem); + const char* pos = repeated_lorem.data(); + const char* end = repeated_lorem.data() + repeated_lorem.size(); + cmatch match; + for (; regex_search(pos, end, match, re); ++pos) { + benchmark::DoNotOptimize(match); + pos = match[0].second; + if (pos == end) { + break; + } + } + } +} + +BENCHMARK_CAPTURE(bm_lorem_search, "bibe", "bibe")->Arg(2)->Arg(3)->Arg(4); +BENCHMARK_CAPTURE(bm_lorem_search, "(bibe)+", "(bibe)+")->Arg(2)->Arg(3)->Arg(4); +BENCHMARK_CAPTURE(bm_lorem_search, "(?:bibe)+", "(?:bibe)+")->Arg(2)->Arg(3)->Arg(4); + +BENCHMARK_MAIN(); diff --git a/stl/inc/regex b/stl/inc/regex index 8b359cfa7e8..46dab2b06d0 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -4128,6 +4128,15 @@ _BidIt _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Skip(_BidIt _First_arg break; } + case _N_rep: + { + _Node_rep* _Node = static_cast<_Node_rep*>(_Nx); + if (_Node->_Min == 0) { + return _First_arg; + } + break; + } + case _N_begin: break; @@ -4139,7 +4148,6 @@ _BidIt _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Skip(_BidIt _First_arg case _N_neg_assert: case _N_back: case _N_endif: - case _N_rep: case _N_end_rep: default: return _First_arg; diff --git a/tests/std/include/test_regex_support.hpp b/tests/std/include/test_regex_support.hpp index 523b3535b30..0cbd643aca1 100644 --- a/tests/std/include/test_regex_support.hpp +++ b/tests/std/include/test_regex_support.hpp @@ -282,8 +282,9 @@ class test_regex { submatches_success = false; break; } - } else if (!actual_capture.matched || actual_capture.first != (mr[0].first + expected_capture.first) - || actual_capture.second != (mr[0].first + expected_capture.second)) { + } else if (!actual_capture.matched + || actual_capture.first != (subject.begin() + expected_capture.first) + || actual_capture.second != (subject.begin() + expected_capture.second)) { submatches_success = false; break; } @@ -297,7 +298,8 @@ class test_regex { for (const auto& expected_capture : capture_groups) { std::string capture = "(unmatched)"; if (expected_capture.first != -1) { - capture.assign(mr[0].first + expected_capture.first, mr[0].first + expected_capture.second); + capture.assign( + subject.begin() + expected_capture.first, subject.begin() + expected_capture.second); } printf(R"(%s"%s" [%td %td])", initial ? "" : ", ", capture.c_str(), expected_capture.first, expected_capture.second); @@ -313,8 +315,8 @@ class test_regex { std::ptrdiff_t last = -1; if (actual_capture.matched) { capture = actual_capture.str(); - first = actual_capture.first - mr[0].first; - last = actual_capture.second - mr[0].first; + first = actual_capture.first - subject.begin(); + last = actual_capture.second - subject.begin(); } printf(R"(%s"%s" [%td %td])", initial ? "" : ", ", capture.c_str(), first, last); initial = false; diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index 59c4f14cc99..7160c9007fc 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -1889,6 +1889,54 @@ void test_gh_5490() { } } +void test_gh_5509() { + // GH-5509 extended the matcher's skip optimization + // to regexes starting with a loop with at least one repetition, + // speeding up searches for such regexes + + { + test_regex char_plus_regex(&g_regexTester, "(a+)"); + char_plus_regex.should_search_match_capture_groups("blwerofaaweraf", "aa", match_default, {{7, 9}}); + char_plus_regex.should_search_fail("blwerofwerf"); + } + + { + test_regex charclass_plus_regex(&g_regexTester, "([fa]+)"); + charclass_plus_regex.should_search_match_capture_groups("blwerofaaweraf", "faa", match_default, {{6, 9}}); + charclass_plus_regex.should_search_fail("blwerower"); + } + + { + test_regex string_plus_regex(&g_regexTester, "((?:aw)+)"); + string_plus_regex.should_search_match_capture_groups("blwerofaawaweraf", "awaw", match_default, {{8, 12}}); + string_plus_regex.should_search_fail("blwerofaerwaf"); + } + + { + test_regex anchored_string_plus_regex(&g_regexTester, "((?:^aw)+)"); + anchored_string_plus_regex.should_search_match_capture_groups( + "blwerofa\nawaweraf", "aw", match_default, {{9, 11}}); + anchored_string_plus_regex.should_search_fail("blwerof\naerwaf"); + } + + { + test_regex anchored_string_plus_regex(&g_regexTester, "((?:$\naw)+)"); + anchored_string_plus_regex.should_search_match_capture_groups( + "blwerofa\nawaweraf", "\naw", match_default, {{8, 11}}); + anchored_string_plus_regex.should_search_fail("blwerof\naerwaf"); + } + + { + test_regex string_star_string_regex(&g_regexTester, "((?:aw)*fa)"); + string_star_string_regex.should_search_match_capture_groups( + "blwerofaawawfaeraf", "fa", match_default, {{6, 8}}); + string_star_string_regex.should_search_match_capture_groups( + "blweroawawfaeraf", "awawfa", match_default, {{6, 12}}); + string_star_string_regex.should_search_match("blwerofaerwaf", "fa"); + string_star_string_regex.should_search_fail("blweroerwaf"); + } +} + int main() { test_dev10_449367_case_insensitivity_should_work(); test_dev11_462743_regex_collate_should_not_disable_regex_icase(); @@ -1936,6 +1984,7 @@ int main() { test_gh_5374(); test_gh_5377(); test_gh_5490(); + test_gh_5509(); return g_regexTester.result(); } From cbd091e4170a84dfb8beca3119ab8e3188988848 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 17 May 2025 08:38:21 +0300 Subject: [PATCH 16/16] Vectorize `rotate` better (#5502) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/rotate.cpp | 58 ++++++++++ stl/inc/algorithm | 13 +++ stl/inc/xutility | 13 +++ stl/src/vector_algorithms.cpp | 101 ++++++++++++++++++ .../test.cpp | 26 ++++- .../VSO_0000000_vector_algorithms/test.cpp | 72 +++++++++++++ 7 files changed, 283 insertions(+), 1 deletion(-) create mode 100644 benchmarks/src/rotate.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 1ce49241b49..338f9c6a239 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -120,6 +120,7 @@ add_benchmark(regex_search src/regex_search.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) add_benchmark(reverse src/reverse.cpp) +add_benchmark(rotate src/rotate.cpp) add_benchmark(search src/search.cpp) add_benchmark(search_n src/search_n.cpp) add_benchmark(std_copy src/std_copy.cpp) diff --git a/benchmarks/src/rotate.cpp b/benchmarks/src/rotate.cpp new file mode 100644 index 00000000000..9ba7c4b898f --- /dev/null +++ b/benchmarks/src/rotate.cpp @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include + +#include "skewed_allocator.hpp" +#include "utility.hpp" + +using namespace std; + +enum class AlgType { Std, Rng }; + +template +void bm_rotate(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + const auto n = static_cast(state.range(1)); + + auto v = random_vector(size); + benchmark::DoNotOptimize(v); + + for (auto _ : state) { + if constexpr (Alg == AlgType::Std) { + rotate(v.begin(), v.begin() + n, v.end()); + } else { + ranges::rotate(v, v.begin() + n); + } + benchmark::DoNotOptimize(v); + } +} + +void common_args(auto bm) { + bm->Args({3333, 2242})->Args({3332, 1666})->Args({3333, 1111})->Args({3333, 501}); + bm->Args({3333, 3300})->Args({3333, 12})->Args({3333, 5})->Args({3333, 1}); + bm->Args({333, 101})->Args({123, 32})->Args({23, 7})->Args({12, 5})->Args({3, 2}); +} + +struct color { + uint16_t h; + uint16_t s; + uint16_t l; +}; + +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); + +BENCHMARK(bm_rotate)->Apply(common_args); +BENCHMARK(bm_rotate)->Apply(common_args); + +BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index fe005e72d51..984878026db 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -5776,6 +5776,19 @@ namespace ranges { } if constexpr (bidirectional_iterator<_It>) { +#if _USE_STD_VECTOR_ALGORITHMS + using _Elem = remove_reference_t>; + + if constexpr (contiguous_iterator<_It> && sized_sentinel_for<_Se, _It> + && conjunction_v<_Is_trivially_ranges_swappable<_Elem>, negation>>) { + if (!_STD is_constant_evaluated()) { + const _It _Last_it = _First + (_Last - _First); + ::__std_rotate(_STD to_address(_First), _STD to_address(_Mid), _STD to_address(_Last_it)); + return {_First + (_Last - _Mid), _Last}; + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + _RANGES _Reverse_common(_First, _Mid); auto _Final = _RANGES _Get_final_iterator_unwrapped<_It>(_Mid, _STD move(_Last)); _RANGES _Reverse_common(_Mid, _Final); diff --git a/stl/inc/xutility b/stl/inc/xutility index 6ceda19327e..854ae81b26c 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -79,6 +79,8 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* _Last1, void* _First2) noexcept; +__declspec(noalias) void __stdcall __std_rotate(void* _First, void* _Mid, void* _Last) noexcept; + __declspec(noalias) size_t __stdcall __std_count_trivial_1( const void* _First, const void* _Last, uint8_t _Val) noexcept; __declspec(noalias) size_t __stdcall __std_count_trivial_2( @@ -6597,6 +6599,17 @@ _CONSTEXPR20 _FwdIt rotate(_FwdIt _First, _FwdIt _Mid, _FwdIt _Last) { } if constexpr (_Is_cpp17_random_iter_v<_FwdIt>) { +#if _USE_STD_VECTOR_ALGORITHMS + using _Elem = remove_reference_t<_Iter_ref_t>; + + if constexpr (conjunction_v>, + _Is_trivially_swappable<_Elem>, negation>>) { + if (!_STD _Is_constant_evaluated()) { + ::__std_rotate(_STD _To_address(_UFirst), _STD _To_address(_UMid), _STD _To_address(_ULast)); + return _First + (_Last - _Mid); + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS _STD reverse(_UFirst, _UMid); _STD reverse(_UMid, _ULast); _STD reverse(_UFirst, _ULast); diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 12558f8f3b4..fb20d8d71c3 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -382,6 +382,107 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( } // extern "C" +namespace { + namespace _Rotating { + // TRANSITION, GH-5506 "VCRuntime: memmove() is surprisingly slow for more than 8 KB on certain CPUs": + // As a workaround, the following code calls memmove() for 8 KB portions. + constexpr size_t _Portion_size = 8192; + constexpr size_t _Portion_mask = _Portion_size - 1; + static_assert((_Portion_size & _Portion_mask) == 0); + + void _Move_to_lower_address(void* _Dest, const void* _Src, const size_t _Size) noexcept { + const size_t _Whole_portions_size = _Size & ~_Portion_mask; + + void* _Dest_end = _Dest; + _Advance_bytes(_Dest_end, _Whole_portions_size); + + while (_Dest != _Dest_end) { + memmove(_Dest, _Src, _Portion_size); + _Advance_bytes(_Dest, _Portion_size); + _Advance_bytes(_Src, _Portion_size); + } + + if (const size_t _Tail = _Size - _Whole_portions_size; _Tail != 0) { + memmove(_Dest, _Src, _Tail); + } + } + + void _Move_to_higher_address(void* const _Dest, const void* const _Src, const size_t _Size) noexcept { + const size_t _Whole_portions_size = _Size & ~_Portion_mask; + + void* _Dest_end = _Dest; + _Advance_bytes(_Dest_end, _Whole_portions_size); + const void* _Src_end = _Src; + _Advance_bytes(_Src_end, _Whole_portions_size); + + if (const size_t _Tail = _Size - _Whole_portions_size; _Tail != 0) { + memmove(_Dest_end, _Src_end, _Tail); + } + + while (_Dest_end != _Dest) { + _Rewind_bytes(_Dest_end, _Portion_size); + _Rewind_bytes(_Src_end, _Portion_size); + memmove(_Dest_end, _Src_end, _Portion_size); + } + } + + constexpr size_t _Buf_size = 512; + + bool _Use_buffer(const size_t _Smaller, const size_t _Larger) noexcept { + return _Smaller <= _Buf_size && (_Smaller <= 128 || _Larger >= _Smaller * 2); + } + } // namespace _Rotating +} // unnamed namespace + +extern "C" { + +__declspec(noalias) void __stdcall __std_rotate(void* _First, void* const _Mid, void* _Last) noexcept { + unsigned char _Buf[_Rotating::_Buf_size]; + + for (;;) { + const size_t _Left = _Byte_length(_First, _Mid); + const size_t _Right = _Byte_length(_Mid, _Last); + + if (_Left <= _Right) { + if (_Left == 0) { + break; + } + + if (_Rotating::_Use_buffer(_Left, _Right)) { + memcpy(_Buf, _First, _Left); + _Rotating::_Move_to_lower_address(_First, _Mid, _Right); + _Advance_bytes(_First, _Right); + memcpy(_First, _Buf, _Left); + break; + } + + void* _Mid2 = _Last; + _Rewind_bytes(_Mid2, _Left); + __std_swap_ranges_trivially_swappable_noalias(_Mid2, _Last, _First); + _Last = _Mid2; + } else { + if (_Right == 0) { + break; + } + + if (_Rotating::_Use_buffer(_Right, _Left)) { + _Rewind_bytes(_Last, _Right); + memcpy(_Buf, _Last, _Right); + void* _Mid2 = _First; + _Advance_bytes(_Mid2, _Right); + _Rotating::_Move_to_higher_address(_Mid2, _First, _Left); + memcpy(_First, _Buf, _Right); + break; + } + + __std_swap_ranges_trivially_swappable_noalias(_Mid, _Last, _First); + _Advance_bytes(_First, _Right); + } + } +} + +} // extern "C" + namespace { namespace _Sorting { enum _Min_max_mode { diff --git a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp index dd00180318d..6e54f8390a1 100644 --- a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp +++ b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp @@ -98,7 +98,7 @@ int main() { picky_contiguous_iterator float_arr_begin(begin(float_arr)); picky_contiguous_iterator float_arr_end(end(float_arr)); - transform(arr_begin, arr_end, float_arr_begin, [](int v) { return static_cast(v); }); + transform(arr_begin, arr_end, float_arr_begin, [](const int v) { return static_cast(v); }); assert(ranges::min(ranges::subrange(float_arr_begin, float_arr_end)) == 200.0); assert(ranges::max(ranges::subrange(float_arr_begin, float_arr_end)) == 390.0); @@ -196,6 +196,30 @@ int main() { ranges::reverse(temp_begin, temp_end); assert(ranges::equal(temp_begin, temp_end, begin(reverse_expected), end(reverse_expected))); } + { + const int rotate_expected[] = { + 250, 270, 280, 290, 300, 310, 320, 250, 340, 250, 250, 370, 380, 390, 200, 210, 220, 250, 240, 250}; + + const _Signed128 rotate_pos = 6; + + auto rot_copy_it = rotate_copy(arr_begin, arr_begin + rotate_pos, arr_end, temp_begin); + assert(equal(temp_begin, temp_end, begin(rotate_expected), end(rotate_expected))); + assert(rot_copy_it == temp_end); + + copy(arr_begin, arr_end, temp_begin); + auto rot_it = rotate(temp_begin, temp_begin + rotate_pos, temp_end); + assert(equal(temp_begin, temp_end, begin(rotate_expected), end(rotate_expected))); + assert(rot_it == temp_end - rotate_pos); + + auto r_rot_copy_it = ranges::rotate_copy(arr_begin, arr_begin + rotate_pos, arr_end, temp_begin).out; + assert(ranges::equal(temp_begin, temp_end, begin(rotate_expected), end(rotate_expected))); + assert(r_rot_copy_it == temp_end); + + ranges::copy(arr_begin, arr_end, temp_begin); + auto r_rot_it = begin(ranges::rotate(temp_begin, temp_begin + rotate_pos, temp_end)); + assert(ranges::equal(temp_begin, temp_end, begin(rotate_expected), end(rotate_expected))); + assert(r_rot_it == temp_end - rotate_pos); + } { // Out of replace family, only replace for 32-bit and 64-bit elements is manually vectorized, // replace_copy is auto vectorized (along with replace_copy_if) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 35dac73c833..1232bde41af 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -743,6 +743,65 @@ void test_reverse_copy(mt19937_64& gen) { } } +template +void last_known_good_rotate( + RanIt first, RanIt mid, RanIt last, vector::value_type>& tmp) { + const auto size_left = mid - first; + const auto size_right = last - mid; + if (size_left <= size_right) { + tmp.assign(first, mid); + move_backward(mid, last, last - size_left); + move(tmp.begin(), tmp.end(), last - size_left); + } else { + tmp.assign(mid, last); + move(first, mid, first + size_right); + move(tmp.begin(), tmp.end(), first); + } +} + +template +void test_case_rotate( + vector& actual, vector& actual_r, vector& expected, const ptrdiff_t pos, vector& tmp) { + const ptrdiff_t shift = static_cast(expected.size()) - pos; + last_known_good_rotate(expected.begin(), expected.begin() + pos, expected.end(), tmp); + const auto it = rotate(actual.begin(), actual.begin() + pos, actual.end()); + assert(expected == actual); + assert(it == actual.begin() + shift); +#if _HAS_CXX20 + const auto rng = ranges::rotate(actual_r.begin(), actual_r.begin() + pos, actual_r.end()); + assert(expected == actual_r); + assert(begin(rng) == actual_r.begin() + shift); + assert(end(rng) == actual_r.end()); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ +} + +template +void test_rotate(mt19937_64& gen) { + vector actual; + vector actual_r; + vector expected; + vector tmp; + actual.reserve(dataCount); + actual_r.reserve(dataCount); + expected.reserve(dataCount); + tmp.reserve(dataCount); + test_case_rotate(actual, actual_r, expected, 0, tmp); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + const T val = static_cast(gen()); // intentionally narrows + actual.push_back(val); + actual_r.push_back(val); + expected.push_back(val); + + uniform_int_distribution dis_pos(0, static_cast(attempts) + 1); + + for (size_t pos_count = 0; pos_count != 5; ++pos_count) { + test_case_rotate(actual, actual_r, expected, dis_pos(gen), tmp); + } + } +} + template FwdIt2 last_known_good_swap_ranges(FwdIt1 first1, const FwdIt1 last1, FwdIt2 dest) { for (; first1 != last1; ++first1, ++dest) { @@ -1182,6 +1241,19 @@ void test_vector_algorithms(mt19937_64& gen) { test_reverse_copy(gen); test_reverse_copy(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_rotate(gen); + test_remove(gen); test_remove(gen); test_remove(gen);