From 9043ffe1b64ae9927ca616dacadc64d127b4c193 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 18 Mar 2025 20:44:44 +0200 Subject: [PATCH 01/30] Use `find` for `search_n` when n=1 --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/search_n.cpp | 57 +++++++++++++++++++ stl/inc/algorithm | 37 ++++++++++++ .../P0896R4_ranges_alg_search_n/test.cpp | 14 +++++ 4 files changed, 109 insertions(+) create mode 100644 benchmarks/src/search_n.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 22503345a4a..d1367bc9d3f 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -124,6 +124,7 @@ add_benchmark(random_integer_generation src/random_integer_generation.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) add_benchmark(search src/search.cpp) +add_benchmark(search_n src/search_n.cpp) add_benchmark(std_copy src/std_copy.cpp) add_benchmark(sv_equal src/sv_equal.cpp) add_benchmark(swap_ranges src/swap_ranges.cpp) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp new file mode 100644 index 00000000000..313b944034e --- /dev/null +++ b/benchmarks/src/search_n.cpp @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include + +#include "skewed_allocator.hpp" + +using namespace std; + +// NB: This particular algorthm has std and ranges implementations with different perf charactterstitics! + +enum class AlgType { Std, Rng }; + +template +void bm(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + + constexpr size_t count = 1; + + constexpr T no_match{'-'}; + constexpr T match{'*'}; + + vector> v(size, no_match); + + fill(v.begin() + v.size() / 2, v.end(), match); + + for (auto _ : state) { + if constexpr (Alg == AlgType::Std) { + benchmark::DoNotOptimize(search_n(v.begin(), v.end(), count, match)); + } else if constexpr (Alg == AlgType::Rng) { + benchmark::DoNotOptimize(ranges::search_n(v, count, match)); + } + } +} + +void common_args(auto bm) { + bm->Arg(3000); +} + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 8109baf3a16..d1f0a6684b9 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -2220,6 +2220,16 @@ _NODISCARD _CONSTEXPR20 _FwdIt search_n( return _First; } + if constexpr (_Is_any_of_v<_Pr, +#if _HAS_CXX20 + _RANGES equal_to, +#endif + equal_to<>>) { + if (_Count == 1) { + return _STD find(_First, _Last, _Val); + } + } + if (static_cast(_Count) > static_cast(_STD _Max_limit<_Iter_diff_t<_FwdIt>>())) { // if the number of _Vals searched for is larger than the longest possible sequence, we can't find it return _Last; @@ -2320,6 +2330,19 @@ namespace ranges { return {_First, _First}; } + if constexpr (_Is_any_of_v<_Pr, _STD equal_to<>, _RANGES equal_to>) { + if (_Count == 1) { + auto _Res = _RANGES find(_First, _Last, _Val, _Pass_fn(_Proj)); + if (_Res != _Last) { + auto _Res_end = _Res; + ++_Res_end; + return {_STD move(_Res), _STD move(_Res_end)}; + } else { + return {_Res, _Res}; + } + } + } + auto _UFirst = _RANGES _Unwrap_iter<_Se>(_STD move(_First)); auto _ULast = _RANGES _Unwrap_sent<_It>(_STD move(_Last)); @@ -2346,6 +2369,20 @@ namespace ranges { return {_First, _First}; } + if constexpr (_Is_any_of_v<_Pr, _STD equal_to<>, _RANGES equal_to>) { + if (_Count == 1) { + auto _Res = _RANGES find(_Range, _Val, _Pass_fn(_Proj)); + auto _Last = _RANGES end(_Range); + if (_Res != _Last) { + auto _Res_end = _Res; + ++_Res_end; + return {_STD move(_Res), _STD move(_Res_end)}; + } else { + return {_Res, _Res}; + } + } + } + if constexpr (sized_range<_Rng>) { const auto _Dist = _RANGES distance(_Range); diff --git a/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp b/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp index 3f3a609e568..fa3e9e7c6e8 100644 --- a/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp +++ b/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp @@ -98,6 +98,20 @@ struct instantiator { assert(result.end() == range.begin()); } + // trivial case: unit needle + { + const auto result = ranges::search_n(range, 1, 0, cmp, get_first); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + { + const auto result = ranges::search_n(ranges::begin(range), ranges::end(range), 1, 0, cmp, get_first); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + // trivial case: range too small { const auto result = ranges::search_n(range, 99999, 0, cmp, get_first); From 7af139a3765f9bffabdd017edf2d1cbe5cdcdc42 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 18 Mar 2025 23:17:19 +0200 Subject: [PATCH 02/30] Actually test predicate-less unit needle --- .../std/tests/P0896R4_ranges_alg_search_n/test.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp b/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp index fa3e9e7c6e8..46a2590bdb7 100644 --- a/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp +++ b/tests/std/tests/P0896R4_ranges_alg_search_n/test.cpp @@ -99,6 +99,20 @@ struct instantiator { } // trivial case: unit needle + { + const auto result = ranges::search_n(range, 1, P{1, 42}); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + { + const auto result = ranges::search_n(ranges::begin(range), ranges::end(range), 1, P{1, 42}); + static_assert(same_as>>); + assert(result.begin() == ranges::next(range.begin(), 1)); + assert(result.end() == ranges::next(range.begin(), 2)); + } + + // trivial case: unit needle with predicate { const auto result = ranges::search_n(range, 1, 0, cmp, get_first); static_assert(same_as>>); From 74116a215d3ace79d1be2e43c6072188bff8a394 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 19 Mar 2025 17:20:22 -0700 Subject: [PATCH 03/30] Fix comment typos. --- benchmarks/src/search_n.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index 313b944034e..ffddae9ca96 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -12,7 +12,7 @@ using namespace std; -// NB: This particular algorthm has std and ranges implementations with different perf charactterstitics! +// NB: This particular algorithm has std and ranges implementations with different perf characteristics! enum class AlgType { Std, Rng }; From d3766eafdc94d002a415809b9b3c9c7897fa55a1 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 19 Mar 2025 17:23:08 -0700 Subject: [PATCH 04/30] Avoid shadowing: count => N --- benchmarks/src/search_n.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index ffddae9ca96..1fab2bf11f5 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -20,7 +20,7 @@ template void bm(benchmark::State& state) { const auto size = static_cast(state.range(0)); - constexpr size_t count = 1; + constexpr size_t N = 1; constexpr T no_match{'-'}; constexpr T match{'*'}; @@ -31,9 +31,9 @@ void bm(benchmark::State& state) { for (auto _ : state) { if constexpr (Alg == AlgType::Std) { - benchmark::DoNotOptimize(search_n(v.begin(), v.end(), count, match)); + benchmark::DoNotOptimize(search_n(v.begin(), v.end(), N, match)); } else if constexpr (Alg == AlgType::Rng) { - benchmark::DoNotOptimize(ranges::search_n(v, count, match)); + benchmark::DoNotOptimize(ranges::search_n(v, N, match)); } } } From 2cb7d1ca06910b3e45a58cfad9c6f01c93d7347a Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 19 Mar 2025 17:24:12 -0700 Subject: [PATCH 05/30] Remove unused ``. --- benchmarks/src/search_n.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index 1fab2bf11f5..b35cfd9eb2d 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include "skewed_allocator.hpp" From d10807f71bdae924858f377cb025c748b0aabe8b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 16 Mar 2025 11:08:37 +0200 Subject: [PATCH 06/30] test --- tests/std/test.lst | 1 + .../env.lst | 7 + .../test.cpp | 120 ++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 tests/std/tests/VSO_0000000_vector_algorithms_search_n/env.lst create mode 100644 tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp diff --git a/tests/std/test.lst b/tests/std/test.lst index 484a11f3a14..e01f4edba20 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -732,6 +732,7 @@ tests\VSO_0000000_type_traits tests\VSO_0000000_vector_algorithms tests\VSO_0000000_vector_algorithms_floats tests\VSO_0000000_vector_algorithms_mismatch_and_lex_compare +tests\VSO_0000000_vector_algorithms_search_n tests\VSO_0000000_wcfb01_idempotent_container_destructors tests\VSO_0000000_wchar_t_filebuf_xsmeown tests\VSO_0095468_clr_exception_ptr_bad_alloc diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/env.lst b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/env.lst new file mode 100644 index 00000000000..78fbe6b49a4 --- /dev/null +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/env.lst @@ -0,0 +1,7 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_matrix.lst +RUNALL_CROSSLIST +* PM_CL="" +* PM_CL="/D_USE_STD_VECTOR_ALGORITHMS=0" diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp new file mode 100644 index 00000000000..8596534bd9c --- /dev/null +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include + +#include "test_vector_algorithms_support.hpp" + +using namespace std; + +template +auto last_known_good_search_n(FwdIt first, FwdIt last, size_t count, T val) { + // Delibarately using simple approach, not smart bidi/random iterators "check from the other end" stuff + if (count == 0) { + return first; + } + + for (; first != last; ++first) { + if (*first == val) { + FwdIt match = first; + size_t match_size = count; + for (;;) { + --match_size; + if (match_size == 0) { + return match; + } + + ++first; + + if (first == last) { + return last; + } + + if (*first != val) { + break; + } + } + } + } + return last; +} + +template +void test_case_search_n(const Container& c, size_t count, T val) { + auto expected = last_known_good_search_n(c.begin(), c.end(), count, val); + auto actual = search_n(c.begin(), c.end(), count, val); + assert(expected == actual); + +#if _HAS_CXX20 + auto ranges_actual = ranges::search_n(c, static_cast(count), val); + assert(expected == begin(ranges_actual)); + if (expected == c.end()) { + assert(end(ranges_actual) == c.end()); + } else { + assert(distance(expected, end(ranges_actual)) == static_cast(count)); + } +#endif // _HAS_CXX20 +} + +template +void test_search_n(mt19937_64& gen) { + constexpr size_t lengthCount = 70; + constexpr size_t patternCount = 5; + using TD = conditional_t; + uniform_int_distribution dis((numeric_limits::min)(), (numeric_limits::max)()); + vector input_src; + vector input; + input_src.reserve(dataCount); + input.reserve(dataCount); + + for (;;) { + for (size_t count = 0; count != lengthCount; ++count) { + input = input_src; + + const T val = static_cast(dis(gen)); + + test_case_search_n(input, count, val); + + binomial_distribution pattern_length_dis(count * 2, 0.5); + uniform_int_distribution pos_dis(0, input.size() - 1); + + for (size_t pattern = 0; pattern != patternCount; ++pattern) { + const size_t pattern_length = pattern_length_dis(gen); + const size_t pattern_pos = pos_dis(gen); + + if (pattern_length + pattern_pos < input.size()) { + fill_n(input.begin() + static_cast(pattern_pos), pattern_length, val); + + test_case_search_n(input, count, val); + } + } + } + + if (input.size() == dataCount) { + break; + } + + input_src.push_back(static_cast(dis(gen))); + } +} + +void test_vector_algorithms(mt19937_64& gen) { + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); + test_search_n(gen); +} + +int main() { + run_randomized_tests_with_different_isa_levels(test_vector_algorithms); +} From d965af630bdbad8e60c13d6ca53eb5b283e0d586 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 16 Mar 2025 11:59:47 +0200 Subject: [PATCH 07/30] benchmark --- benchmarks/src/search_n.cpp | 95 ++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index b35cfd9eb2d..d5cd7e17fe9 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include "skewed_allocator.hpp" @@ -15,42 +17,101 @@ using namespace std; enum class AlgType { Std, Rng }; -template -void bm(benchmark::State& state) { - const auto size = static_cast(state.range(0)); +enum class PartternType { + TwoZones, + RareSignleMatches, + DenseSmallSequences, +}; - constexpr size_t N = 1; +template +void bm(benchmark::State& state) { + const auto size = static_cast(state.range(0)); + const auto n = static_cast(state.range(1)); constexpr T no_match{'-'}; constexpr T match{'*'}; vector> v(size, no_match); - fill(v.begin() + v.size() / 2, v.end(), match); + if constexpr (Parttern == PartternType::TwoZones) { + fill(v.begin() + v.size() / 2, v.end(), match); + } else if constexpr (Parttern == PartternType::RareSignleMatches) { + if (size != 0 && n != 0) { + mt19937 gen{275423}; + + uniform_int_distribution pos_dis(0, size - 1); + + const size_t single_match_amount = size / n; + + for (size_t i = 0; i != single_match_amount; ++i) { + v[pos_dis(gen)] = match; + } + } + } else if constexpr (Parttern == PartternType::DenseSmallSequences) { + if (size != 0 && n != 0) { + mt19937 gen{7687239}; + + uniform_int_distribution len_dis(0, n - 1); + + size_t cur_len = len_dis(gen); + + for (size_t i = 0; i != size; ++i) { + if (cur_len != 0) { + v[i] = match; + --cur_len; + } else { + cur_len = len_dis(gen); + } + } + } + } for (auto _ : state) { if constexpr (Alg == AlgType::Std) { - benchmark::DoNotOptimize(search_n(v.begin(), v.end(), N, match)); + benchmark::DoNotOptimize(search_n(v.begin(), v.end(), n, match)); } else if constexpr (Alg == AlgType::Rng) { - benchmark::DoNotOptimize(ranges::search_n(v, N, match)); + benchmark::DoNotOptimize(ranges::search_n(v, n, match)); } } } -void common_args(auto bm) { - bm->Arg(3000); +void common_args_large_counts(auto bm) { + bm->ArgPair(3000, 200)->ArgPair(3000, 40)->ArgPair(3000, 20)->ArgPair(3000, 10)->ArgPair(3000, 5); } -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); +void common_args(auto bm) { + common_args_large_counts(bm); + bm->ArgPair(3000, 2)->ArgPair(3000, 1); +} -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args_large_counts); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); BENCHMARK_MAIN(); From ab6df2e3440115cd7e3ed563dbbc2948af2eb6f8 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 16 Mar 2025 13:26:38 +0200 Subject: [PATCH 08/30] skeleton --- stl/inc/algorithm | 95 +++++++++++++++++++++++++++++++++++ stl/src/vector_algorithms.cpp | 58 +++++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index d1f0a6684b9..6a923f850d2 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -79,6 +79,11 @@ __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept; __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; + +const void* __stdcall __std_search_n_1(const void* _First, const void* _Last, size_t _Count, uint8_t _Value) noexcept; +const void* __stdcall __std_search_n_2(const void* _First, const void* _Last, size_t _Count, uint16_t _Value) noexcept; +const void* __stdcall __std_search_n_4(const void* _First, const void* _Last, size_t _Count, uint32_t _Value) noexcept; +const void* __stdcall __std_search_n_8(const void* _First, const void* _Last, size_t _Count, uint64_t _Value) noexcept; } // extern "C" _STD_BEGIN @@ -205,6 +210,33 @@ __declspec(noalias) void _Replace_vectorized( } } +template +_Ty* _Search_n_vectorized(_Ty* const _First, _Ty* const _Last, const size_t _Count, const _TVal _Val) noexcept { + if constexpr (is_pointer_v<_Ty>) { +#ifdef _WIN64 + return const_cast<_Ty*>( + static_cast(::__std_search_n_8(_First, _Last, _Count, reinterpret_cast(_Val)))); +#else // ^^^ defined(_WIN64) / !defined(_WIN64) vvv + return const_cast<_Ty*>( + static_cast(::__std_search_n_4(_First, _Last, _Count, reinterpret_cast(_Val)))); +#endif // ^^^ !defined(_WIN64) ^^^ + } else if constexpr (sizeof(_Ty) == 1) { + return const_cast<_Ty*>( + static_cast(::__std_search_n_1(_First, _Last, _Count, static_cast(_Val)))); + } else if constexpr (sizeof(_Ty) == 2) { + return const_cast<_Ty*>( + static_cast(::__std_search_n_2(_First, _Last, _Count, static_cast(_Val)))); + } else if constexpr (sizeof(_Ty) == 4) { + return const_cast<_Ty*>( + static_cast(::__std_search_n_4(_First, _Last, _Count, static_cast(_Val)))); + } else if constexpr (sizeof(_Ty) == 8) { + return const_cast<_Ty*>( + static_cast(::__std_search_n_8(_First, _Last, _Count, static_cast(_Val)))); + } else { + _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size + } +} + // Can we activate the vector algorithms for find_first_of? template constexpr bool _Vector_alg_in_find_first_of_is_safe = _Equal_memcmp_is_safe<_It1, _It2, _Pr>; @@ -219,6 +251,14 @@ template constexpr bool _Vector_alg_in_ranges_replace_is_safe = _Vector_alg_in_replace_is_safe<_Iter, _Ty1> // can search and replace && _Vector_alg_in_find_is_safe_elem<_Ty2, _Iter_value_t<_Iter>>; // replacement fits + +template +constexpr bool _Vector_alg_in_search_n_is_safe = _Vector_alg_in_find_is_safe<_Iter, _Ty> + && _Is_any_of_v<_Pr, +#if _HAS_CXX20 + ranges::equal_to, +#endif + equal_to<>>; _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -2239,6 +2279,29 @@ _NODISCARD _CONSTEXPR20 _FwdIt search_n( auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); if constexpr (_Is_ranges_random_iter_v<_FwdIt>) { +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Vector_alg_in_search_n_is_safe) { + if (!_STD _Is_constant_evaluated()) { + if (!_STD _Could_compare_equal_to_value_type(_Val)) { + return _Last; + } + + const auto _First_ptr = _STD _To_address(_UFirst); + const auto _Result = + _STD _Search_n_vectorized(_First_ptr, _STD _To_address(_ULast), static_cast(_Count), _Val); + + if constexpr (is_pointer_v) { + _UFirst = _Result; + } else { + _UFirst += (_Result - _First_ptr); + } + + _STD _Seek_wrapped(_Last, _UFirst); + return _Last; + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + const auto _Count_diff = static_cast<_Iter_diff_t<_FwdIt>>(_Count); auto _UOld_first = _UFirst; for (_Iter_diff_t<_FwdIt> _Inc = 0; _Count_diff <= _ULast - _UOld_first;) { // enough room, look for a match @@ -2411,6 +2474,38 @@ namespace ranges { return {_First, _First}; } +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Vector_alg_in_search_n_is_safe<_It, _Ty, _Pr> && is_same_v<_Pj, identity>) { + if (!_STD is_constant_evaluated()) { + if (!_STD _Could_compare_equal_to_value_type<_It>(_Val)) { + _First += _Dist; + return {_First, _First}; + } + + const auto _First_ptr = _STD _To_address(_First); + const auto _Last_ptr = _First_ptr + _Dist; + const auto _Result = _STD _Search_n_vectorized( + _First_ptr, _First_ptr + _Dist, static_cast(_Count), _Val); + + if constexpr (is_pointer_v<_It>) { + if (_Result != _Last_ptr) { + return {_Result, _Result + _Count}; + } else { + return {_Result, _Result}; + } + } else { + if (_Result != _Last_ptr) { + _First += _Result - _First_ptr; + return {_First, _First + _Count}; + } else { + _First += _Dist; + return {_First, _First}; + } + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + auto _Last = _RANGES next(_First, _Count); auto _Mid1 = _First; auto _Mid2 = _Last; diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 41c2daac2f6..21f95b79f67 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2930,6 +2930,44 @@ namespace { return _Result; } + template + const void* __stdcall __std_search_n_impl( + const void* _First, const void* const _Last, const size_t _Count, const _Ty _Val) noexcept { + auto _Match_start = static_cast(_First); + auto _Last_ptr = static_cast(_Last); + + if (static_cast(_Last_ptr - _Match_start) < _Count) { + return _Last_ptr; + } + + auto _Mid1 = _Match_start; + auto _Match_end = _Match_start + _Count; + auto _Mid2 = _Match_end; + for (;;) { + // Invariants: _Match_end - _Match_start == _Count, [_Match_start, _Mid1) and [_Mid2, _Match_end) match + // _Val: + // + // _Match_start _Mid1 _Mid2 _Match_end + // |=============|????????|========|??????????... + + --_Mid2; + if (*_Mid2 == _Val) { // match; + if (_Mid1 == _Mid2) { // [_Mid1, _Mid2) is empty, so [_Match_start, _Match_end) all match + return _Match_start; + } + } else { // mismatch; skip past it + _Match_start = _Mid2 + 1; + + if (static_cast(_Last_ptr - _Match_start) < _Count) { // not enough space left + return _Last_ptr; + } + + _Mid1 = _Match_end; + _Match_end = _Match_start + _Count; + _Mid2 = _Match_end; + } + } + } #ifndef _M_ARM64EC namespace __std_find_meow_of_bitmap_details { __m256i _Bitmap_step(const __m256i _Bitmap, const __m256i _Data) noexcept { @@ -4565,6 +4603,26 @@ __declspec(noalias) size_t __stdcall __std_count_trivial_8( return __std_count_trivial_impl<_Count_traits_8>(_First, _Last, _Val); } +const void* __stdcall __std_search_n_1( + const void* const _First, const void* const _Last, const size_t _Count, const uint8_t _Value) noexcept { + return __std_search_n_impl<_Find_traits_1>(_First, _Last, _Count, _Value); +} + +const void* __stdcall __std_search_n_2( + const void* const _First, const void* const _Last, const size_t _Count, const uint16_t _Value) noexcept { + return __std_search_n_impl<_Find_traits_2>(_First, _Last, _Count, _Value); +} + +const void* __stdcall __std_search_n_4( + const void* const _First, const void* const _Last, const size_t _Count, const uint32_t _Value) noexcept { + return __std_search_n_impl<_Find_traits_4>(_First, _Last, _Count, _Value); +} + +const void* __stdcall __std_search_n_8( + const void* const _First, const void* const _Last, const size_t _Count, const uint64_t _Value) noexcept { + return __std_search_n_impl<_Find_traits_8>(_First, _Last, _Count, _Value); +} + const void* __stdcall __std_find_first_of_trivial_1( const void* const _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { return __std_find_first_of::_Dispatch_ptr(_First1, _Last1, _First2, _Last2); From d47a02df72fbd4084d92bcd47c326a6ec31562fe Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 16 Mar 2025 22:58:35 +0200 Subject: [PATCH 09/30] More benchmarks, initial vectorization --- benchmarks/src/search_n.cpp | 6 ++--- stl/src/vector_algorithms.cpp | 51 ++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index d5cd7e17fe9..5469f2c91cf 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -13,7 +13,7 @@ using namespace std; -// NB: This particular algorithm has std and ranges implementations with different perf characteristics! +// NB: This particular algorithm has std and ranges non-vectorized implementations with different perf characteristics! enum class AlgType { Std, Rng }; @@ -76,12 +76,12 @@ void bm(benchmark::State& state) { } void common_args_large_counts(auto bm) { - bm->ArgPair(3000, 200)->ArgPair(3000, 40)->ArgPair(3000, 20)->ArgPair(3000, 10)->ArgPair(3000, 5); + bm->ArgPair(3000, 200)->ArgPair(3000, 40)->ArgPair(3000, 20)->ArgPair(3000, 10)->ArgPair(3000, 9)->ArgPair(3000, 8); } void common_args(auto bm) { common_args_large_counts(bm); - bm->ArgPair(3000, 2)->ArgPair(3000, 1); + bm->ArgPair(3000, 5)->ArgPair(3000, 4)->ArgPair(3000, 2)->ArgPair(3000, 1); } diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 21f95b79f67..14c169de8ac 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2933,6 +2933,56 @@ namespace { template const void* __stdcall __std_search_n_impl( const void* _First, const void* const _Last, const size_t _Count, const _Ty _Val) noexcept { + if (_Count == 0) { + return _First; + } else if (_Count == 1) { + return __std_find_trivial_impl<_Traits>(_First, _Last, _Val); + } + + auto _Mid1 = static_cast(_First); + + const size_t _Length = _Byte_length(_First, _Last); + const size_t _Cmp_size = _Count * sizeof(_Ty); + if (_Cmp_size <= 8 && _Length >= 32 && _Use_avx2()) { + const size_t _Sh1 = _Cmp_size < 4 ? _Cmp_size - 2 : 2; + const size_t _Sh2 = _Cmp_size < 4 ? 0 : _Cmp_size - 4; + + const __m256i _Comparand = _Traits::_Set_avx(_Val); + + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Length & ~size_t{0x1F}); + + size_t _Carry = 0; + do { + const __m256i _Data = _mm256_loadu_si256(reinterpret_cast(_First)); + const auto _Mask = static_cast(_mm256_movemask_epi8(_Traits::_Cmp_avx(_Comparand, _Data))); + + const uint32_t _MskN = ~_Mask; + + if (_Carry + _tzcnt_u32(_MskN) >= _Cmp_size) { + _Rewind_bytes(_First, _Carry); + return _First; + } + + uint32_t _MskX = _Mask; + _MskX = (_Mask >> 1) & _Mask; + _MskX = (_MskX >> _Sh1) & _MskX; + _MskX = (_MskX >> _Sh2) & _MskX; + + if (_MskX != 0) { + _Advance_bytes(_First, _tzcnt_u32(_MskX)); + return _First; + } + + _Carry = _lzcnt_u32(_MskN); + + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); + + _Mid1 = static_cast(_First); + _Rewind_bytes(_First, _Carry); + } + auto _Match_start = static_cast(_First); auto _Last_ptr = static_cast(_Last); @@ -2940,7 +2990,6 @@ namespace { return _Last_ptr; } - auto _Mid1 = _Match_start; auto _Match_end = _Match_start + _Count; auto _Mid2 = _Match_end; for (;;) { From ee1f8f427ae5e690f9107f7fdaeb03d859c52b6f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 17 Mar 2025 23:09:23 +0200 Subject: [PATCH 10/30] Drop RareSignleMatches benchmark --- benchmarks/src/search_n.cpp | 27 +-------------------------- 1 file changed, 1 insertion(+), 26 deletions(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index 5469f2c91cf..412c1ec0f46 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -19,7 +19,6 @@ enum class AlgType { Std, Rng }; enum class PartternType { TwoZones, - RareSignleMatches, DenseSmallSequences, }; @@ -35,18 +34,6 @@ void bm(benchmark::State& state) { if constexpr (Parttern == PartternType::TwoZones) { fill(v.begin() + v.size() / 2, v.end(), match); - } else if constexpr (Parttern == PartternType::RareSignleMatches) { - if (size != 0 && n != 0) { - mt19937 gen{275423}; - - uniform_int_distribution pos_dis(0, size - 1); - - const size_t single_match_amount = size / n; - - for (size_t i = 0; i != single_match_amount; ++i) { - v[pos_dis(gen)] = match; - } - } } else if constexpr (Parttern == PartternType::DenseSmallSequences) { if (size != 0 && n != 0) { mt19937 gen{7687239}; @@ -75,41 +62,29 @@ void bm(benchmark::State& state) { } } -void common_args_large_counts(auto bm) { - bm->ArgPair(3000, 200)->ArgPair(3000, 40)->ArgPair(3000, 20)->ArgPair(3000, 10)->ArgPair(3000, 9)->ArgPair(3000, 8); -} - void common_args(auto bm) { - common_args_large_counts(bm); + bm->ArgPair(3000, 200)->ArgPair(3000, 40)->ArgPair(3000, 20)->ArgPair(3000, 10)->ArgPair(3000, 9)->ArgPair(3000, 8); bm->ArgPair(3000, 5)->ArgPair(3000, 4)->ArgPair(3000, 2)->ArgPair(3000, 1); } BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args_large_counts); -BENCHMARK(bm)->Apply(common_args_large_counts); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args_large_counts); -BENCHMARK(bm)->Apply(common_args_large_counts); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args_large_counts); -BENCHMARK(bm)->Apply(common_args_large_counts); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args_large_counts); -BENCHMARK(bm)->Apply(common_args_large_counts); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); From 8c4a691773c6e034b9286a43fe7f793eb634c688 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 17 Mar 2025 23:43:29 +0200 Subject: [PATCH 11/30] more specialization for sizes --- stl/src/vector_algorithms.cpp | 92 ++++++++++++++++++++++++++--------- 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 14c169de8ac..a26550e18bb 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2930,6 +2930,50 @@ namespace { return _Result; } + struct _Search_n_traits_1 : _Find_traits_1 { + using _MskX_t = uint64_t; + static constexpr size_t _Max_count = 8; + static constexpr size_t _Scale = 1; + static constexpr size_t _Carry_adjust = 32; + + static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { + return _mm256_movemask_epi8(_mm256_cmpeq_epi8(_Comparand, _Data)); + } + }; + + struct _Search_n_traits_2 : _Find_traits_2 { + using _MskX_t = uint64_t; + static constexpr size_t _Max_count = 4; + static constexpr size_t _Scale = 2; + static constexpr size_t _Carry_adjust = 32; + + static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { + return _mm256_movemask_epi8(_mm256_cmpeq_epi16(_Comparand, _Data)); + } + }; + + struct _Search_n_traits_4 : _Find_traits_4 { + using _MskX_t = uint32_t; + static constexpr size_t _Max_count = 8; + static constexpr size_t _Scale = 1; + static constexpr size_t _Carry_adjust = 8; + + static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { + return _mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpeq_epi32(_Comparand, _Data))); + } + }; + + struct _Search_n_traits_8 : _Find_traits_8 { + using _MskX_t = uint32_t; + static constexpr size_t _Max_count = 4; + static constexpr size_t _Scale = 1; + static constexpr size_t _Carry_adjust = 4; + + static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { + return _mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(_Comparand, _Data))); + } + }; + template const void* __stdcall __std_search_n_impl( const void* _First, const void* const _Last, const size_t _Count, const _Ty _Val) noexcept { @@ -2941,46 +2985,48 @@ namespace { auto _Mid1 = static_cast(_First); - const size_t _Length = _Byte_length(_First, _Last); - const size_t _Cmp_size = _Count * sizeof(_Ty); - if (_Cmp_size <= 8 && _Length >= 32 && _Use_avx2()) { - const size_t _Sh1 = _Cmp_size < 4 ? _Cmp_size - 2 : 2; - const size_t _Sh2 = _Cmp_size < 4 ? 0 : _Cmp_size - 4; + const size_t _Length = _Byte_length(_First, _Last); + if (_Count <= _Traits::_Max_count && _Length >= 32 && _Use_avx2()) { + const size_t _Bits_count = _Count * _Traits::_Scale; + const size_t _Sh1 = _Bits_count < 4 ? _Bits_count - 2 : 2; + const size_t _Sh2 = sizeof(_Ty) <= 4 ? (_Bits_count < 4 ? 0 : _Bits_count - 4) : 0; const __m256i _Comparand = _Traits::_Set_avx(_Val); const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Length & ~size_t{0x1F}); - size_t _Carry = 0; + uint32_t _Carry = 0; do { const __m256i _Data = _mm256_loadu_si256(reinterpret_cast(_First)); - const auto _Mask = static_cast(_mm256_movemask_epi8(_Traits::_Cmp_avx(_Comparand, _Data))); + const auto _Mask = static_cast(_Traits::_Cmp_avx_elem_mask(_Comparand, _Data)); - const uint32_t _MskN = ~_Mask; + using _MskX_t = _Traits::_MskX_t; - if (_Carry + _tzcnt_u32(_MskN) >= _Cmp_size) { - _Rewind_bytes(_First, _Carry); - return _First; + _MskX_t _MskX = _MskX_t{_Carry} | (_MskX_t{_Mask} << _Traits::_Carry_adjust); + if constexpr (_Traits::_Scale == 1) { + _MskX = (_MskX >> 1) & _MskX; + _MskX = (_MskX >> _Sh1) & _MskX; + } else if constexpr (_Traits::_Scale == 2) { + _MskX = (_MskX >> 2) & _MskX; + } + if constexpr (_Traits::_Scale * _Traits::_Max_count > 4) { + _MskX = (_MskX >> _Sh2) & _MskX; } - - uint32_t _MskX = _Mask; - _MskX = (_Mask >> 1) & _Mask; - _MskX = (_MskX >> _Sh1) & _MskX; - _MskX = (_MskX >> _Sh2) & _MskX; if (_MskX != 0) { - _Advance_bytes(_First, _tzcnt_u32(_MskX)); + int _Shift = static_cast(_tzcnt_u64(_MskX)) - _Traits::_Carry_adjust; + _Advance_bytes(_First, _Shift * (sizeof(_Ty) / _Traits::_Scale)); return _First; } - _Carry = _lzcnt_u32(_MskN); + _Carry = _Mask; _Advance_bytes(_First, 32); } while (_First != _Stop_at); + _Rewind_bytes(_First, _Count * (sizeof(_Ty))); _Mid1 = static_cast(_First); - _Rewind_bytes(_First, _Carry); } auto _Match_start = static_cast(_First); @@ -4654,22 +4700,22 @@ __declspec(noalias) size_t __stdcall __std_count_trivial_8( const void* __stdcall __std_search_n_1( const void* const _First, const void* const _Last, const size_t _Count, const uint8_t _Value) noexcept { - return __std_search_n_impl<_Find_traits_1>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Search_n_traits_1>(_First, _Last, _Count, _Value); } const void* __stdcall __std_search_n_2( const void* const _First, const void* const _Last, const size_t _Count, const uint16_t _Value) noexcept { - return __std_search_n_impl<_Find_traits_2>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Search_n_traits_2>(_First, _Last, _Count, _Value); } const void* __stdcall __std_search_n_4( const void* const _First, const void* const _Last, const size_t _Count, const uint32_t _Value) noexcept { - return __std_search_n_impl<_Find_traits_4>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Search_n_traits_4>(_First, _Last, _Count, _Value); } const void* __stdcall __std_search_n_8( const void* const _First, const void* const _Last, const size_t _Count, const uint64_t _Value) noexcept { - return __std_search_n_impl<_Find_traits_8>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Search_n_traits_8>(_First, _Last, _Count, _Value); } const void* __stdcall __std_find_first_of_trivial_1( From 0fcaa86ff349fc44bbc3ef0a56a1167f09af55cc Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 21 Mar 2025 20:48:34 +0200 Subject: [PATCH 12/30] more n! --- stl/src/vector_algorithms.cpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index a26550e18bb..8ff2d574444 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2932,7 +2932,7 @@ namespace { struct _Search_n_traits_1 : _Find_traits_1 { using _MskX_t = uint64_t; - static constexpr size_t _Max_count = 8; + static constexpr size_t _Max_count = 16; static constexpr size_t _Scale = 1; static constexpr size_t _Carry_adjust = 32; @@ -2943,7 +2943,7 @@ namespace { struct _Search_n_traits_2 : _Find_traits_2 { using _MskX_t = uint64_t; - static constexpr size_t _Max_count = 4; + static constexpr size_t _Max_count = 8; static constexpr size_t _Scale = 2; static constexpr size_t _Carry_adjust = 32; @@ -2987,9 +2987,11 @@ namespace { const size_t _Length = _Byte_length(_First, _Last); if (_Count <= _Traits::_Max_count && _Length >= 32 && _Use_avx2()) { + constexpr auto _Max_bits = _Traits::_Max_count * _Traits::_Scale; const size_t _Bits_count = _Count * _Traits::_Scale; const size_t _Sh1 = _Bits_count < 4 ? _Bits_count - 2 : 2; - const size_t _Sh2 = sizeof(_Ty) <= 4 ? (_Bits_count < 4 ? 0 : _Bits_count - 4) : 0; + const size_t _Sh2 = _Max_bits > 4 ? (_Bits_count < 4 ? 0 : (_Bits_count < 8 ? _Bits_count - 4 : 4)) : 0; + const size_t _Sh3 = _Max_bits > 8 ? (_Bits_count < 8 ? 0 : _Bits_count - 8) : 0; const __m256i _Comparand = _Traits::_Set_avx(_Val); @@ -3010,8 +3012,11 @@ namespace { } else if constexpr (_Traits::_Scale == 2) { _MskX = (_MskX >> 2) & _MskX; } - if constexpr (_Traits::_Scale * _Traits::_Max_count > 4) { + if constexpr (_Max_bits > 4) { _MskX = (_MskX >> _Sh2) & _MskX; + if constexpr (_Max_bits > 8) { + _MskX = (_MskX >> _Sh3) & _MskX; + } } if (_MskX != 0) { From cccd2413e3c9ee3e05684d8d32da076b104bd38c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 21 Mar 2025 20:52:37 +0200 Subject: [PATCH 13/30] more n in becnhmark! --- benchmarks/src/search_n.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index 412c1ec0f46..cd2d6cde1d8 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -24,8 +24,8 @@ enum class PartternType { template void bm(benchmark::State& state) { - const auto size = static_cast(state.range(0)); - const auto n = static_cast(state.range(1)); + const auto size = static_cast(state.range(0)); + const auto n = static_cast(state.range(1)); constexpr T no_match{'-'}; constexpr T match{'*'}; @@ -63,11 +63,11 @@ void bm(benchmark::State& state) { } void common_args(auto bm) { - bm->ArgPair(3000, 200)->ArgPair(3000, 40)->ArgPair(3000, 20)->ArgPair(3000, 10)->ArgPair(3000, 9)->ArgPair(3000, 8); - bm->ArgPair(3000, 5)->ArgPair(3000, 4)->ArgPair(3000, 2)->ArgPair(3000, 1); + for (const auto& n : {40, 18, 16, 14, 10, 8, 5, 4, 3, 2, 1}) { + bm->ArgPair(3000, n); + } } - BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); BENCHMARK(bm)->Apply(common_args); From 93139bec34516d37310c6b8f6676fda7dcb172a1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 21 Mar 2025 21:28:20 +0200 Subject: [PATCH 14/30] thresholds! --- stl/src/vector_algorithms.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 8ff2d574444..955df66ac48 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2954,7 +2954,7 @@ namespace { struct _Search_n_traits_4 : _Find_traits_4 { using _MskX_t = uint32_t; - static constexpr size_t _Max_count = 8; + static constexpr size_t _Max_count = 4; static constexpr size_t _Scale = 1; static constexpr size_t _Carry_adjust = 8; @@ -2965,7 +2965,7 @@ namespace { struct _Search_n_traits_8 : _Find_traits_8 { using _MskX_t = uint32_t; - static constexpr size_t _Max_count = 4; + static constexpr size_t _Max_count = 2; static constexpr size_t _Scale = 1; static constexpr size_t _Carry_adjust = 4; @@ -2989,7 +2989,7 @@ namespace { if (_Count <= _Traits::_Max_count && _Length >= 32 && _Use_avx2()) { constexpr auto _Max_bits = _Traits::_Max_count * _Traits::_Scale; const size_t _Bits_count = _Count * _Traits::_Scale; - const size_t _Sh1 = _Bits_count < 4 ? _Bits_count - 2 : 2; + const size_t _Sh1 = _Max_bits > 2 ? (_Bits_count < 4 ? _Bits_count - 2 : 2) : 0; const size_t _Sh2 = _Max_bits > 4 ? (_Bits_count < 4 ? 0 : (_Bits_count < 8 ? _Bits_count - 4 : 4)) : 0; const size_t _Sh3 = _Max_bits > 8 ? (_Bits_count < 8 ? 0 : _Bits_count - 8) : 0; @@ -3008,7 +3008,9 @@ namespace { _MskX_t _MskX = _MskX_t{_Carry} | (_MskX_t{_Mask} << _Traits::_Carry_adjust); if constexpr (_Traits::_Scale == 1) { _MskX = (_MskX >> 1) & _MskX; - _MskX = (_MskX >> _Sh1) & _MskX; + if constexpr (_Max_bits > 2) { + _MskX = (_MskX >> _Sh1) & _MskX; + } } else if constexpr (_Traits::_Scale == 2) { _MskX = (_MskX >> 2) & _MskX; } From 6071339c7f18eba5af5592f7ce04b9211631ed9b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 21 Mar 2025 23:36:05 +0200 Subject: [PATCH 15/30] Simplify, optimize x86 --- stl/src/vector_algorithms.cpp | 108 ++++++++++++---------------------- 1 file changed, 38 insertions(+), 70 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 955df66ac48..05ad5fa6c15 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2930,50 +2930,6 @@ namespace { return _Result; } - struct _Search_n_traits_1 : _Find_traits_1 { - using _MskX_t = uint64_t; - static constexpr size_t _Max_count = 16; - static constexpr size_t _Scale = 1; - static constexpr size_t _Carry_adjust = 32; - - static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { - return _mm256_movemask_epi8(_mm256_cmpeq_epi8(_Comparand, _Data)); - } - }; - - struct _Search_n_traits_2 : _Find_traits_2 { - using _MskX_t = uint64_t; - static constexpr size_t _Max_count = 8; - static constexpr size_t _Scale = 2; - static constexpr size_t _Carry_adjust = 32; - - static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { - return _mm256_movemask_epi8(_mm256_cmpeq_epi16(_Comparand, _Data)); - } - }; - - struct _Search_n_traits_4 : _Find_traits_4 { - using _MskX_t = uint32_t; - static constexpr size_t _Max_count = 4; - static constexpr size_t _Scale = 1; - static constexpr size_t _Carry_adjust = 8; - - static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { - return _mm256_movemask_ps(_mm256_castsi256_ps(_mm256_cmpeq_epi32(_Comparand, _Data))); - } - }; - - struct _Search_n_traits_8 : _Find_traits_8 { - using _MskX_t = uint32_t; - static constexpr size_t _Max_count = 2; - static constexpr size_t _Scale = 1; - static constexpr size_t _Carry_adjust = 4; - - static int _Cmp_avx_elem_mask(const __m256i _Data, const __m256i _Comparand) noexcept { - return _mm256_movemask_pd(_mm256_castsi256_pd(_mm256_cmpeq_epi64(_Comparand, _Data))); - } - }; - template const void* __stdcall __std_search_n_impl( const void* _First, const void* const _Last, const size_t _Count, const _Ty _Val) noexcept { @@ -2986,12 +2942,11 @@ namespace { auto _Mid1 = static_cast(_First); const size_t _Length = _Byte_length(_First, _Last); - if (_Count <= _Traits::_Max_count && _Length >= 32 && _Use_avx2()) { - constexpr auto _Max_bits = _Traits::_Max_count * _Traits::_Scale; - const size_t _Bits_count = _Count * _Traits::_Scale; - const size_t _Sh1 = _Max_bits > 2 ? (_Bits_count < 4 ? _Bits_count - 2 : 2) : 0; - const size_t _Sh2 = _Max_bits > 4 ? (_Bits_count < 4 ? 0 : (_Bits_count < 8 ? _Bits_count - 4 : 4)) : 0; - const size_t _Sh3 = _Max_bits > 8 ? (_Bits_count < 8 ? 0 : _Bits_count - 8) : 0; + if (_Count <= (16 / sizeof(_Ty)) && _Length >= 32 && _Use_avx2()) { + const int _Bits_count = static_cast(_Count * sizeof(_Ty)); + const int _Sh1 = sizeof(_Ty) == 1 ? (_Bits_count < 4 ? _Bits_count - 2 : 2) : 0; + const int _Sh2 = sizeof(_Ty) < 4 ? (_Bits_count < 4 ? 0 : (_Bits_count < 8 ? _Bits_count - 4 : 4)) : 0; + const int _Sh3 = sizeof(_Ty) < 8 ? (_Bits_count < 8 ? 0 : _Bits_count - 8) : 0; const __m256i _Comparand = _Traits::_Set_avx(_Val); @@ -3001,29 +2956,42 @@ namespace { uint32_t _Carry = 0; do { const __m256i _Data = _mm256_loadu_si256(reinterpret_cast(_First)); - const auto _Mask = static_cast(_Traits::_Cmp_avx_elem_mask(_Comparand, _Data)); + const auto _Mask = static_cast(_mm256_movemask_epi8(_Traits::_Cmp_avx(_Comparand, _Data))); - using _MskX_t = _Traits::_MskX_t; + uint64_t _MskX = uint64_t{_Carry} | (uint64_t{_Mask} << 32); - _MskX_t _MskX = _MskX_t{_Carry} | (_MskX_t{_Mask} << _Traits::_Carry_adjust); - if constexpr (_Traits::_Scale == 1) { + if constexpr (sizeof(_Ty) == 1) { _MskX = (_MskX >> 1) & _MskX; - if constexpr (_Max_bits > 2) { - _MskX = (_MskX >> _Sh1) & _MskX; - } - } else if constexpr (_Traits::_Scale == 2) { + _MskX = __ull_rshift(_MskX, _Sh1) & _MskX; + } + if constexpr (sizeof(_Ty) == 2) { _MskX = (_MskX >> 2) & _MskX; } - if constexpr (_Max_bits > 4) { - _MskX = (_MskX >> _Sh2) & _MskX; - if constexpr (_Max_bits > 8) { - _MskX = (_MskX >> _Sh3) & _MskX; - } + if constexpr (sizeof(_Ty) < 4) { + _MskX = __ull_rshift(_MskX, _Sh2) & _MskX; + } + if constexpr (sizeof(_Ty) == 4) { + _MskX = (_MskX >> 4) & _MskX; + } + if constexpr (sizeof(_Ty) < 8) { + _MskX = __ull_rshift(_MskX, _Sh3) & _MskX; + } + if constexpr (sizeof(_Ty) == 8) { + _MskX = (_MskX >> 8) & _MskX; } if (_MskX != 0) { - int _Shift = static_cast(_tzcnt_u64(_MskX)) - _Traits::_Carry_adjust; - _Advance_bytes(_First, _Shift * (sizeof(_Ty) / _Traits::_Scale)); +#ifdef _M_IX86 + const uint32_t _MskLow = static_cast(_MskX); + const int _Shift = _MskLow != 0 ? static_cast(_tzcnt_u32(_MskLow)) - 32 + : static_cast(_tzcnt_u32(static_cast(_MskX >> 32))); + +#elifdef _M_X64 + const int _Shift = static_cast(_tzcnt_u64(_MskX)) - 32; +#else +#error Unsupported architecture +#endif + _Advance_bytes(_First, _Shift); return _First; } @@ -3032,8 +3000,8 @@ namespace { _Advance_bytes(_First, 32); } while (_First != _Stop_at); - _Rewind_bytes(_First, _Count * (sizeof(_Ty))); _Mid1 = static_cast(_First); + _Rewind_bytes(_First, _lzcnt_u32(~_Carry)); } auto _Match_start = static_cast(_First); @@ -4707,22 +4675,22 @@ __declspec(noalias) size_t __stdcall __std_count_trivial_8( const void* __stdcall __std_search_n_1( const void* const _First, const void* const _Last, const size_t _Count, const uint8_t _Value) noexcept { - return __std_search_n_impl<_Search_n_traits_1>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Find_traits_1>(_First, _Last, _Count, _Value); } const void* __stdcall __std_search_n_2( const void* const _First, const void* const _Last, const size_t _Count, const uint16_t _Value) noexcept { - return __std_search_n_impl<_Search_n_traits_2>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Find_traits_2>(_First, _Last, _Count, _Value); } const void* __stdcall __std_search_n_4( const void* const _First, const void* const _Last, const size_t _Count, const uint32_t _Value) noexcept { - return __std_search_n_impl<_Search_n_traits_4>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Find_traits_4>(_First, _Last, _Count, _Value); } const void* __stdcall __std_search_n_8( const void* const _First, const void* const _Last, const size_t _Count, const uint64_t _Value) noexcept { - return __std_search_n_impl<_Search_n_traits_8>(_First, _Last, _Count, _Value); + return __std_search_n_impl<_Find_traits_8>(_First, _Last, _Count, _Value); } const void* __stdcall __std_find_first_of_trivial_1( From 1c7b6d3363cf5c58614b734f16c023141e1d6e7a Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 22 Mar 2025 15:41:09 +0200 Subject: [PATCH 16/30] ARM64EC --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 05ad5fa6c15..29629c0172a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2940,7 +2940,7 @@ namespace { } auto _Mid1 = static_cast(_First); - +#ifndef _M_ARM64EC const size_t _Length = _Byte_length(_First, _Last); if (_Count <= (16 / sizeof(_Ty)) && _Length >= 32 && _Use_avx2()) { const int _Bits_count = static_cast(_Count * sizeof(_Ty)); @@ -3003,7 +3003,7 @@ namespace { _Mid1 = static_cast(_First); _Rewind_bytes(_First, _lzcnt_u32(~_Carry)); } - +#endif // !_M_ARM64EC auto _Match_start = static_cast(_First); auto _Last_ptr = static_cast(_Last); From 1ab6a558bc50d7ef5ff7ac03b664e5752219267f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 22 Mar 2025 15:42:28 +0200 Subject: [PATCH 17/30] vzerouuper --- stl/src/vector_algorithms.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 29629c0172a..f522b057390 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2943,6 +2943,8 @@ namespace { #ifndef _M_ARM64EC const size_t _Length = _Byte_length(_First, _Last); if (_Count <= (16 / sizeof(_Ty)) && _Length >= 32 && _Use_avx2()) { + _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 + const int _Bits_count = static_cast(_Count * sizeof(_Ty)); const int _Sh1 = sizeof(_Ty) == 1 ? (_Bits_count < 4 ? _Bits_count - 2 : 2) : 0; const int _Sh2 = sizeof(_Ty) < 4 ? (_Bits_count < 4 ? 0 : (_Bits_count < 8 ? _Bits_count - 4 : 4)) : 0; From 37fe2bcbf176ba6a3a20a8c7c052099f2d3ddb5a Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 22 Mar 2025 23:30:10 +0200 Subject: [PATCH 18/30] avoid unnecessary bit width conversion --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f522b057390..52b39491f46 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2989,7 +2989,7 @@ namespace { : static_cast(_tzcnt_u32(static_cast(_MskX >> 32))); #elifdef _M_X64 - const int _Shift = static_cast(_tzcnt_u64(_MskX)) - 32; + const long long _Shift = static_cast(_tzcnt_u64(_MskX)) - 32; #else #error Unsupported architecture #endif From 8373fede2ef3a54b90dce42090ae9cb42aad8a30 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 25 Mar 2025 08:26:24 +0200 Subject: [PATCH 19/30] 120 --- stl/src/vector_algorithms.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 86acc6cec0d..624f6f94071 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3094,7 +3094,8 @@ namespace { uint32_t _Carry = 0; do { const __m256i _Data = _mm256_loadu_si256(reinterpret_cast(_First)); - const auto _Mask = static_cast(_mm256_movemask_epi8(_Traits::_Cmp_avx(_Comparand, _Data))); + + const auto _Mask = static_cast(_mm256_movemask_epi8(_Traits::_Cmp_avx(_Comparand, _Data))); uint64_t _MskX = uint64_t{_Carry} | (uint64_t{_Mask} << 32); From b3b492b7f800b0b02b256bbe537cf0f3636ee3ec Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 25 Mar 2025 08:39:06 +0200 Subject: [PATCH 20/30] 120 --- stl/src/vector_algorithms.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 624f6f94071..1f8a4f15189 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3122,8 +3122,9 @@ namespace { if (_MskX != 0) { #ifdef _M_IX86 const uint32_t _MskLow = static_cast(_MskX); - const int _Shift = _MskLow != 0 ? static_cast(_tzcnt_u32(_MskLow)) - 32 - : static_cast(_tzcnt_u32(static_cast(_MskX >> 32))); + + const int _Shift = _MskLow != 0 ? static_cast(_tzcnt_u32(_MskLow)) - 32 + : static_cast(_tzcnt_u32(static_cast(_MskX >> 32))); #elifdef _M_X64 const long long _Shift = static_cast(_tzcnt_u64(_MskX)) - 32; From 49916b7b7c30f946bf7ba6d0e31291345a0f433c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 14:24:28 -0700 Subject: [PATCH 21/30] Drop unnecessary parens. --- stl/inc/algorithm | 2 +- stl/src/vector_algorithms.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 8838f003184..bdcf24123fb 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -2334,7 +2334,7 @@ _NODISCARD _CONSTEXPR20 _FwdIt search_n( if constexpr (is_pointer_v) { _UFirst = _Result; } else { - _UFirst += (_Result - _First_ptr); + _UFirst += _Result - _First_ptr; } _STD _Seek_wrapped(_Last, _UFirst); diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f25ce5c2c94..4667c15a4cd 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3078,7 +3078,7 @@ namespace { auto _Mid1 = static_cast(_First); #ifndef _M_ARM64EC const size_t _Length = _Byte_length(_First, _Last); - if (_Count <= (16 / sizeof(_Ty)) && _Length >= 32 && _Use_avx2()) { + if (_Count <= 16 / sizeof(_Ty) && _Length >= 32 && _Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 const int _Bits_count = static_cast(_Count * sizeof(_Ty)); From b73964b1f0ff2aed19c4ffe07dddee5739cf0e26 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 14:33:46 -0700 Subject: [PATCH 22/30] Use `_Last_ptr`. --- stl/inc/algorithm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index bdcf24123fb..566a097abae 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -2525,8 +2525,8 @@ namespace ranges { const auto _First_ptr = _STD _To_address(_First); const auto _Last_ptr = _First_ptr + _Dist; - const auto _Result = _STD _Search_n_vectorized( - _First_ptr, _First_ptr + _Dist, static_cast(_Count), _Val); + const auto _Result = + _STD _Search_n_vectorized(_First_ptr, _Last_ptr, static_cast(_Count), _Val); if constexpr (is_pointer_v<_It>) { if (_Result != _Last_ptr) { From 7e702ed431c799755a377b464314f9ea06bbe663 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 14:39:48 -0700 Subject: [PATCH 23/30] Fix typos. --- benchmarks/src/search_n.cpp | 46 +++++++++---------- .../test.cpp | 2 +- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index cd2d6cde1d8..eb2900c07b0 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -17,12 +17,12 @@ using namespace std; enum class AlgType { Std, Rng }; -enum class PartternType { +enum class PatternType { TwoZones, DenseSmallSequences, }; -template +template void bm(benchmark::State& state) { const auto size = static_cast(state.range(0)); const auto n = static_cast(state.range(1)); @@ -32,9 +32,9 @@ void bm(benchmark::State& state) { vector> v(size, no_match); - if constexpr (Parttern == PartternType::TwoZones) { + if constexpr (Pattern == PatternType::TwoZones) { fill(v.begin() + v.size() / 2, v.end(), match); - } else if constexpr (Parttern == PartternType::DenseSmallSequences) { + } else if constexpr (Pattern == PatternType::DenseSmallSequences) { if (size != 0 && n != 0) { mt19937 gen{7687239}; @@ -68,25 +68,25 @@ void common_args(auto bm) { } } -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); - -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); - -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); - -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); -BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); + +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); +BENCHMARK(bm)->Apply(common_args); BENCHMARK_MAIN(); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp index 8596534bd9c..80e2966a5df 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -14,7 +14,7 @@ using namespace std; template auto last_known_good_search_n(FwdIt first, FwdIt last, size_t count, T val) { - // Delibarately using simple approach, not smart bidi/random iterators "check from the other end" stuff + // Deliberately using simple approach, not smart bidi/random iterators "check from the other end" stuff if (count == 0) { return first; } From 0f60ba75ab9df568a484afb98e109d21245983b9 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 14:43:48 -0700 Subject: [PATCH 24/30] Add newlines. --- stl/src/vector_algorithms.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 4667c15a4cd..738469c9cb1 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3103,18 +3103,23 @@ namespace { _MskX = (_MskX >> 1) & _MskX; _MskX = __ull_rshift(_MskX, _Sh1) & _MskX; } + if constexpr (sizeof(_Ty) == 2) { _MskX = (_MskX >> 2) & _MskX; } + if constexpr (sizeof(_Ty) < 4) { _MskX = __ull_rshift(_MskX, _Sh2) & _MskX; } + if constexpr (sizeof(_Ty) == 4) { _MskX = (_MskX >> 4) & _MskX; } + if constexpr (sizeof(_Ty) < 8) { _MskX = __ull_rshift(_MskX, _Sh3) & _MskX; } + if constexpr (sizeof(_Ty) == 8) { _MskX = (_MskX >> 8) & _MskX; } From a4ac68df8bbcb3e695ccf234861577bb5ece5e07 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 15:08:51 -0700 Subject: [PATCH 25/30] `_Bits_count` => `_Bytes_count` --- stl/src/vector_algorithms.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 738469c9cb1..a54a130aaae 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3081,10 +3081,10 @@ namespace { if (_Count <= 16 / sizeof(_Ty) && _Length >= 32 && _Use_avx2()) { _Zeroupper_on_exit _Guard; // TRANSITION, DevCom-10331414 - const int _Bits_count = static_cast(_Count * sizeof(_Ty)); - const int _Sh1 = sizeof(_Ty) == 1 ? (_Bits_count < 4 ? _Bits_count - 2 : 2) : 0; - const int _Sh2 = sizeof(_Ty) < 4 ? (_Bits_count < 4 ? 0 : (_Bits_count < 8 ? _Bits_count - 4 : 4)) : 0; - const int _Sh3 = sizeof(_Ty) < 8 ? (_Bits_count < 8 ? 0 : _Bits_count - 8) : 0; + const int _Bytes_count = static_cast(_Count * sizeof(_Ty)); + const int _Sh1 = sizeof(_Ty) == 1 ? (_Bytes_count < 4 ? _Bytes_count - 2 : 2) : 0; + const int _Sh2 = sizeof(_Ty) < 4 ? (_Bytes_count < 4 ? 0 : (_Bytes_count < 8 ? _Bytes_count - 4 : 4)) : 0; + const int _Sh3 = sizeof(_Ty) < 8 ? (_Bytes_count < 8 ? 0 : _Bytes_count - 8) : 0; const __m256i _Comparand = _Traits::_Set_avx(_Val); From 80d160ed846fe539effbe05666f0b5a769f08d6d Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 15:26:17 -0700 Subject: [PATCH 26/30] Add const. --- stl/src/vector_algorithms.cpp | 4 ++-- .../std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index a54a130aaae..623e49632b6 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3149,8 +3149,8 @@ namespace { _Rewind_bytes(_First, _lzcnt_u32(~_Carry)); } #endif // !_M_ARM64EC - auto _Match_start = static_cast(_First); - auto _Last_ptr = static_cast(_Last); + auto _Match_start = static_cast(_First); + const auto _Last_ptr = static_cast(_Last); if (static_cast(_Last_ptr - _Match_start) < _Count) { return _Last_ptr; diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp index 80e2966a5df..b9b248c3a0f 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -13,7 +13,7 @@ using namespace std; template -auto last_known_good_search_n(FwdIt first, FwdIt last, size_t count, T val) { +auto last_known_good_search_n(FwdIt first, const FwdIt last, const size_t count, const T val) { // Deliberately using simple approach, not smart bidi/random iterators "check from the other end" stuff if (count == 0) { return first; From 0352dd6c126e3c412ba73726822fe9d2a50838f3 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 15:44:37 -0700 Subject: [PATCH 27/30] Adjust headers. --- benchmarks/src/search_n.cpp | 1 - tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmarks/src/search_n.cpp b/benchmarks/src/search_n.cpp index eb2900c07b0..c20bb673875 100644 --- a/benchmarks/src/search_n.cpp +++ b/benchmarks/src/search_n.cpp @@ -6,7 +6,6 @@ #include #include #include -#include #include #include "skewed_allocator.hpp" diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp index b9b248c3a0f..6a0b337183f 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include From 5db47ac69a59d5e8036e292544b70c043c9ab546 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 16:16:02 -0700 Subject: [PATCH 28/30] Simplify last_known_good_search_n further. --- .../test.cpp | 26 +++++++------------ 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp index 6a0b337183f..5e331a285c6 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -21,26 +21,20 @@ auto last_known_good_search_n(FwdIt first, const FwdIt last, const size_t count, return first; } + size_t found = 0; + FwdIt match{}; for (; first != last; ++first) { if (*first == val) { - FwdIt match = first; - size_t match_size = count; - for (;;) { - --match_size; - if (match_size == 0) { - return match; - } - - ++first; - - if (first == last) { - return last; - } + ++found; + if (found == 1) { + match = first; + } - if (*first != val) { - break; - } + if (found == count) { + return match; } + } else { + found = 0; } } return last; From 0cbca6683d14be96d2734256ef3b59a7613d80bd Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 16:33:33 -0700 Subject: [PATCH 29/30] Guard `input.size() - 1` against empty input. --- .../std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp index 5e331a285c6..59bd70e0b55 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -76,6 +76,10 @@ void test_search_n(mt19937_64& gen) { test_case_search_n(input, count, val); + if (input.empty()) { + continue; + } + binomial_distribution pattern_length_dis(count * 2, 0.5); uniform_int_distribution pos_dis(0, input.size() - 1); From e7ed6b490d77f25251f329e70b09cda589c3b8df Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Sun, 20 Apr 2025 16:37:40 -0700 Subject: [PATCH 30/30] Allow the pattern to be placed at the last possible position. --- tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp index 59bd70e0b55..04b3907b83c 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms_search_n/test.cpp @@ -87,7 +87,7 @@ void test_search_n(mt19937_64& gen) { const size_t pattern_length = pattern_length_dis(gen); const size_t pattern_pos = pos_dis(gen); - if (pattern_length + pattern_pos < input.size()) { + if (pattern_length + pattern_pos <= input.size()) { fill_n(input.begin() + static_cast(pattern_pos), pattern_length, val); test_case_search_n(input, count, val);