From 2b28a09efd1bd0f1e738f2551cf0ad27fca4d2a1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 10 Mar 2024 10:59:51 +0200 Subject: [PATCH 01/17] Vectorize find_first_of --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/find_first_of.cpp | 45 +++++++++++ stl/inc/algorithm | 52 +++++++++++++ stl/src/vector_algorithms.cpp | 78 +++++++++++++++++++ .../VSO_0000000_vector_algorithms/test.cpp | 58 ++++++++++++++ 5 files changed, 234 insertions(+) create mode 100644 benchmarks/src/find_first_of.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 2903ee13245..8a463a154f8 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -110,6 +110,7 @@ endfunction() add_benchmark(bitset_to_string src/bitset_to_string.cpp) add_benchmark(find_and_count src/find_and_count.cpp) +add_benchmark(find_first_of src/find_first_of.cpp) add_benchmark(locale_classic src/locale_classic.cpp) add_benchmark(minmax_element src/minmax_element.cpp) add_benchmark(path_lexically_normal src/path_lexically_normal.cpp) diff --git a/benchmarks/src/find_first_of.cpp b/benchmarks/src/find_first_of.cpp new file mode 100644 index 00000000000..ac7bc88b356 --- /dev/null +++ b/benchmarks/src/find_first_of.cpp @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include + +using namespace std; + +template +void bm(benchmark::State& state) { + vector h(HSize, T{'.'}); + vector n(NSize); + iota(n.begin(), n.end(), T{'a'}); + + static_assert(Pos < HSize); + static_assert(Which < NSize); + h.at(Pos) = n.at(Which); + + for (auto _ : state) { + benchmark::DoNotOptimize(find_first_of(h.begin(), h.end(), n.begin(), n.end())); + } +} + +BENCHMARK(bm); +BENCHMARK(bm); + +BENCHMARK(bm); +BENCHMARK(bm); + +BENCHMARK(bm); +BENCHMARK(bm); + +BENCHMARK(bm); +BENCHMARK(bm); + +BENCHMARK(bm); +BENCHMARK(bm); + +BENCHMARK(bm); +BENCHMARK(bm); + +BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 06152ed95db..36fab1e1087 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -58,6 +58,11 @@ const void* __stdcall __std_find_last_trivial_2(const void* _First, const void* const void* __stdcall __std_find_last_trivial_4(const void* _First, const void* _Last, uint32_t _Val) noexcept; const void* __stdcall __std_find_last_trivial_8(const void* _First, const void* _Last, uint64_t _Val) noexcept; +const void* __stdcall __std_find_first_of_trivial_1( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; +const void* __stdcall __std_find_first_of_trivial_2( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept; + __declspec(noalias) _Min_max_1i __stdcall __std_minmax_1i(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_1u __stdcall __std_minmax_1u(const void* _First, const void* _Last) noexcept; __declspec(noalias) _Min_max_2i __stdcall __std_minmax_2i(const void* _First, const void* _Last) noexcept; @@ -160,6 +165,33 @@ _Ty* __std_find_last_trivial(_Ty* const _First, _Ty* const _Last, const _TVal _V static_assert(_Always_false<_Ty>, "Unexpected size"); } } + +template +_Ty* __std_find_first_of_trivial( + _Ty* const _First1, _Ty* const _Last1, _Ty* const _First2, _Ty* const _Last2) noexcept { + if constexpr (sizeof(_Ty) == 1) { + return const_cast<_Ty*>( + static_cast(::__std_find_first_of_trivial_1(_First1, _Last1, _First2, _Last2))); + } else if constexpr (sizeof(_Ty) == 2) { + return const_cast<_Ty*>( + static_cast(::__std_find_first_of_trivial_2(_First1, _Last1, _First2, _Last2))); + } else { + static_assert(_Always_false<_Ty>, "Unexpected size"); + } +} + +// Can we activate the vector algorithms for find_first_of? +template , class _Ty2 = _Iter_value_t<_It2>> +_INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = + _Iterator_is_contiguous<_It1> && _Iterator_is_contiguous<_It2> // contigous + && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile + && is_same_v<_Ty1, _Ty2> // same types + && sizeof(_Ty1) <= 2 // pcmpestri compatible size + && disjunction_v< +#ifdef __cpp_lib_byte + is_same<_Ty1, byte>, // byte or... +#endif // defined(__cpp_lib_byte) + is_integral<_Ty1>>; // ...integer _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -3321,6 +3353,26 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( const auto _ULast1 = _STD _Get_unwrapped(_Last1); const auto _UFirst2 = _STD _Get_unwrapped(_First2); const auto _ULast2 = _STD _Get_unwrapped(_Last2); +#if _USE_STD_VECTOR_ALGORITHMS + using _Ty1 = _Iter_value_t<_FwdIt1>; + + constexpr ptrdiff_t _Threshold = 16; // vectorization is likely to be a win after this size + + if constexpr (_Vector_alg_in_find_first_of_is_safe<_FwdIt1, _FwdIt2> + && _Is_any_of_v<_Pr, equal_to<>, equal_to<_Ty1>>) { + if (!_STD _Is_constant_evaluated() && _ULast1 - _UFirst1 >= _Threshold) { + const auto _First1_ptr = _STD _To_address(_UFirst1); + const auto _Result = _STD __std_find_first_of_trivial(_First1_ptr, _STD _To_address(_ULast1), _STD _To_address(_UFirst2), _STD _To_address(_ULast2)); + + if constexpr (is_pointer_v) { + _UFirst1 = _Result; + } else { + _UFirst1 += (_Result - _First1_ptr); + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS + for (; _UFirst1 != _ULast1; ++_UFirst1) { for (auto _UMid2 = _UFirst2; _UMid2 != _ULast2; ++_UMid2) { if (_Pred(*_UFirst1, *_UMid2)) { diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index bf9de5a308d..21740f391fd 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2075,6 +2075,74 @@ namespace { } return _Result; } + + template + const void* __stdcall __std_find_first_of_trivial_impl( + const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { + + constexpr bool _Bytes = sizeof(_Ty) == 1; + constexpr int _Op = + (_Bytes ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; + constexpr int _Part_size_el = _Bytes ? 16 : 8; + + const size_t _Needle_length = _Byte_length(_First2, _Last2); + + if (_Use_sse42() && _Needle_length <= 16) { + const int _Needle_length_el = static_cast(_Byte_length(_First2, _Last2) / sizeof(_Ty)); + + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _First2, _Needle_length); + const __m128i _Needle = _mm_load_si128(reinterpret_cast(_Tmp1)); + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + + while (_First1 != _Stop_at) { + const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); + + if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + return _First1; + } + + _Advance_bytes(_First1, 16); + } + + const size_t _Last_part_size = (_Haystack_length & 0xF); + const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); + + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First1, _Last_part_size); + const __m128i _Haystack_last_part = _mm_loadu_si128(reinterpret_cast(_Tmp2)); + + if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + return _First1; + } + + _Advance_bytes(_First1, _Last_part_size); + return _First1; + } + + auto _Ptr_haystack = static_cast(_First1); + const auto _Ptr_haystack_end = static_cast(_Last1); + const auto _Ptr_needle = static_cast(_First2); + const auto _Ptr_needle_end = static_cast(_Last2); + + for (; _Ptr_haystack != _Ptr_haystack_end; ++_Ptr_haystack) { + for (auto _Ptr = _Ptr_needle; _Ptr != _Ptr_needle_end; ++_Ptr) { + if (*_Ptr_haystack == *_Ptr) { + return _Ptr_haystack; + } + } + } + + return _Ptr_haystack; + } + } // unnamed namespace extern "C" { @@ -2155,6 +2223,16 @@ __declspec(noalias) size_t return __std_count_trivial_impl<_Find_traits_8>(_First, _Last, _Val); } +const void* __stdcall __std_find_first_of_trivial_1( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { + return __std_find_first_of_trivial_impl(_First1, _Last1, _First2, _Last2); +} + +const void* __stdcall __std_find_first_of_trivial_2( + const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept { + return __std_find_first_of_trivial_impl(_First1, _Last1, _First2, _Last2); +} + } // extern "C" #ifndef _M_ARM64EC diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 510c7b99353..ebde31e892d 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -120,6 +120,18 @@ auto last_known_good_find_last(FwdIt first, FwdIt last, T v) { } } +template +auto last_known_good_find_first_of(FwdItH h_first, FwdItH h_last, FwdItN n_first, FwdItN n_last) { + for (; h_first != h_last; ++h_first) { + for (FwdItN n = n_first; n != n_last; ++n) { + if (*h_first == *n) { + return h_first; + } + } + } + return h_first; +} + template void test_case_find(const vector& input, T v) { auto expected = last_known_good_find(input.begin(), input.end(), v); @@ -211,6 +223,42 @@ void test_find_last(mt19937_64& gen) { } #endif // _HAS_CXX23 +template +void test_case_find_first_of(const vector& input_haystack, const vector& input_needle) { + auto expected = last_known_good_find_first_of( + input_haystack.begin(), input_haystack.end(), input_needle.begin(), input_needle.end()); + auto actual = find_first_of(input_haystack.begin(), input_haystack.end(), input_needle.begin(), input_needle.end()); + assert(expected == actual); +} + +template +void test_find_first_of(mt19937_64& gen) { + constexpr size_t needleDataCount = 30; + using TD = conditional_t; + uniform_int_distribution dis('a', 'z'); + vector input_haystack; + vector input_needle; + input_haystack.reserve(dataCount); + input_needle.reserve(needleDataCount); + for (;;) { + + input_needle.clear(); + + test_case_find_first_of(input_haystack, input_needle); + for (size_t attempts = 0; attempts < needleDataCount; ++attempts) { + input_needle.push_back(static_cast(dis(gen))); + test_case_find_first_of(input_haystack, input_needle); + } + + if (input_haystack.size() == dataCount) { + break; + } + + input_haystack.push_back(static_cast(dis(gen))); + } +} + + template void test_min_max_element(mt19937_64& gen) { using Limits = numeric_limits; @@ -437,6 +485,16 @@ void test_vector_algorithms(mt19937_64& gen) { test_find_last(gen); #endif // _HAS_CXX23 + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_find_first_of(gen); + test_min_max_element(gen); test_min_max_element(gen); test_min_max_element(gen); From 9a3cc9fc19ea21835ec4cf664b8e44e139d174b6 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 10 Mar 2024 16:35:16 +0200 Subject: [PATCH 02/17] align load --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 21740f391fd..c204ffedcc7 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2115,7 +2115,7 @@ namespace { alignas(16) uint8_t _Tmp2[16]; memcpy(_Tmp2, _First1, _Last_part_size); - const __m128i _Haystack_last_part = _mm_loadu_si128(reinterpret_cast(_Tmp2)); + const __m128i _Haystack_last_part = _mm_load_si128(reinterpret_cast(_Tmp2)); if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op)) { const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op); From 8ed06e3aa79b0b3533483c2dde911086b6a8b274 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 10 Mar 2024 16:44:31 +0200 Subject: [PATCH 03/17] format --- stl/inc/algorithm | 5 +++-- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 36fab1e1087..37aa225601a 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -184,7 +184,7 @@ _Ty* __std_find_first_of_trivial( template , class _Ty2 = _Iter_value_t<_It2>> _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = _Iterator_is_contiguous<_It1> && _Iterator_is_contiguous<_It2> // contigous - && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile + && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile && is_same_v<_Ty1, _Ty2> // same types && sizeof(_Ty1) <= 2 // pcmpestri compatible size && disjunction_v< @@ -3362,7 +3362,8 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( && _Is_any_of_v<_Pr, equal_to<>, equal_to<_Ty1>>) { if (!_STD _Is_constant_evaluated() && _ULast1 - _UFirst1 >= _Threshold) { const auto _First1_ptr = _STD _To_address(_UFirst1); - const auto _Result = _STD __std_find_first_of_trivial(_First1_ptr, _STD _To_address(_ULast1), _STD _To_address(_UFirst2), _STD _To_address(_ULast2)); + const auto _Result = _STD __std_find_first_of_trivial( + _First1_ptr, _STD _To_address(_ULast1), _STD _To_address(_UFirst2), _STD _To_address(_ULast2)); if constexpr (is_pointer_v) { _UFirst1 = _Result; diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index ebde31e892d..608c70f7bce 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -234,7 +234,7 @@ void test_case_find_first_of(const vector& input_haystack, const vector& i template void test_find_first_of(mt19937_64& gen) { constexpr size_t needleDataCount = 30; - using TD = conditional_t; + using TD = conditional_t; uniform_int_distribution dis('a', 'z'); vector input_haystack; vector input_needle; @@ -253,12 +253,11 @@ void test_find_first_of(mt19937_64& gen) { if (input_haystack.size() == dataCount) { break; } - + input_haystack.push_back(static_cast(dis(gen))); } } - template void test_min_max_element(mt19937_64& gen) { using Limits = numeric_limits; From a1241dfe4a9d8efb3827b0510f0a278d1461983e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 10 Mar 2024 20:17:24 +0200 Subject: [PATCH 04/17] the fall back is unnecessary --- stl/inc/algorithm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 37aa225601a..c774831bafe 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -3370,6 +3370,8 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( } else { _UFirst1 += (_Result - _First1_ptr); } + _STD _Seek_wrapped(_First1, _UFirst1); + return _First1; } } #endif // _USE_STD_VECTOR_ALGORITHMS From 2ee66d31d68e7b5df21a99f8a6054108596ec9a3 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 10 Mar 2024 23:41:35 +0200 Subject: [PATCH 05/17] arm --- stl/src/vector_algorithms.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index c204ffedcc7..67b558ad87b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2079,7 +2079,7 @@ namespace { template const void* __stdcall __std_find_first_of_trivial_impl( const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { - +#ifndef _M_ARM64EC constexpr bool _Bytes = sizeof(_Ty) == 1; constexpr int _Op = (_Bytes ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; @@ -2126,6 +2126,7 @@ namespace { _Advance_bytes(_First1, _Last_part_size); return _First1; } +#endif // !_M_ARM64EC auto _Ptr_haystack = static_cast(_First1); const auto _Ptr_haystack_end = static_cast(_Last1); From 09531cae60457b53ab4c8c53dee65d6ff04ebb82 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 11 Mar 2024 11:41:48 +0200 Subject: [PATCH 06/17] embed predicate check --- stl/inc/algorithm | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index c774831bafe..508a87029df 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -181,7 +181,7 @@ _Ty* __std_find_first_of_trivial( } // Can we activate the vector algorithms for find_first_of? -template , class _Ty2 = _Iter_value_t<_It2>> +template , class _Ty2 = _Iter_value_t<_It2>> _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = _Iterator_is_contiguous<_It1> && _Iterator_is_contiguous<_It2> // contigous && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile @@ -191,7 +191,12 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = #ifdef __cpp_lib_byte is_same<_Ty1, byte>, // byte or... #endif // defined(__cpp_lib_byte) - is_integral<_Ty1>>; // ...integer + is_integral<_Ty1>> // ...integer + && _Is_any_of_v<_Pr, // predicate is equal +#if _HAS_CXX20 + _RANGES equal_to, +#endif // _HAS_CXX20 + equal_to<>, equal_to<_Ty1>>; _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS @@ -3358,8 +3363,7 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( constexpr ptrdiff_t _Threshold = 16; // vectorization is likely to be a win after this size - if constexpr (_Vector_alg_in_find_first_of_is_safe<_FwdIt1, _FwdIt2> - && _Is_any_of_v<_Pr, equal_to<>, equal_to<_Ty1>>) { + if constexpr (_Vector_alg_in_find_first_of_is_safe<_FwdIt1, _FwdIt2, _Pr>) { if (!_STD _Is_constant_evaluated() && _ULast1 - _UFirst1 >= _Threshold) { const auto _First1_ptr = _STD _To_address(_UFirst1); const auto _Result = _STD __std_find_first_of_trivial( From 05f6312bac0dc2f619ba3ccc19fe13f8eac82088 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 11 Mar 2024 12:07:40 +0200 Subject: [PATCH 07/17] fix test --- stl/inc/algorithm | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 508a87029df..936bd80c683 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -194,7 +194,7 @@ _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = is_integral<_Ty1>> // ...integer && _Is_any_of_v<_Pr, // predicate is equal #if _HAS_CXX20 - _RANGES equal_to, + ranges::equal_to, #endif // _HAS_CXX20 equal_to<>, equal_to<_Ty1>>; _STD_END @@ -3359,8 +3359,6 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( const auto _UFirst2 = _STD _Get_unwrapped(_First2); const auto _ULast2 = _STD _Get_unwrapped(_Last2); #if _USE_STD_VECTOR_ALGORITHMS - using _Ty1 = _Iter_value_t<_FwdIt1>; - constexpr ptrdiff_t _Threshold = 16; // vectorization is likely to be a win after this size if constexpr (_Vector_alg_in_find_first_of_is_safe<_FwdIt1, _FwdIt2, _Pr>) { From f283ddaef2e529f94fc2cd135ac1deda390af851 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 09:06:30 +0200 Subject: [PATCH 08/17] constantness mix +coverage --- stl/inc/algorithm | 33 ++++++++++--------- .../VSO_0000000_vector_algorithms/test.cpp | 18 +++++++++- 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 936bd80c683..5cfc27c0d14 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -166,32 +166,33 @@ _Ty* __std_find_last_trivial(_Ty* const _First, _Ty* const _Last, const _TVal _V } } -template -_Ty* __std_find_first_of_trivial( - _Ty* const _First1, _Ty* const _Last1, _Ty* const _First2, _Ty* const _Last2) noexcept { - if constexpr (sizeof(_Ty) == 1) { - return const_cast<_Ty*>( - static_cast(::__std_find_first_of_trivial_1(_First1, _Last1, _First2, _Last2))); - } else if constexpr (sizeof(_Ty) == 2) { - return const_cast<_Ty*>( - static_cast(::__std_find_first_of_trivial_2(_First1, _Last1, _First2, _Last2))); +template +_Ty1* __std_find_first_of_trivial( + _Ty1* const _First1, _Ty1* const _Last1, _Ty2* const _First2, _Ty2* const _Last2) noexcept { + if constexpr (sizeof(_Ty1) == 1) { + return const_cast<_Ty1*>( + static_cast(::__std_find_first_of_trivial_1(_First1, _Last1, _First2, _Last2))); + } else if constexpr (sizeof(_Ty1) == 2) { + return const_cast<_Ty1*>( + static_cast(::__std_find_first_of_trivial_2(_First1, _Last1, _First2, _Last2))); } else { - static_assert(_Always_false<_Ty>, "Unexpected size"); + static_assert(_Always_false<_Ty1>, "Unexpected size"); } } // Can we activate the vector algorithms for find_first_of? template , class _Ty2 = _Iter_value_t<_It2>> _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = - _Iterator_is_contiguous<_It1> && _Iterator_is_contiguous<_It2> // contigous + _Iterators_are_contiguous<_It1,_It2> // contiguous && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile - && is_same_v<_Ty1, _Ty2> // same types - && sizeof(_Ty1) <= 2 // pcmpestri compatible size && disjunction_v< #ifdef __cpp_lib_byte - is_same<_Ty1, byte>, // byte or... + conjunction, is_same<_Ty2, byte>>, // byte or... #endif // defined(__cpp_lib_byte) - is_integral<_Ty1>> // ...integer + conjunction, is_integral<_Ty2>>> // ...integer + && sizeof(_Ty1) == sizeof(_Ty2) // same sizes + && is_signed_v<_Ty1> == is_signed_v<_Ty2> // same signedness + && sizeof(_Ty1) <= 2 // pcmpestri compatible size && _Is_any_of_v<_Pr, // predicate is equal #if _HAS_CXX20 ranges::equal_to, @@ -3370,7 +3371,7 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( if constexpr (is_pointer_v) { _UFirst1 = _Result; } else { - _UFirst1 += (_Result - _First1_ptr); + _UFirst1 += _Result - _First1_ptr; } _STD _Seek_wrapped(_First1, _UFirst1); return _First1; diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 608c70f7bce..52ea16ba5dc 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -240,8 +240,8 @@ void test_find_first_of(mt19937_64& gen) { vector input_needle; input_haystack.reserve(dataCount); input_needle.reserve(needleDataCount); - for (;;) { + for (;;) { input_needle.clear(); test_case_find_first_of(input_haystack, input_needle); @@ -258,6 +258,14 @@ void test_find_first_of(mt19937_64& gen) { } } +template +void test_find_first_of_containers() { + C1 haystack{'m', 'e', 'o', 'w', 'C', 'A', 'T', 'S'}; + C2 needle{'R', 'S', 'T'}; + const auto result = find_first_of(haystack.begin(), haystack.end(), needle.begin(), needle.end()); + assert(result == haystack.begin() + 6); +} + template void test_min_max_element(mt19937_64& gen) { using Limits = numeric_limits; @@ -494,6 +502,14 @@ void test_vector_algorithms(mt19937_64& gen) { test_find_first_of(gen); test_find_first_of(gen); + test_find_first_of_containers, vector>(); + test_find_first_of_containers, vector>(); + test_find_first_of_containers, vector>(); + test_find_first_of_containers, const vector>(); + test_find_first_of_containers, const vector>(); + test_find_first_of_containers, vector>(); + test_find_first_of_containers, vector>(); + test_min_max_element(gen); test_min_max_element(gen); test_min_max_element(gen); From f83cfbb3211fbc793ae180721cdcb3383be8aeab Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 09:27:56 +0200 Subject: [PATCH 09/17] at() -> [] --- benchmarks/src/find_first_of.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/src/find_first_of.cpp b/benchmarks/src/find_first_of.cpp index ac7bc88b356..80b2c9b8c9c 100644 --- a/benchmarks/src/find_first_of.cpp +++ b/benchmarks/src/find_first_of.cpp @@ -17,7 +17,7 @@ void bm(benchmark::State& state) { static_assert(Pos < HSize); static_assert(Which < NSize); - h.at(Pos) = n.at(Which); + h[Pos] = n[Which[)]; for (auto _ : state) { benchmark::DoNotOptimize(find_first_of(h.begin(), h.end(), n.begin(), n.end())); From 7e332243ae5d134541786e69ed26fafd284dbfde Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 09:29:06 +0200 Subject: [PATCH 10/17] Missing include --- benchmarks/src/find_first_of.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/src/find_first_of.cpp b/benchmarks/src/find_first_of.cpp index 80b2c9b8c9c..2d95dfb7845 100644 --- a/benchmarks/src/find_first_of.cpp +++ b/benchmarks/src/find_first_of.cpp @@ -6,6 +6,7 @@ #include #include #include +#include using namespace std; From 697bf8a29a889ee6702faefecf1a608f2d678183 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 09:37:07 +0200 Subject: [PATCH 11/17] Scope and not naming _Bytes --- stl/src/vector_algorithms.cpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 67b558ad87b..6466dcea32d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2080,14 +2080,13 @@ namespace { const void* __stdcall __std_find_first_of_trivial_impl( const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC - constexpr bool _Bytes = sizeof(_Ty) == 1; - constexpr int _Op = - (_Bytes ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; - constexpr int _Part_size_el = _Bytes ? 16 : 8; - const size_t _Needle_length = _Byte_length(_First2, _Last2); if (_Use_sse42() && _Needle_length <= 16) { + constexpr int _Op = + (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; + constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; + const int _Needle_length_el = static_cast(_Byte_length(_First2, _Last2) / sizeof(_Ty)); alignas(16) uint8_t _Tmp1[16]; From dae5a818df5e527d09d9d6e7d8b6827199946f8b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 10:26:53 +0200 Subject: [PATCH 12/17] ranges + coverage --- stl/inc/algorithm | 31 ++++++++++++++++--- .../VSO_0000000_vector_algorithms/test.cpp | 8 +++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 5cfc27c0d14..e6f2cf1415a 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -180,10 +180,12 @@ _Ty1* __std_find_first_of_trivial( } } +constexpr ptrdiff_t _Threshold_find_first_of = 16; // find_first_of vectorization is likely to be a win after this size + // Can we activate the vector algorithms for find_first_of? template , class _Ty2 = _Iter_value_t<_It2>> _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = - _Iterators_are_contiguous<_It1,_It2> // contiguous + _Iterators_are_contiguous<_It1, _It2> // contiguous && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile && disjunction_v< #ifdef __cpp_lib_byte @@ -3360,10 +3362,8 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( const auto _UFirst2 = _STD _Get_unwrapped(_First2); const auto _ULast2 = _STD _Get_unwrapped(_Last2); #if _USE_STD_VECTOR_ALGORITHMS - constexpr ptrdiff_t _Threshold = 16; // vectorization is likely to be a win after this size - if constexpr (_Vector_alg_in_find_first_of_is_safe<_FwdIt1, _FwdIt2, _Pr>) { - if (!_STD _Is_constant_evaluated() && _ULast1 - _UFirst1 >= _Threshold) { + if (!_STD _Is_constant_evaluated() && _ULast1 - _UFirst1 >= _Threshold_find_first_of) { const auto _First1_ptr = _STD _To_address(_UFirst1); const auto _Result = _STD __std_find_first_of_trivial( _First1_ptr, _STD _To_address(_ULast1), _STD _To_address(_UFirst2), _STD _To_address(_ULast2)); @@ -3456,6 +3456,29 @@ namespace ranges { _STL_INTERNAL_STATIC_ASSERT(sentinel_for<_Se2, _It2>); _STL_INTERNAL_STATIC_ASSERT(indirectly_comparable<_It1, _It2, _Pr, _Pj1, _Pj2>); +#if _USE_STD_VECTOR_ALGORITHMS + if constexpr (_Vector_alg_in_find_first_of_is_safe<_It1, _It2, _Pr> && sized_sentinel_for<_Se1, _It1> + && sized_sentinel_for<_Se2, _It2> && is_same_v<_Pj1, identity> && is_same_v<_Pj2, identity>) { + if (!_STD _Is_constant_evaluated() && _Last1 - _First1 >= _Threshold_find_first_of) { + const auto _Count1 = _Last1 - _First1; + const auto _First1_ptr = _STD _To_address(_First1); + const auto _Last1_ptr = _First1_ptr + _Count1; + + const auto _Count2 = _Last2 - _First2; + const auto _First2_ptr = _STD _To_address(_First2); + const auto _Last2_ptr = _First2_ptr + _Count2; + + const auto _Result = + _STD __std_find_first_of_trivial(_First1_ptr, _Last1_ptr, _First2_ptr, _Last2_ptr); + + if constexpr (is_pointer_v<_It1>) { + return _Result; + } else { + return _First1 + _Result - _First1_ptr; + } + } + } +#endif // _USE_STD_VECTOR_ALGORITHMS for (; _First1 != _Last1; ++_First1) { for (auto _Mid2 = _First2; _Mid2 != _Last2; ++_Mid2) { if (_STD invoke(_Pred, _STD invoke(_Proj1, *_First1), _STD invoke(_Proj2, *_Mid2))) { diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 52ea16ba5dc..af4f710cddd 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -229,6 +229,10 @@ void test_case_find_first_of(const vector& input_haystack, const vector& i input_haystack.begin(), input_haystack.end(), input_needle.begin(), input_needle.end()); auto actual = find_first_of(input_haystack.begin(), input_haystack.end(), input_needle.begin(), input_needle.end()); assert(expected == actual); +#if _HAS_CXX20 + auto ranges_actual = ranges::find_first_of(input_haystack, input_needle); + assert(expected == ranges_actual); +#endif // _HAS_CXX20 } template @@ -264,6 +268,10 @@ void test_find_first_of_containers() { C2 needle{'R', 'S', 'T'}; const auto result = find_first_of(haystack.begin(), haystack.end(), needle.begin(), needle.end()); assert(result == haystack.begin() + 6); +#if _HAS_CXX20 + const auto ranges_result = ranges::find_first_of(haystack, needle); + assert(ranges_result == haystack.begin() + 6); +#endif // _HAS_CXX20 } template From b7d69f6810d31028a3c784906b881f93522a8848 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 10:39:23 +0200 Subject: [PATCH 13/17] _Needle_length --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 6466dcea32d..c38395d1435 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2087,7 +2087,7 @@ namespace { (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - const int _Needle_length_el = static_cast(_Byte_length(_First2, _Last2) / sizeof(_Ty)); + const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); alignas(16) uint8_t _Tmp1[16]; memcpy(_Tmp1, _First2, _Needle_length); From f5b8f78d0f739d12bb0c4dab6f7c434bd77ce6c5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 10:40:06 +0200 Subject: [PATCH 14/17] fix benchmark --- benchmarks/src/find_first_of.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/src/find_first_of.cpp b/benchmarks/src/find_first_of.cpp index 2d95dfb7845..170793f1e58 100644 --- a/benchmarks/src/find_first_of.cpp +++ b/benchmarks/src/find_first_of.cpp @@ -18,7 +18,7 @@ void bm(benchmark::State& state) { static_assert(Pos < HSize); static_assert(Which < NSize); - h[Pos] = n[Which[)]; + h[Pos] = n[Which]; for (auto _ : state) { benchmark::DoNotOptimize(find_first_of(h.begin(), h.end(), n.begin(), n.end())); From 2c9979bc8002a1116f59fc7a94be4d68e4339b02 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 20 Mar 2024 11:04:18 +0200 Subject: [PATCH 15/17] _INLINE_VAR --- stl/inc/algorithm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index e6f2cf1415a..39f44fc3a95 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -180,7 +180,8 @@ _Ty1* __std_find_first_of_trivial( } } -constexpr ptrdiff_t _Threshold_find_first_of = 16; // find_first_of vectorization is likely to be a win after this size +// find_first_of vectorization is likely to be a win after this size (in elements) +_INLINE_VAR constexpr ptrdiff_t _Threshold_find_first_of = 16; // Can we activate the vector algorithms for find_first_of? template , class _Ty2 = _Iter_value_t<_It2>> From 1ab536cd0db1fc59804a30a55757a226333112ba Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 21 Mar 2024 08:51:26 +0200 Subject: [PATCH 16/17] must reuse! --- stl/inc/algorithm | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 39f44fc3a95..a3a0623eb71 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -184,23 +184,10 @@ _Ty1* __std_find_first_of_trivial( _INLINE_VAR constexpr ptrdiff_t _Threshold_find_first_of = 16; // Can we activate the vector algorithms for find_first_of? -template , class _Ty2 = _Iter_value_t<_It2>> +template _INLINE_VAR constexpr bool _Vector_alg_in_find_first_of_is_safe = - _Iterators_are_contiguous<_It1, _It2> // contiguous - && !_Iterator_is_volatile<_It1> && !_Iterator_is_volatile<_It2> // non-volatile - && disjunction_v< -#ifdef __cpp_lib_byte - conjunction, is_same<_Ty2, byte>>, // byte or... -#endif // defined(__cpp_lib_byte) - conjunction, is_integral<_Ty2>>> // ...integer - && sizeof(_Ty1) == sizeof(_Ty2) // same sizes - && is_signed_v<_Ty1> == is_signed_v<_Ty2> // same signedness - && sizeof(_Ty1) <= 2 // pcmpestri compatible size - && _Is_any_of_v<_Pr, // predicate is equal -#if _HAS_CXX20 - ranges::equal_to, -#endif // _HAS_CXX20 - equal_to<>, equal_to<_Ty1>>; + _Equal_memcmp_is_safe<_It1, _It2, _Pr> // can replace value comparison with bitwise comparison + && sizeof(_Iter_value_t<_It1>) <= 2; // pcmpestri compatible size _STD_END #endif // _USE_STD_VECTOR_ALGORITHMS From 478512415c7035f62278238fc6b2c40dc795fa98 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 21 Mar 2024 00:36:33 -0700 Subject: [PATCH 17/17] Code review feedback. --- stl/inc/algorithm | 6 +++--- stl/src/vector_algorithms.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index a3a0623eb71..b00dc4ee874 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -3350,7 +3350,7 @@ _NODISCARD _CONSTEXPR20 _FwdIt1 find_first_of( const auto _UFirst2 = _STD _Get_unwrapped(_First2); const auto _ULast2 = _STD _Get_unwrapped(_Last2); #if _USE_STD_VECTOR_ALGORITHMS - if constexpr (_Vector_alg_in_find_first_of_is_safe<_FwdIt1, _FwdIt2, _Pr>) { + if constexpr (_Vector_alg_in_find_first_of_is_safe) { if (!_STD _Is_constant_evaluated() && _ULast1 - _UFirst1 >= _Threshold_find_first_of) { const auto _First1_ptr = _STD _To_address(_UFirst1); const auto _Result = _STD __std_find_first_of_trivial( @@ -3447,7 +3447,7 @@ namespace ranges { #if _USE_STD_VECTOR_ALGORITHMS if constexpr (_Vector_alg_in_find_first_of_is_safe<_It1, _It2, _Pr> && sized_sentinel_for<_Se1, _It1> && sized_sentinel_for<_Se2, _It2> && is_same_v<_Pj1, identity> && is_same_v<_Pj2, identity>) { - if (!_STD _Is_constant_evaluated() && _Last1 - _First1 >= _Threshold_find_first_of) { + if (!_STD is_constant_evaluated() && _Last1 - _First1 >= _Threshold_find_first_of) { const auto _Count1 = _Last1 - _First1; const auto _First1_ptr = _STD _To_address(_First1); const auto _Last1_ptr = _First1_ptr + _Count1; @@ -3462,7 +3462,7 @@ namespace ranges { if constexpr (is_pointer_v<_It1>) { return _Result; } else { - return _First1 + _Result - _First1_ptr; + return _First1 + (_Result - _First1_ptr); } } } diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 761f8f8a199..176abaa94e4 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2043,7 +2043,7 @@ namespace { _Advance_bytes(_First1, 16); } - const size_t _Last_part_size = (_Haystack_length & 0xF); + const size_t _Last_part_size = _Haystack_length & 0xF; const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); alignas(16) uint8_t _Tmp2[16];