From ee910355df196630cb2d3903f55267962d53cec1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 22:18:03 +0300 Subject: [PATCH 1/4] vectorize find_first_of for long needle --- stl/src/vector_algorithms.cpp | 39 ++++++++++++++++--- .../VSO_0000000_vector_algorithms/test.cpp | 2 +- 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 1c5fc74e809..95add25e31a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2021,18 +2021,21 @@ namespace { const void* __stdcall __std_find_first_of_trivial_impl( const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC - const size_t _Needle_length = _Byte_length(_First2, _Last2); - - if (_Use_sse42() && _Needle_length <= 16) { + if (_Use_sse42()) { constexpr int _Op = (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); + const size_t _Needle_length = _Byte_length(_First2, _Last2); + const void* _Last_needle = _First2; + _Advance_bytes(_Last_needle, _Needle_length & ~size_t{0xF}); + + const int _Last_needle_length = static_cast(_Needle_length & 0xF); alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _First2, _Needle_length); - const __m128i _Needle = _mm_load_si128(reinterpret_cast(_Tmp1)); + memcpy(_Tmp1, _First2, _Last_needle_length); + const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); + const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); const size_t _Haystack_length = _Byte_length(_First1, _Last1); const void* _Stop_at = _First1; @@ -2041,6 +2044,18 @@ namespace { while (_First1 != _Stop_at) { const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); + for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { + const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + + if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + return _First1; + } + } + + const int _Needle_length_el = _Last_needle_length_el; + const __m128i _Needle = _Last_needle_val; if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); _Advance_bytes(_First1, _Pos * sizeof(_Ty)); @@ -2057,6 +2072,18 @@ namespace { memcpy(_Tmp2, _First1, _Last_part_size); const __m128i _Haystack_last_part = _mm_load_si128(reinterpret_cast(_Tmp2)); + for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { + const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + + if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_last_part, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_last_part, _Last_part_size_el, _Op); + _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + return _First1; + } + } + + const int _Needle_length_el = _Last_needle_length_el; + const __m128i _Needle = _Last_needle_val; if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op)) { const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op); _Advance_bytes(_First1, _Pos * sizeof(_Ty)); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index e263e59d628..88d9b9d914f 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -238,7 +238,7 @@ void test_case_find_first_of(const vector& input_haystack, const vector& i template void test_find_first_of(mt19937_64& gen) { - constexpr size_t needleDataCount = 30; + constexpr size_t needleDataCount = 50; using TD = conditional_t; uniform_int_distribution dis('a', 'z'); vector input_haystack; From d39748a7f675c291c5669f6f69c889bf3b18fac6 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 2 Apr 2024 22:28:15 +0300 Subject: [PATCH 2/4] format --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 95add25e31a..b64f3a14685 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2034,7 +2034,7 @@ namespace { alignas(16) uint8_t _Tmp1[16]; memcpy(_Tmp1, _First2, _Last_needle_length); - const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); + const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); const size_t _Haystack_length = _Byte_length(_First1, _Last1); From a30f11a57127a2b46de779ab382d38c8e3d3bcea Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 3 Apr 2024 10:19:42 +0300 Subject: [PATCH 3/4] best of parts --- stl/src/vector_algorithms.cpp | 151 ++++++++++++++++++++++++---------- 1 file changed, 106 insertions(+), 45 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b64f3a14685..ca5f729fc7b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2017,7 +2017,7 @@ namespace { return _Result; } - template + template const void* __stdcall __std_find_first_of_trivial_impl( const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC @@ -2025,73 +2025,134 @@ namespace { constexpr int _Op = (sizeof(_Ty) == 1 ? _SIDD_UBYTE_OPS : _SIDD_UWORD_OPS) | _SIDD_CMP_EQUAL_ANY | _SIDD_LEAST_SIGNIFICANT; constexpr int _Part_size_el = sizeof(_Ty) == 1 ? 16 : 8; - const size_t _Needle_length = _Byte_length(_First2, _Last2); - const void* _Last_needle = _First2; - _Advance_bytes(_Last_needle, _Needle_length & ~size_t{0xF}); - - const int _Last_needle_length = static_cast(_Needle_length & 0xF); - alignas(16) uint8_t _Tmp1[16]; - memcpy(_Tmp1, _First2, _Last_needle_length); - const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); - const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); + if (_Needle_length <= 16) { + // Special handling of small needle + // The generic branch could also handle it but with slightly worse performance - const size_t _Haystack_length = _Byte_length(_First1, _Last1); - const void* _Stop_at = _First1; - _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + const int _Needle_length_el = static_cast(_Needle_length / sizeof(_Ty)); - while (_First1 != _Stop_at) { - const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _First2, _Needle_length); + const __m128i _Needle = _mm_load_si128(reinterpret_cast(_Tmp1)); - for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { - const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); - if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op); + while (_First1 != _Stop_at) { + const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); + if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); _Advance_bytes(_First1, _Pos * sizeof(_Ty)); return _First1; } + + _Advance_bytes(_First1, 16); } - const int _Needle_length_el = _Last_needle_length_el; - const __m128i _Needle = _Last_needle_val; - if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); + const size_t _Last_part_size = _Haystack_length & 0xF; + const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); + + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First1, _Last_part_size); + const __m128i _Haystack_part = _mm_load_si128(reinterpret_cast(_Tmp2)); + + if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op); _Advance_bytes(_First1, _Pos * sizeof(_Ty)); return _First1; } - _Advance_bytes(_First1, 16); - } + _Advance_bytes(_First1, _Last_part_size); + return _First1; + } else { + const void* _Last_needle = _First2; + _Advance_bytes(_Last_needle, _Needle_length & ~size_t{0xF}); - const size_t _Last_part_size = _Haystack_length & 0xF; - const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); + const int _Last_needle_length = static_cast(_Needle_length & 0xF); - alignas(16) uint8_t _Tmp2[16]; - memcpy(_Tmp2, _First1, _Last_part_size); - const __m128i _Haystack_last_part = _mm_load_si128(reinterpret_cast(_Tmp2)); + alignas(16) uint8_t _Tmp1[16]; + memcpy(_Tmp1, _Last_needle, _Last_needle_length); + const __m128i _Last_needle_val = _mm_load_si128(reinterpret_cast(_Tmp1)); + const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); - for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { - const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + constexpr int _Not_found = 16; // arbitrary value greater than any found value - if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_last_part, _Last_part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_last_part, _Last_part_size_el, _Op); - _Advance_bytes(_First1, _Pos * sizeof(_Ty)); - return _First1; + int _Found_pos = _Not_found; + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Haystack_length & ~size_t{0xF}); + + while (_First1 != _Stop_at) { + const __m128i _Haystack_part = _mm_loadu_si128(static_cast(_First1)); + + for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; + _Advance_bytes(_Cur_needle, 16)) { + const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_part, _Part_size_el, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + } + + if (const int _Needle_length_el = _Last_needle_length_el; _Needle_length_el != 0) { + const __m128i _Needle = _Last_needle_val; + if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op)) { + const int _Pos = + _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Part_size_el, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + } + + if (_Found_pos != _Not_found) { + _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); + return _First1; + } + + _Advance_bytes(_First1, 16); } - } - const int _Needle_length_el = _Last_needle_length_el; - const __m128i _Needle = _Last_needle_val; - if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op)) { - const int _Pos = _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_last_part, _Last_part_size_el, _Op); - _Advance_bytes(_First1, _Pos * sizeof(_Ty)); + const size_t _Last_part_size = _Haystack_length & 0xF; + const int _Last_part_size_el = static_cast(_Last_part_size / sizeof(_Ty)); + + alignas(16) uint8_t _Tmp2[16]; + memcpy(_Tmp2, _First1, _Last_part_size); + const __m128i _Haystack_part = _mm_load_si128(reinterpret_cast(_Tmp2)); + + _Found_pos = _Last_part_size_el; + + for (const void* _Cur_needle = _First2; _Cur_needle != _Last_needle; _Advance_bytes(_Cur_needle, 16)) { + const __m128i _Needle = _mm_loadu_si128(static_cast(_Cur_needle)); + + if (_mm_cmpestrc(_Needle, _Part_size_el, _Haystack_part, _Last_part_size_el, _Op)) { + const int _Pos = _mm_cmpestri(_Needle, _Part_size_el, _Haystack_part, _Last_part_size_el, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + } + + if (const int _Needle_length_el = _Last_needle_length_el; _Needle_length_el != 0) { + const __m128i _Needle = _Last_needle_val; + if (_mm_cmpestrc(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op)) { + const int _Pos = + _mm_cmpestri(_Needle, _Needle_length_el, _Haystack_part, _Last_part_size_el, _Op); + if (_Pos < _Found_pos) { + _Found_pos = _Pos; + } + } + } + + _Advance_bytes(_First1, _Found_pos * sizeof(_Ty)); return _First1; } - - _Advance_bytes(_First1, _Last_part_size); - return _First1; } #endif // !_M_ARM64EC From dc3a1cfb25dc3f9337a4e44d47b937a43f56f54c Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 3 Apr 2024 10:24:33 +0300 Subject: [PATCH 4/4] format --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index ca5f729fc7b..512a916a35b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2017,7 +2017,7 @@ namespace { return _Result; } - template + template const void* __stdcall __std_find_first_of_trivial_impl( const void* _First1, const void* const _Last1, const void* const _First2, const void* const _Last2) noexcept { #ifndef _M_ARM64EC