From b120c1148e27a938759586e3804ee9f682534d7f Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Wed, 4 Mar 2026 11:50:44 +0000 Subject: [PATCH 1/6] Add Neon implementations of `find_last_of` --- stl/inc/xutility | 2 +- stl/src/vector_algorithms.cpp | 224 ++++++++++++++++++++++++++++++++-- 2 files changed, 217 insertions(+), 9 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index 2926b495856..810b60c6195 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -81,7 +81,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_FIND_END _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_FIND_FIRST_OF _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_FIND_LAST _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC -#define _VECTORIZED_FIND_LAST_OF _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_FIND_LAST_OF _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_INCLUDES _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_IS_SORTED_UNTIL _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC #define _VECTORIZED_MINMAX _VECTORIZED_FOR_X64_X86_ARM64_ARM64EC diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cd323c958cc..b1690e02677 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -6214,6 +6214,64 @@ namespace { return static_cast(-1); } + + template + size_t _Impl_last_neon( + const void* const _Haystack, const size_t _Haystack_length, const uint8x16x2_t _Bitmap) noexcept { + const auto _Hay_ptr = static_cast(_Haystack); + + size_t _Ix = _Haystack_length; + while (_Ix >= 16) { + _Ix -= 16; + const auto _Eq = _Bitmap_details::_Do_bitmap(_Hay_ptr + _Ix, _Bitmap); + + auto _Mask = _Finding::_Find_traits_1::_Mask_q(_Eq); + if constexpr (_Pred == _Predicate::_None_of) { + _Mask ^= 0xFFFF'FFFF'FFFF'FFFF; + } + + if (_Mask != 0) { + return _Ix + _Finding::_Get_last_h_pos_q(_Mask); + } + } + + if (_Ix != 0) { + alignas(16) uint8_t _Bitmap_scalar_tail[32]; + vst1q_u8(_Bitmap_scalar_tail, _Bitmap.val[0]); + vst1q_u8(_Bitmap_scalar_tail + 16, _Bitmap.val[1]); + + do { + const _Ty _Val = _Hay_ptr[_Ix]; + + if constexpr (sizeof(_Val) > 1) { + if (_Val >= 256) { + if constexpr (_Pred == _Predicate::_Any_of) { + continue; + } else { + return _Ix; + } + } + } + + const auto _Val_u8 = static_cast(_Val); + const uint8_t _Hi = static_cast(_Val_u8 >> 3); + const uint8_t _Lo = static_cast(_Val_u8 & 0x7); + const bool _Hit = ((_Bitmap_scalar_tail[_Hi] >> _Lo) & 0x1) != 0; + + if constexpr (_Pred == _Predicate::_Any_of) { + if (_Hit) { + return _Ix; + } + } else { + if (!_Hit) { + return _Ix; + } + } + } while (--_Ix != 0); + } + + return static_cast(-1); + } #else // ^^^ defined(_M_ARM64) || defined(_M_ARM64EC) / !defined(_M_ARM64) && !defined(_M_ARM64EC) vvv template bool _Use_bitmap_avx(const size_t _Count1, const size_t _Count2) noexcept { @@ -7434,6 +7492,134 @@ namespace { return static_cast(-1); } + template + size_t _Shuffle_impl(const void* const _First1, const size_t _Count1, const void* const _First2, + const size_t _Count2) noexcept { + using _Traits = _First_of::_Find_first_of_traits<_Ty>; + constexpr size_t _Vec_size_elem = 16 / sizeof(_Ty); + + const void* const _Last1 = static_cast(_First1) + _Count1; + const void* const _Last2 = static_cast(_First2) + _Count2; + + const size_t _Haystack_length = _Byte_length(_First1, _Last1); + const size_t _Vec_bytes = _Haystack_length & ~size_t{0xF}; + const void* _Stop1 = _Last1; + _Rewind_bytes(_Stop1, _Vec_bytes); + + const void* _Cur1 = _Last1; + if (_Cur1 != _Stop1) { + const size_t _Needle_length = _Byte_length(_First2, _Last2); + const void* _Stop2 = _First2; + _Advance_bytes(_Stop2, _Needle_length & ~size_t{0xF}); + + // Set _Lanes to 1 when we have 0 tail elements, + // to stop the compiler from complaining about an empty array. + constexpr size_t _Lanes = _Unrolled_tail_elems == 0 ? 1 : _Unrolled_tail_elems; + _First_of::_Find_first_of_vec_t<_Ty> _Needle_tail[_Lanes]; + + if constexpr (_Unrolled_tail_elems > 0) { + for (size_t _Lane = 0; _Lane < _Unrolled_tail_elems; ++_Lane) { + _Needle_tail[_Lane] = _Traits::_Set_neon_q(*(static_cast(_Stop2) + _Lane)); + } + } + + do { + _Rewind_bytes(_Cur1, 16); + const auto _Data1 = _Traits::_Load_q(_Cur1); + + auto _Eq = _Traits::_Set_neon_q(0); + + const void* _Ptr2 = _First2; + for (; _Ptr2 != _Stop2; _Advance_bytes(_Ptr2, 16)) { + const auto _Data2s0 = _Traits::_Load_q(_Ptr2); + _Eq = _Traits::_Or( + _Eq, _First_of::_Shuffle_step<_Traits, _Vec_size_elem, _Ty>(_Data1, _Data2s0)); + } + + if constexpr (_Unrolled_tail_elems > 0) { + // Unrolled tail. + for (size_t _Lane = 0; _Lane < _Unrolled_tail_elems; ++_Lane) { + const auto _Cmp = _Traits::_Cmp_neon_q(_Data1, _Needle_tail[_Lane]); + _Eq = _Traits::_Or(_Eq, _Cmp); + } + } + + // We unroll by a maximum of 4 (for 2-byte and 1-byte element types), so we need a non-unrolled + // tail loop. + if constexpr (_Unrolled_tail_elems == 4) { + _Advance_bytes(_Ptr2, sizeof(_Ty) * _Unrolled_tail_elems); + + for (; _Ptr2 != _Last2; _Advance_bytes(_Ptr2, sizeof(_Ty))) { + const auto _Needle = _Traits::_Set_neon_q(*(static_cast(_Ptr2))); + const auto _Cmp = _Traits::_Cmp_neon_q(_Data1, _Needle); + _Eq = _Traits::_Or(_Eq, _Cmp); + } + } + + uint64_t _Match = _Traits::_Mask_q(_Eq); + if constexpr (_Pred == _Predicate::_None_of) { + _Match ^= 0xFFFF'FFFF'FFFF'FFFF; + } + + if (_Match != 0) { + const auto _Pos = _Finding::_Get_last_h_pos_q(_Match); + _Advance_bytes(_Cur1, _Pos - (sizeof(_Ty) - 1)); + return _Byte_length(_First1, _Cur1) / sizeof(_Ty); + } + } while (_Cur1 != _Stop1); + } + + const auto _Rem1 = _Byte_length(_First1, _Cur1) / sizeof(_Ty); + + return _Fallback<_Ty, _Pred>(_First1, _Rem1, _First2, _Count2); + } + + template + size_t __stdcall _Shuffle_impl_dispatch(const void* const _First1, const size_t _Count1, + const void* const _First2, const size_t _Count2) noexcept { + + constexpr size_t _Vec_size_elems = 16 / sizeof(_Ty); + const size_t _Last_tail_length_el = _Count2 & (_Vec_size_elems - 1); + + if (_Last_tail_length_el == 0) { + return _Shuffle_impl<_Ty, _Pred, 0>(_First1, _Count1, _First2, _Count2); + } else if (_Last_tail_length_el == 1) { + return _Shuffle_impl<_Ty, _Pred, 1>(_First1, _Count1, _First2, _Count2); + } else if (_Last_tail_length_el == 2) { + if constexpr (sizeof(_Ty) <= 4) { + return _Shuffle_impl<_Ty, _Pred, 2>(_First1, _Count1, _First2, _Count2); + } + } else if (_Last_tail_length_el == 3) { + if constexpr (sizeof(_Ty) <= 4) { + return _Shuffle_impl<_Ty, _Pred, 3>(_First1, _Count1, _First2, _Count2); + } + } else if (_Last_tail_length_el >= 4) { + if constexpr (sizeof(_Ty) <= 2) { + return _Shuffle_impl<_Ty, _Pred, 4>(_First1, _Count1, _First2, _Count2); + } + } + + _STL_UNREACHABLE; + } + + size_t _Fallback_find_not_2(const void* const _First1, const size_t _Count1, const void* const _First2, + const size_t _Count2) noexcept { + auto _Haystack_start = static_cast(_First1); + const auto _Last2 = static_cast(_First2) + _Count2; + + for (auto _Ptr = _Haystack_start + _Count1 - 1;; --_Ptr) { + if (_Finding::_Find_impl<_Finding::_Find_traits_2, _Finding::_Predicate::_Equal>( + _First2, _Last2, *_Ptr) + == _Last2) { + return static_cast(_Ptr - _Haystack_start); + } + + if (_Ptr == _Haystack_start) { + return static_cast(-1); + } + } + } + #if !defined(_M_ARM64) && !defined(_M_ARM64EC) template size_t _Impl(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, @@ -7608,7 +7794,34 @@ namespace { const size_t _Count2) noexcept { using namespace _Bitmap_impl; -#if !defined(_M_ARM64) && !defined(_M_ARM64EC) +#if defined(_M_ARM64) || defined(_M_ARM64EC) + if (_Use_bitmap_neon<_Ty>(_Count1, _Count2)) { + if constexpr (sizeof(_Ty) < 8) { + uint8x16x2_t _Bitmap; + if (_Bitmap_details::_Make_bitmap<_Ty>(_First2, _Count2, _Bitmap)) { + return _Impl_last_neon<_Ty, _Pred>(_First1, _Count1, _Bitmap); + } + } + } else if (_Use_bitmap_scalar<_Ty>(_Count1, _Count2)) { + alignas(32) _Scalar_table_t _Table = {}; + if (_Build_scalar_table<_Ty>(_First2, _Count2, _Table)) { + return _Impl_last_scalar<_Ty, _Pred>(_First1, _Count1, _Table); + } + } + + // For _None_of, it can be faster to Find Haystack elements in Needle for longer Needles or + // non-vectorized Haystack lengths. + if constexpr (_Pred == _Predicate::_None_of && sizeof(_Ty) >= 2) { + static_assert(sizeof(_Ty) == 2); + + // Heuristic of Haystack smaller than Neon width, or Needle at least twice the Neon width. + if (_Count1 * sizeof(_Ty) < 16 || _Count2 * sizeof(_Ty) >= 32) { + return _Fallback_find_not_2(_First1, _Count1, _First2, _Count2); + } + } + + return _Shuffle_impl_dispatch<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); +#else // ^^^ defined(_M_ARM64) || defined(_M_ARM64EC) / !defined(_M_ARM64) && !defined(_M_ARM64EC) vvv if (_Use_sse42()) { const auto _Strat = _Pick_strategy<_Ty>(_Count1, _Count2, _Use_avx2()); @@ -7625,9 +7838,7 @@ namespace { } return _Impl<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); - } else -#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ - { + } else { alignas(32) _Scalar_table_t _Table = {}; if (_Build_scalar_table<_Ty>(_First2, _Count2, _Table)) { return _Impl_last_scalar<_Ty, _Pred>(_First1, _Count1, _Table); @@ -7635,6 +7846,7 @@ namespace { return _Fallback<_Ty, _Pred>(_First1, _Count1, _First2, _Count2); } +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ } } // namespace _Last_of } // namespace _Find_meow_of @@ -7686,7 +7898,6 @@ __declspec(noalias) size_t __stdcall __std_find_first_of_trivial_pos_8(const voi _Haystack, _Haystack_length, _Needle, _Needle_length); } -#ifndef _M_ARM64 __declspec(noalias) size_t __stdcall __std_find_last_of_trivial_pos_1(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { return _Find_meow_of::_Last_of::_Dispatch_pos( @@ -7698,7 +7909,6 @@ __declspec(noalias) size_t __stdcall __std_find_last_of_trivial_pos_2(const void return _Find_meow_of::_Last_of::_Dispatch_pos( _Haystack, _Haystack_length, _Needle, _Needle_length); } -#endif // ^^^ !defined(_M_ARM64) ^^^ __declspec(noalias) size_t __stdcall __std_find_first_not_of_trivial_pos_1(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { @@ -7712,7 +7922,6 @@ __declspec(noalias) size_t __stdcall __std_find_first_not_of_trivial_pos_2(const _Haystack, _Haystack_length, _Needle, _Needle_length); } -#ifndef _M_ARM64 __declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_1(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { return _Find_meow_of::_Last_of::_Dispatch_pos( @@ -7724,7 +7933,6 @@ __declspec(noalias) size_t __stdcall __std_find_last_not_of_trivial_pos_2(const return _Find_meow_of::_Last_of::_Dispatch_pos( _Haystack, _Haystack_length, _Needle, _Needle_length); } -#endif // ^^^ !defined(_M_ARM64) ^^^ } // extern "C" From 8f86fd2ac9c0525e6d1cf4bc87e19715bb348f17 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 5 Mar 2026 00:13:06 +0000 Subject: [PATCH 2/6] Guard Neon-specific functions --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b1690e02677..cea9ae5c8cc 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -7492,6 +7492,7 @@ namespace { return static_cast(-1); } +#if defined(_M_ARM64) || defined(_M_ARM64EC) template size_t _Shuffle_impl(const void* const _First1, const size_t _Count1, const void* const _First2, const size_t _Count2) noexcept { @@ -7619,8 +7620,7 @@ namespace { } } } - -#if !defined(_M_ARM64) && !defined(_M_ARM64EC) +#else // ^^^ defined(_M_ARM64) || defined(_M_ARM64EC) / !defined(_M_ARM64) && !defined(_M_ARM64EC) vvv template size_t _Impl(const void* const _Haystack, const size_t _Haystack_length, const void* const _Needle, const size_t _Needle_length) noexcept { From 3639be4a338039853ef7660da25731f7ffa71385 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 5 Mar 2026 00:35:55 +0000 Subject: [PATCH 3/6] Apply slightly confusing clang-format --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cea9ae5c8cc..3c7850b090d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -7690,7 +7690,7 @@ namespace { const int _Last_needle_length_el = _Last_needle_length / sizeof(_Ty); // equal to npos when treated as size_t; also less than any found value - constexpr int _Not_found = -1; + constexpr int _Not_found = -1; #pragma warning(push) #pragma warning(disable : 4324) // structure was padded due to alignment specifier From 4255ee42a0cf34261f12fe530b9c4df1031e0906 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 5 Mar 2026 10:25:27 +0000 Subject: [PATCH 4/6] Fix index decrement in `last_of` Neon bitmap tail --- stl/src/vector_algorithms.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 3c7850b090d..dd2fc9ce3b2 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -6241,6 +6241,7 @@ namespace { vst1q_u8(_Bitmap_scalar_tail + 16, _Bitmap.val[1]); do { + --_Ix; const _Ty _Val = _Hay_ptr[_Ix]; if constexpr (sizeof(_Val) > 1) { @@ -6267,7 +6268,7 @@ namespace { return _Ix; } } - } while (--_Ix != 0); + } while (_Ix != 0); } return static_cast(-1); From 79dc8d0cf180c2bb3fa87ea1a59fee868734bc71 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 5 Mar 2026 10:28:01 +0000 Subject: [PATCH 5/6] Extract `first_of` index increment for symmetry --- stl/src/vector_algorithms.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index dd2fc9ce3b2..cfba86f9704 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -6209,7 +6209,9 @@ namespace { return _Ix; } } - } while (++_Ix != _Haystack_length); + + ++_Ix; + } while (_Ix != _Haystack_length); } return static_cast(-1); From bd147f361bf904362852975ac9e2a4ace3bc84d9 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 5 Mar 2026 15:56:06 -0800 Subject: [PATCH 6/6] Add const. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cfba86f9704..a1542d56a64 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -7608,8 +7608,8 @@ namespace { size_t _Fallback_find_not_2(const void* const _First1, const size_t _Count1, const void* const _First2, const size_t _Count2) noexcept { - auto _Haystack_start = static_cast(_First1); - const auto _Last2 = static_cast(_First2) + _Count2; + const auto _Haystack_start = static_cast(_First1); + const auto _Last2 = static_cast(_First2) + _Count2; for (auto _Ptr = _Haystack_start + _Count1 - 1;; --_Ptr) { if (_Finding::_Find_impl<_Finding::_Find_traits_2, _Finding::_Predicate::_Equal>(