From 8027e53e296a6b8feee9c84bdbf1dad3212d6038 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Tue, 16 Dec 2025 14:09:45 +0000 Subject: [PATCH 1/7] Add Neon implementation of `find` for ARM64 targets --- stl/inc/xutility | 2 +- stl/src/vector_algorithms.cpp | 331 ++++++++++++++++++++++++++++++++-- 2 files changed, 318 insertions(+), 15 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index 06c1dfa8dcc..2e32f0f0a24 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -82,7 +82,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_BITSET_FROM_STRING _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_BITSET_TO_STRING _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_COUNT _VECTORIZED_FOR_X64_X86 -#define _VECTORIZED_FIND _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_FIND _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_FIND_END _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_FIND_FIRST_OF _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_FIND_LAST _VECTORIZED_FOR_X64_X86 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0f84855383d..f7337994541 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3661,10 +3661,212 @@ const void* __stdcall __std_is_sorted_until_d( } // extern "C" -#ifndef _M_ARM64 namespace { namespace _Finding { -#ifdef _M_ARM64EC +#ifdef _M_ARM64 + struct _Find_traits_1 { + static uint8x16_t _Load_q(const void* _Ptr) noexcept { + return vld1q_u8(static_cast(_Ptr)); + } + + static uint8x8_t _Load(const void* _Ptr) noexcept { + return vld1_u8(static_cast(_Ptr)); + } + + static uint8x16_t _Set_neon_q(const uint8_t _Val) noexcept { + return vdupq_n_u8(_Val); + } + + static uint8x8_t _Set_neon(const uint8_t _Val) noexcept { + return vdup_n_u8(_Val); + } + + static uint8x16_t _Cmp_neon_q(const uint8x16_t _Lhs, const uint8x16_t _Rhs) noexcept { + return vceqq_u8(_Lhs, _Rhs); + } + + static uint8x8_t _Cmp_neon(const uint8x8_t _Lhs, const uint8x8_t _Rhs) noexcept { + return vceq_u8(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 16 8-bit values into a 64-bit Mask of 16 4-bit values. + static uint64_t _Mask_q(const uint8x16_t _Val) noexcept { + const uint8x8_t _Res = vshrn_n_u16(vreinterpretq_u16_u8(_Val), 4); + return vget_lane_u64(vreinterpret_u64_u8(_Res), 0); + } + + static uint64_t _Mask(const uint8x8_t _Val) noexcept { + return vget_lane_u64(vreinterpret_u64_u8(_Val), 0); + } + + static uint64_t _Match_mask_q(const uint8x16_t _Val) noexcept { + const uint8x16_t _Res = vpmaxq_u8(_Val, _Val); + return vgetq_lane_u64(vreinterpretq_u64_u8(_Res), 0); + } + + static uint64_t _Match_mask_q(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { + auto _Cmp = vreinterpretq_u64_u8(vorrq_u8(_Cmp_lo, _Cmp_hi)); + return vgetq_lane_u64(vpaddq_u64(_Cmp, _Cmp), 0); + } + + static uint8x16_t _Not_q(const uint8x16_t _Val) { + return vmvnq_u8(_Val); + } + + static uint8x8_t _Not(const uint8x8_t _Val) { + return vmvn_u8(_Val); + } + }; + + struct _Find_traits_2 { + static uint16x8_t _Load_q(const void* _Ptr) noexcept { + return vld1q_u16(static_cast(_Ptr)); + } + + static uint16x4_t _Load(const void* _Ptr) noexcept { + return vld1_u16(static_cast(_Ptr)); + } + + static uint16x8_t _Set_neon_q(const uint16_t _Val) noexcept { + return vdupq_n_u16(_Val); + } + + static uint16x4_t _Set_neon(const uint16_t _Val) noexcept { + return vdup_n_u16(_Val); + } + + static uint16x8_t _Cmp_neon_q(const uint16x8_t _Lhs, const uint16x8_t _Rhs) noexcept { + return vceqq_u16(_Lhs, _Rhs); + } + + static uint16x4_t _Cmp_neon(const uint16x4_t _Lhs, const uint16x4_t _Rhs) noexcept { + return vceq_u16(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 8 16-bit values into a 64-bit Mask of 8 8-bit values. + static uint64_t _Mask_q(const uint16x8_t _Val) noexcept { + const uint16x4_t _Res = vshrn_n_u32(vreinterpretq_u32_u16(_Val), 8); + return vget_lane_u64(vreinterpret_u64_u16(_Res), 0); + } + + static uint64_t _Mask(const uint16x4_t _Val) noexcept { + return vget_lane_u64(vreinterpret_u64_u16(_Val), 0); + } + + static uint64_t _Match_mask_q(const uint16x8_t _Val) noexcept { + const uint16x8_t _Res = vpmaxq_u16(_Val, _Val); + return vgetq_lane_u64(vreinterpretq_u64_u16(_Res), 0); + } + + static uint64_t _Match_mask_q(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { + uint8x8_t _Cmp = vaddhn_u16(_Cmp_lo, _Cmp_hi); + return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); + } + + static uint16x8_t _Not_q(const uint16x8_t _Val) { + return vmvnq_u16(_Val); + } + + static uint16x4_t _Not(const uint16x4_t _Val) { + return vmvn_u16(_Val); + } + }; + + struct _Find_traits_4 { + static uint32x4_t _Load_q(const void* _Ptr) noexcept { + return vld1q_u32(static_cast(_Ptr)); + } + + static uint32x2_t _Load(const void* _Ptr) noexcept { + return vld1_u32(static_cast(_Ptr)); + } + + static uint32x4_t _Set_neon_q(const uint32_t _Val) noexcept { + return vdupq_n_u32(_Val); + } + + static uint32x2_t _Set_neon(const uint32_t _Val) noexcept { + return vdup_n_u32(_Val); + } + + static uint32x4_t _Cmp_neon_q(const uint32x4_t _Lhs, const uint32x4_t _Rhs) noexcept { + return vceqq_u32(_Lhs, _Rhs); + } + + static uint32x2_t _Cmp_neon(const uint32x2_t _Lhs, const uint32x2_t _Rhs) noexcept { + return vceq_u32(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 4 32-bit values into a 64-bit Mask of 4 16-bit values. + static uint64_t _Mask_q(const uint32x4_t _Val) noexcept { + const uint32x2_t _Res = vshrn_n_u64(vreinterpretq_u64_u32(_Val), 16); + return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); + } + + static uint64_t _Mask(const uint32x2_t _Val) noexcept { + return vget_lane_u64(vreinterpret_u64_u32(_Val), 0); + } + + static uint64_t _Match_mask_q(const uint32x4_t _Val) noexcept { + const uint32x4_t _Res = vpmaxq_u32(_Val, _Val); + return vgetq_lane_u64(vreinterpretq_u64_u32(_Res), 0); + } + + static uint64_t _Match_mask_q(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { + uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u32(_Cmp_lo), vreinterpretq_u16_u32(_Cmp_hi)); + return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); + } + + static uint32x4_t _Not_q(const uint32x4_t _Val) { + return vmvnq_u32(_Val); + } + + static uint32x2_t _Not(const uint32x2_t _Val) { + return vmvn_u32(_Val); + } + }; + + struct _Find_traits_8 { + static uint64x2_t _Load_q(const void* _Ptr) noexcept { + return vld1q_u64(static_cast(_Ptr)); + } + + static uint64x2_t _Set_neon_q(const uint64_t _Val) noexcept { + return vdupq_n_u64(_Val); + } + + static uint64x2_t _Cmp_neon_q(const uint64x2_t _Lhs, const uint64x2_t _Rhs) noexcept { + return vceqq_u64(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 2 64-bit values into a 64-bit Mask of 2 32-bit values. + static uint64_t _Mask_q(const uint64x2_t _Val) noexcept { + const uint32x2_t _Res = vmovn_u64(_Val); + return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); + } + + static uint64_t _Match_mask_q(const uint64x2_t _Val) noexcept { + return _Mask_q(_Val); + } + + static uint64_t _Match_mask_q(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { + uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u64(_Cmp_lo), vreinterpretq_u16_u64(_Cmp_hi)); + return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); + } + + static uint64x2_t _Not_q(const uint64x2_t _Val) { + return vreinterpretq_u64_u8(vmvnq_u8(vreinterpretq_u8_u64(_Val))); + } + }; + + static unsigned long _Get_first_h_pos_q(uint64_t _Mask) { + return _CountTrailingZeros64(_Mask) >> 2; + } + + static unsigned long _Get_first_h_pos_d(uint64_t _Mask) { + return _CountTrailingZeros64(_Mask) >> 3; + } +#elif defined(_M_ARM64EC) using _Find_traits_1 = void; using _Find_traits_2 = void; using _Find_traits_4 = void; @@ -3743,6 +3945,7 @@ namespace { }; #endif // ^^^ !defined(_M_ARM64EC) ^^^ +#ifndef _M_ARM64 // TRANSITION, ABI: used only in functions preserved for binary compatibility template const void* __stdcall _Find_unsized_impl(const void* const _First, const _Ty _Val) noexcept { @@ -3752,13 +3955,114 @@ namespace { } return _Ptr; } +#endif // ^^^ !defined(_M_ARM64) ^^^ enum class _Predicate { _Equal, _Not_equal }; + template <_Predicate _Pred, class _Ty> + static const void* _Find_scalar_tail(const void* _First, const void* const _Last, const _Ty _Val) noexcept { + auto _Ptr = static_cast(_First); + if constexpr (_Pred == _Predicate::_Not_equal) { + while (_Ptr != _Last && *_Ptr == _Val) { + ++_Ptr; + } + } else { + while (_Ptr != _Last && *_Ptr != _Val) { + ++_Ptr; + } + } + return _Ptr; + } + // The below functions have exactly the same signature as the extern "C" functions, up to calling convention. // This makes sure the template specialization can be fused with the extern "C" function. // In optimized builds it avoids an extra call, as these functions are too large to inline. +#ifdef _M_ARM64 + template + const void* __stdcall _Find_impl(const void* _First, const void* const _Last, const _Ty _Val) noexcept { + const size_t _Size_bytes = _Byte_length(_First, _Last); + + if (const size_t _Neon_size = _Size_bytes & ~size_t{0x1F}; _Neon_size != 0) { + const auto _Comparand = _Traits::_Set_neon_q(_Val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Neon_size); + + do { + const auto _Data_lo = _Traits::_Load_q(static_cast(_First) + 0); + const auto _Data_hi = _Traits::_Load_q(static_cast(_First) + 16); + + auto _Comparison_lo = _Traits::_Cmp_neon_q(_Data_lo, _Comparand); + auto _Comparison_hi = _Traits::_Cmp_neon_q(_Data_hi, _Comparand); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Comparison_lo = _Traits::_Not_q(_Comparison_lo); + _Comparison_hi = _Traits::_Not_q(_Comparison_hi); + } + + // Use a fast check for the termination condition. + uint64_t _Any_match = _Traits::_Match_mask_q(_Comparison_lo, _Comparison_hi); + + if (_Any_match != 0) { + auto _Mask_lo = _Traits::_Mask_q(_Comparison_lo); + if (_Mask_lo != 0) { + const auto _Offset = _Get_first_h_pos_q(_Mask_lo); + _Advance_bytes(_First, _Offset); + return _First; + } + + auto _Mask_hi = _Traits::_Mask_q(_Comparison_hi); + const auto _Offset = _Get_first_h_pos_q(_Mask_hi) + 16; + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); + } + + if ((_Size_bytes & size_t{0x10}) != 0) { + const auto _Comparand = _Traits::_Set_neon_q(_Val); + const auto _Data = _Traits::_Load_q(_First); + + auto _Comparison = _Traits::_Cmp_neon_q(_Data, _Comparand); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Comparison = _Traits::_Not_q(_Comparison); + } + + auto _Match = _Traits::_Mask_q(_Comparison); + if (_Match != 0) { + const auto _Offset = _Get_first_h_pos_q(_Match); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 16); + } + + if constexpr (sizeof(_Ty) < 8) { + if ((_Size_bytes & size_t{0x08}) != 0) { + const auto _Comparand = _Traits::_Set_neon(_Val); + const auto _Data = _Traits::_Load(_First); + + auto _Comparison = _Traits::_Cmp_neon(_Data, _Comparand); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Comparison = _Traits::_Not(_Comparison); + } + + auto _Match = _Traits::_Mask(_Comparison); + if (_Match != 0) { + const auto _Offset = _Get_first_h_pos_d(_Match); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 8); + } + } + + return _Find_scalar_tail<_Pred, _Ty>(_First, _Last, _Val); + } +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv template const void* __stdcall _Find_impl(const void* _First, const void* const _Last, const _Ty _Val) noexcept { #ifndef _M_ARM64EC @@ -3835,17 +4139,8 @@ namespace { } while (_First != _Stop_at); } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - auto _Ptr = static_cast(_First); - if constexpr (_Pred == _Predicate::_Not_equal) { - while (_Ptr != _Last && *_Ptr == _Val) { - ++_Ptr; - } - } else { - while (_Ptr != _Last && *_Ptr != _Val) { - ++_Ptr; - } - } - return _Ptr; + + return _Find_scalar_tail<_Pred, _Ty>(_First, _Last, _Val); } template @@ -4244,11 +4539,13 @@ namespace { } } } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // namespace _Finding } // unnamed namespace extern "C" { +#ifndef _M_ARM64 // TRANSITION, ABI: preserved for binary compatibility const void* __stdcall __std_find_trivial_unsized_1(const void* const _First, const uint8_t _Val) noexcept { // C23 7.27.5.2 "The memchr generic function"/2 says "The implementation shall behave as if @@ -4274,10 +4571,11 @@ const void* __stdcall __std_find_trivial_unsized_4(const void* const _First, con const void* __stdcall __std_find_trivial_unsized_8(const void* const _First, const uint64_t _Val) noexcept { return _Finding::_Find_unsized_impl(_First, _Val); } +#endif // ^^^ !defined(_M_ARM64) ^^^ const void* __stdcall __std_find_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { -#ifdef _M_ARM64EC +#if defined(_M_ARM64) || defined(_M_ARM64EC) auto _Result = memchr(_First, _Val, _Byte_length(_First, _Last)); return _Result ? _Result : _Last; #else @@ -4305,6 +4603,7 @@ const void* __stdcall __std_find_trivial_8( return _Finding::_Find_impl<_Finding::_Find_traits_8, _Finding::_Predicate::_Equal>(_First, _Last, _Val); } +#ifndef _M_ARM64 const void* __stdcall __std_find_last_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { return _Finding::_Find_last_impl<_Finding::_Find_traits_1, _Finding::_Predicate::_Equal>(_First, _Last, _Val); @@ -4324,6 +4623,7 @@ const void* __stdcall __std_find_last_trivial_8( const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { return _Finding::_Find_last_impl<_Finding::_Find_traits_8, _Finding::_Predicate::_Equal>(_First, _Last, _Val); } +#endif // ^^^ !defined(_M_ARM64) ^^^ const void* __stdcall __std_find_not_ch_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { @@ -4345,6 +4645,7 @@ const void* __stdcall __std_find_not_ch_8( return _Finding::_Find_impl<_Finding::_Find_traits_8, _Finding::_Predicate::_Not_equal>(_First, _Last, _Val); } +#ifndef _M_ARM64 __declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { return _Finding::_Find_last_pos_impl<_Finding::_Find_traits_1, _Finding::_Predicate::_Not_equal>( @@ -4404,9 +4705,11 @@ const void* __stdcall __std_search_n_8( const void* const _First, const void* const _Last, const size_t _Count, const uint64_t _Value) noexcept { return _Finding::_Search_n_impl<_Finding::_Find_traits_8>(_First, _Last, _Count, _Value); } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // extern "C" +#ifndef _M_ARM64 namespace { namespace _Counting { #ifdef _M_ARM64EC From 70a9ec7860261a9ecff18361c0fefa3498dd9151 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Sun, 11 Jan 2026 01:37:05 +0000 Subject: [PATCH 2/7] Remove unused helpers --- stl/src/vector_algorithms.cpp | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f7337994541..60ba2e06407 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3699,11 +3699,6 @@ namespace { return vget_lane_u64(vreinterpret_u64_u8(_Val), 0); } - static uint64_t _Match_mask_q(const uint8x16_t _Val) noexcept { - const uint8x16_t _Res = vpmaxq_u8(_Val, _Val); - return vgetq_lane_u64(vreinterpretq_u64_u8(_Res), 0); - } - static uint64_t _Match_mask_q(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { auto _Cmp = vreinterpretq_u64_u8(vorrq_u8(_Cmp_lo, _Cmp_hi)); return vgetq_lane_u64(vpaddq_u64(_Cmp, _Cmp), 0); @@ -3753,11 +3748,6 @@ namespace { return vget_lane_u64(vreinterpret_u64_u16(_Val), 0); } - static uint64_t _Match_mask_q(const uint16x8_t _Val) noexcept { - const uint16x8_t _Res = vpmaxq_u16(_Val, _Val); - return vgetq_lane_u64(vreinterpretq_u64_u16(_Res), 0); - } - static uint64_t _Match_mask_q(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { uint8x8_t _Cmp = vaddhn_u16(_Cmp_lo, _Cmp_hi); return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); @@ -3807,11 +3797,6 @@ namespace { return vget_lane_u64(vreinterpret_u64_u32(_Val), 0); } - static uint64_t _Match_mask_q(const uint32x4_t _Val) noexcept { - const uint32x4_t _Res = vpmaxq_u32(_Val, _Val); - return vgetq_lane_u64(vreinterpretq_u64_u32(_Res), 0); - } - static uint64_t _Match_mask_q(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u32(_Cmp_lo), vreinterpretq_u16_u32(_Cmp_hi)); return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); @@ -3845,10 +3830,6 @@ namespace { return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); } - static uint64_t _Match_mask_q(const uint64x2_t _Val) noexcept { - return _Mask_q(_Val); - } - static uint64_t _Match_mask_q(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u64(_Cmp_lo), vreinterpretq_u16_u64(_Cmp_hi)); return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); From 2e2fa57730f869e9ac59a60c32a749715f0acc24 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Sun, 11 Jan 2026 01:33:57 +0000 Subject: [PATCH 3/7] Top level const on _Load helpers --- stl/src/vector_algorithms.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 60ba2e06407..60bc8bcff9b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3665,11 +3665,11 @@ namespace { namespace _Finding { #ifdef _M_ARM64 struct _Find_traits_1 { - static uint8x16_t _Load_q(const void* _Ptr) noexcept { + static uint8x16_t _Load_q(const void* const _Ptr) noexcept { return vld1q_u8(static_cast(_Ptr)); } - static uint8x8_t _Load(const void* _Ptr) noexcept { + static uint8x8_t _Load(const void* const _Ptr) noexcept { return vld1_u8(static_cast(_Ptr)); } @@ -3714,11 +3714,11 @@ namespace { }; struct _Find_traits_2 { - static uint16x8_t _Load_q(const void* _Ptr) noexcept { + static uint16x8_t _Load_q(const void* const _Ptr) noexcept { return vld1q_u16(static_cast(_Ptr)); } - static uint16x4_t _Load(const void* _Ptr) noexcept { + static uint16x4_t _Load(const void* const _Ptr) noexcept { return vld1_u16(static_cast(_Ptr)); } @@ -3763,11 +3763,11 @@ namespace { }; struct _Find_traits_4 { - static uint32x4_t _Load_q(const void* _Ptr) noexcept { + static uint32x4_t _Load_q(const void* const _Ptr) noexcept { return vld1q_u32(static_cast(_Ptr)); } - static uint32x2_t _Load(const void* _Ptr) noexcept { + static uint32x2_t _Load(const void* const _Ptr) noexcept { return vld1_u32(static_cast(_Ptr)); } @@ -3812,7 +3812,7 @@ namespace { }; struct _Find_traits_8 { - static uint64x2_t _Load_q(const void* _Ptr) noexcept { + static uint64x2_t _Load_q(const void* const _Ptr) noexcept { return vld1q_u64(static_cast(_Ptr)); } From f861bfd30f50103979d0fee58e3f0929963fd324 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Sat, 10 Jan 2026 15:13:20 +0000 Subject: [PATCH 4/7] Do negation in GPR for Not Equal predicate --- stl/src/vector_algorithms.cpp | 73 +++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 60bc8bcff9b..10d4c0926dd 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3699,17 +3699,15 @@ namespace { return vget_lane_u64(vreinterpret_u64_u8(_Val), 0); } - static uint64_t _Match_mask_q(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { + static uint64_t _Match_mask_eq(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { auto _Cmp = vreinterpretq_u64_u8(vorrq_u8(_Cmp_lo, _Cmp_hi)); return vgetq_lane_u64(vpaddq_u64(_Cmp, _Cmp), 0); } - static uint8x16_t _Not_q(const uint8x16_t _Val) { - return vmvnq_u8(_Val); - } - - static uint8x8_t _Not(const uint8x8_t _Val) { - return vmvn_u8(_Val); + static uint64_t _Match_mask_ne(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { + auto _Cmp = vminq_u8(_Cmp_lo, _Cmp_hi); + auto _Comb = vreinterpretq_u64_u8(vpminq_u8(_Cmp, _Cmp)); + return vgetq_lane_u64(_Comb, 0) ^ 0xFFFF'FFFF'FFFF'FFFF; } }; @@ -3748,17 +3746,15 @@ namespace { return vget_lane_u64(vreinterpret_u64_u16(_Val), 0); } - static uint64_t _Match_mask_q(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { + static uint64_t _Match_mask_eq(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { uint8x8_t _Cmp = vaddhn_u16(_Cmp_lo, _Cmp_hi); return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); } - static uint16x8_t _Not_q(const uint16x8_t _Val) { - return vmvnq_u16(_Val); - } - - static uint16x4_t _Not(const uint16x4_t _Val) { - return vmvn_u16(_Val); + static uint64_t _Match_mask_ne(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { + auto _Cmp = vminq_u16(_Cmp_lo, _Cmp_hi); + auto _Comb = vreinterpretq_u64_u16(vpminq_u16(_Cmp, _Cmp)); + return vgetq_lane_u64(_Comb, 0) ^ 0xFFFF'FFFF'FFFF'FFFF; } }; @@ -3797,17 +3793,15 @@ namespace { return vget_lane_u64(vreinterpret_u64_u32(_Val), 0); } - static uint64_t _Match_mask_q(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { + static uint64_t _Match_mask_eq(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u32(_Cmp_lo), vreinterpretq_u16_u32(_Cmp_hi)); return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); } - static uint32x4_t _Not_q(const uint32x4_t _Val) { - return vmvnq_u32(_Val); - } - - static uint32x2_t _Not(const uint32x2_t _Val) { - return vmvn_u32(_Val); + static uint64_t _Match_mask_ne(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { + auto _Cmp = vminq_u32(_Cmp_lo, _Cmp_hi); + auto _Comb = vreinterpretq_u64_u32(vpminq_u32(_Cmp, _Cmp)); + return vgetq_lane_u64(_Comb, 0) ^ 0xFFFF'FFFF'FFFF'FFFF; } }; @@ -3830,13 +3824,13 @@ namespace { return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); } - static uint64_t _Match_mask_q(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { + static uint64_t _Match_mask_eq(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u64(_Cmp_lo), vreinterpretq_u16_u64(_Cmp_hi)); return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); } - static uint64x2_t _Not_q(const uint64x2_t _Val) { - return vreinterpretq_u64_u8(vmvnq_u8(vreinterpretq_u8_u64(_Val))); + static uint64_t _Match_mask_ne(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { + return _Mask_q(vandq_u64(_Cmp_lo, _Cmp_hi)) ^ 0xFFFF'FFFF'FFFF'FFFF; } }; @@ -3975,23 +3969,32 @@ namespace { auto _Comparison_lo = _Traits::_Cmp_neon_q(_Data_lo, _Comparand); auto _Comparison_hi = _Traits::_Cmp_neon_q(_Data_hi, _Comparand); - if constexpr (_Pred == _Predicate::_Not_equal) { - _Comparison_lo = _Traits::_Not_q(_Comparison_lo); - _Comparison_hi = _Traits::_Not_q(_Comparison_hi); - } // Use a fast check for the termination condition. - uint64_t _Any_match = _Traits::_Match_mask_q(_Comparison_lo, _Comparison_hi); + uint64_t _Any_match = 0; + if constexpr (_Pred == _Predicate::_Not_equal) { + _Any_match = _Traits::_Match_mask_ne(_Comparison_lo, _Comparison_hi); + } else { + _Any_match = _Traits::_Match_mask_eq(_Comparison_lo, _Comparison_hi); + } if (_Any_match != 0) { auto _Mask_lo = _Traits::_Mask_q(_Comparison_lo); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Mask_lo ^= 0xFFFF'FFFF'FFFF'FFFF; + } + if (_Mask_lo != 0) { const auto _Offset = _Get_first_h_pos_q(_Mask_lo); _Advance_bytes(_First, _Offset); return _First; } - auto _Mask_hi = _Traits::_Mask_q(_Comparison_hi); + auto _Mask_hi = _Traits::_Mask_q(_Comparison_hi); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Mask_hi ^= 0xFFFF'FFFF'FFFF'FFFF; + } + const auto _Offset = _Get_first_h_pos_q(_Mask_hi) + 16; _Advance_bytes(_First, _Offset); return _First; @@ -4006,11 +4009,12 @@ namespace { const auto _Data = _Traits::_Load_q(_First); auto _Comparison = _Traits::_Cmp_neon_q(_Data, _Comparand); + + auto _Match = _Traits::_Mask_q(_Comparison); if constexpr (_Pred == _Predicate::_Not_equal) { - _Comparison = _Traits::_Not_q(_Comparison); + _Match ^= 0xFFFF'FFFF'FFFF'FFFF; } - auto _Match = _Traits::_Mask_q(_Comparison); if (_Match != 0) { const auto _Offset = _Get_first_h_pos_q(_Match); _Advance_bytes(_First, _Offset); @@ -4026,11 +4030,12 @@ namespace { const auto _Data = _Traits::_Load(_First); auto _Comparison = _Traits::_Cmp_neon(_Data, _Comparand); + + auto _Match = _Traits::_Mask(_Comparison); if constexpr (_Pred == _Predicate::_Not_equal) { - _Comparison = _Traits::_Not(_Comparison); + _Match ^= 0xFFFF'FFFF'FFFF'FFFF; } - auto _Match = _Traits::_Mask(_Comparison); if (_Match != 0) { const auto _Offset = _Get_first_h_pos_d(_Match); _Advance_bytes(_First, _Offset); From d8003128101d717d20cc5cf607617b4f332bbcb4 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Mon, 12 Jan 2026 13:51:03 +0000 Subject: [PATCH 5/7] Style fixes - Remove redundant static modifiers - Add noexcept where missing - Add top level const where missing --- stl/src/vector_algorithms.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 10d4c0926dd..dcd43672efb 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3834,11 +3834,11 @@ namespace { } }; - static unsigned long _Get_first_h_pos_q(uint64_t _Mask) { + unsigned long _Get_first_h_pos_q(const uint64_t _Mask) noexcept { return _CountTrailingZeros64(_Mask) >> 2; } - static unsigned long _Get_first_h_pos_d(uint64_t _Mask) { + unsigned long _Get_first_h_pos_d(const uint64_t _Mask) noexcept { return _CountTrailingZeros64(_Mask) >> 3; } #elif defined(_M_ARM64EC) @@ -3935,7 +3935,7 @@ namespace { enum class _Predicate { _Equal, _Not_equal }; template <_Predicate _Pred, class _Ty> - static const void* _Find_scalar_tail(const void* _First, const void* const _Last, const _Ty _Val) noexcept { + const void* _Find_scalar_tail(const void* _First, const void* const _Last, const _Ty _Val) noexcept { auto _Ptr = static_cast(_First); if constexpr (_Pred == _Predicate::_Not_equal) { while (_Ptr != _Last && *_Ptr == _Val) { From b8f1ec44670b5c581986475f7e9c50a619060efe Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 13 Jan 2026 12:16:50 -0800 Subject: [PATCH 6/7] Add const. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index dcd43672efb..56ceb4b034f 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3935,7 +3935,7 @@ namespace { enum class _Predicate { _Equal, _Not_equal }; template <_Predicate _Pred, class _Ty> - const void* _Find_scalar_tail(const void* _First, const void* const _Last, const _Ty _Val) noexcept { + const void* _Find_scalar_tail(const void* const _First, const void* const _Last, const _Ty _Val) noexcept { auto _Ptr = static_cast(_First); if constexpr (_Pred == _Predicate::_Not_equal) { while (_Ptr != _Last && *_Ptr == _Val) { From 9f8fd2685a43f5c1608b25943480210d6353f69b Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 13 Jan 2026 12:21:26 -0800 Subject: [PATCH 7/7] Deduce `_Ty` when calling `_Find_scalar_tail`. --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 56ceb4b034f..a1665e1cf04 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -4046,7 +4046,7 @@ namespace { } } - return _Find_scalar_tail<_Pred, _Ty>(_First, _Last, _Val); + return _Find_scalar_tail<_Pred>(_First, _Last, _Val); } #else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv template @@ -4126,7 +4126,7 @@ namespace { } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - return _Find_scalar_tail<_Pred, _Ty>(_First, _Last, _Val); + return _Find_scalar_tail<_Pred>(_First, _Last, _Val); } template