From ff5e7623b548490d78bbb59a55faea8b56010aec Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 6 May 2024 20:51:19 +0300 Subject: [PATCH 01/23] additional dispatch level, separate SSE traits --- stl/src/vector_algorithms.cpp | 231 +++++++++++++++++++++++----------- 1 file changed, 158 insertions(+), 73 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 69cb787e350..9a8d3aa0a12 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -540,7 +540,7 @@ namespace { _Mode_both = _Mode_min | _Mode_max, }; - struct _Minmax_traits_1 { + struct _Minmax_traits_1_base { static constexpr bool _Is_floating = false; using _Signed_t = int8_t; @@ -555,7 +555,11 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 256; +#endif //_M_ARM64EC + }; +#ifndef _M_ARM64EC + struct _Minmax_traits_1_sse : _Minmax_traits_1_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -638,10 +642,10 @@ namespace { static __m128i _Mask_cast(__m128i _Mask) noexcept { return _Mask; } -#endif // !_M_ARM64EC }; +#endif // !_M_ARM64EC - struct _Minmax_traits_2 { + struct _Minmax_traits_2_base { static constexpr bool _Is_floating = false; using _Signed_t = int16_t; @@ -656,7 +660,11 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 65536; +#endif // !_M_ARM64EC + }; +#ifndef _M_ARM64EC + struct _Minmax_traits_2_sse : _Minmax_traits_2_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -740,10 +748,10 @@ namespace { static __m128i _Mask_cast(__m128i _Mask) noexcept { return _Mask; } -#endif // !_M_ARM64EC }; +#endif // !_M_ARM64EC - struct _Minmax_traits_4 { + struct _Minmax_traits_4_base { static constexpr bool _Is_floating = false; using _Signed_t = int32_t; @@ -762,7 +770,11 @@ namespace { static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 0x1'0000'0000ULL; #endif // ^^^ 64-bit ^^^ +#endif // !_M_ARM64EC + }; +#ifndef _M_ARM64EC + struct _Minmax_traits_4_sse : _Minmax_traits_4_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -842,10 +854,10 @@ namespace { static __m128i _Mask_cast(__m128i _Mask) noexcept { return _Mask; } -#endif // !_M_ARM64EC }; +#endif // !_M_ARM64EC - struct _Minmax_traits_8 { + struct _Minmax_traits_8_base { static constexpr bool _Is_floating = false; using _Signed_t = int64_t; @@ -859,7 +871,11 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = false; +#endif // !_M_ARM64EC + }; +#ifndef _M_ARM64EC + struct _Minmax_traits_8_sse : _Minmax_traits_8_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -947,10 +963,10 @@ namespace { static __m128i _Mask_cast(__m128i _Mask) noexcept { return _Mask; } -#endif // !_M_ARM64EC }; +#endif // !_M_ARM64EC - struct _Minmax_traits_f { + struct _Minmax_traits_f_base { static constexpr bool _Is_floating = true; using _Signed_t = float; @@ -969,7 +985,11 @@ namespace { static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 0x1'0000'0000ULL; #endif // ^^^ 64-bit ^^^ +#endif // !_M_ARM64EC + }; +#ifndef _M_ARM64EC + struct _Minmax_traits_f_sse : _Minmax_traits_f_base { static __m128 _Load(const void* _Src) noexcept { return _mm_loadu_ps(reinterpret_cast(_Src)); } @@ -1047,10 +1067,10 @@ namespace { static __m128i _Mask_cast(__m128 _Mask) noexcept { return _mm_castps_si128(_Mask); } -#endif // !_M_ARM64EC }; +#endif // !_M_ARM64EC - struct _Minmax_traits_d { + struct _Minmax_traits_d_base { static constexpr bool _Is_floating = true; using _Signed_t = double; @@ -1064,7 +1084,11 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = false; +#endif // !_M_ARM64EC + }; +#ifndef _M_ARM64EC + struct _Minmax_traits_d_sse : _Minmax_traits_d_base { static __m128d _Load(const void* _Src) noexcept { return _mm_loadu_pd(reinterpret_cast(_Src)); } @@ -1151,15 +1175,57 @@ namespace { static __m128i _Mask_cast(__m128d _Mask) noexcept { return _mm_castpd_si128(_Mask); } + }; +#endif // !_M_ARM64EC + + struct _Minmax_traits_1 { + using _Base = _Minmax_traits_1_base; +#ifndef _M_ARM64EC + using _Sse = _Minmax_traits_1_sse; +#endif // !_M_ARM64EC + }; + + struct _Minmax_traits_2 { + using _Base = _Minmax_traits_2_base; +#ifndef _M_ARM64EC + using _Sse = _Minmax_traits_2_sse; +#endif // !_M_ARM64EC + }; + + struct _Minmax_traits_4 { + using _Base = _Minmax_traits_4_base; +#ifndef _M_ARM64EC + using _Sse = _Minmax_traits_4_sse; +#endif // !_M_ARM64EC + }; + + struct _Minmax_traits_8 { + using _Base = _Minmax_traits_8_base; +#ifndef _M_ARM64EC + using _Sse = _Minmax_traits_8_sse; #endif // !_M_ARM64EC }; - // __std_minmax_element_impl has exactly the same signature as the extern "C" functions - // (__std_min_element_N, __std_max_element_N, __std_minmax_element_N), up to calling convention. - // This makes sure the template specialization is fused with the extern "C" function. - // In optimized builds it avoids an extra call, as this function is too large to inline. + struct _Minmax_traits_f { + using _Base = _Minmax_traits_f_base; +#ifndef _M_ARM64EC + using _Sse = _Minmax_traits_f_sse; +#endif // !_M_ARM64EC + }; + + struct _Minmax_traits_d { + using _Base = _Minmax_traits_d_base; +#ifndef _M_ARM64EC + using _Sse = _Minmax_traits_d_sse; +#endif // !_M_ARM64EC + }; + + // __std_minmax_element_impl and __std_minmax_element_disp have exactly the same signature + // as the extern "C" functions (__std_min_element_N, __std_max_element_N, __std_minmax_element_N), + // up to calling convention. + // This makes sure the template specialization can be tail called without filling in the params. template <_Min_max_mode _Mode, class _Traits> - auto __stdcall __std_minmax_element_impl(const void* _First, const void* const _Last, const bool _Sign) noexcept { + auto __std_minmax_element_impl(const void* _First, const void* const _Last, const bool _Sign) noexcept { _Min_max_element_t _Res = {_First, _First}; auto _Cur_min_val = _Traits::_Init_min_val; auto _Cur_max_val = _Traits::_Init_max_val; @@ -1377,10 +1443,19 @@ namespace { } } - // __std_minmax_impl has exactly the same signature as the extern "C" functions + template <_Min_max_mode _Mode, class _Traits> + auto __std_minmax_element_disp(const void* _First, const void* const _Last, const bool _Sign) noexcept { +#ifdef _M_ARM64EC + using _Inner_traits = _Traits::_Base; +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv + using _Inner_traits = _Traits::_Sse; +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + return __std_minmax_element_impl<_Mode, _Inner_traits>(_First, _Last, _Sign); + } + + // __std_minmax_impl and __std_minmax_disp have exactly the same signature as the extern "C" functions // (__std_min_Nn, __std_max_Nn, __std_minmax_Nn), up to calling convention. - // This makes sure the template specialization is fused with the extern "C" function. - // In optimized builds it avoids an extra call, as this function is too large to inline. + // This makes sure the template specialization can be tail called without filling in the params. template <_Min_max_mode _Mode, class _Traits, bool _Sign> auto __stdcall __std_minmax_impl(const void* _First, const void* const _Last) noexcept { using _Ty = std::conditional_t<_Sign, typename _Traits::_Signed_t, typename _Traits::_Unsigned_t>; @@ -1514,220 +1589,230 @@ namespace { } } + template <_Min_max_mode _Mode, class _Traits, bool _Sign> + auto __stdcall __std_minmax_disp(const void* _First, const void* const _Last) noexcept { +#ifdef _M_ARM64EC + using _Inner_traits = _Traits::_Base; +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv + using _Inner_traits = _Traits::_Sse; +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + return __std_minmax_impl<_Mode, _Inner_traits, _Sign>(_First, _Last); + } + } // unnamed namespace extern "C" { const void* __stdcall __std_min_element_1( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_min, _Minmax_traits_1>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_1>(_First, _Last, _Signed); } const void* __stdcall __std_min_element_2( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_min, _Minmax_traits_2>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_2>(_First, _Last, _Signed); } const void* __stdcall __std_min_element_4( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_min, _Minmax_traits_4>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_4>(_First, _Last, _Signed); } const void* __stdcall __std_min_element_8( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_min, _Minmax_traits_8>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_8>(_First, _Last, _Signed); } -const void* __stdcall __std_min_element_f( // __std_minmax_element_impl's "signature" comment explains `bool _Unused` +const void* __stdcall __std_min_element_f( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_impl<_Mode_min, _Minmax_traits_f>(_First, _Last, _Unused); + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_f>(_First, _Last, _Unused); } -const void* __stdcall __std_min_element_d( // __std_minmax_element_impl's "signature" comment explains `bool _Unused` +const void* __stdcall __std_min_element_d( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_impl<_Mode_min, _Minmax_traits_d>(_First, _Last, _Unused); + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_d>(_First, _Last, _Unused); } const void* __stdcall __std_max_element_1( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_max, _Minmax_traits_1>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_1>(_First, _Last, _Signed); } const void* __stdcall __std_max_element_2( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_max, _Minmax_traits_2>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_2>(_First, _Last, _Signed); } const void* __stdcall __std_max_element_4( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_max, _Minmax_traits_4>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_4>(_First, _Last, _Signed); } const void* __stdcall __std_max_element_8( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_max, _Minmax_traits_8>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_8>(_First, _Last, _Signed); } -const void* __stdcall __std_max_element_f( // __std_minmax_element_impl's "signature" comment explains `bool _Unused` +const void* __stdcall __std_max_element_f( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_impl<_Mode_max, _Minmax_traits_f>(_First, _Last, _Unused); + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_f>(_First, _Last, _Unused); } -const void* __stdcall __std_max_element_d( // __std_minmax_element_impl's "signature" comment explains `bool _Unused` +const void* __stdcall __std_max_element_d( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_impl<_Mode_max, _Minmax_traits_d>(_First, _Last, _Unused); + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_d>(_First, _Last, _Unused); } _Min_max_element_t __stdcall __std_minmax_element_1( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_both, _Minmax_traits_1>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_1>(_First, _Last, _Signed); } _Min_max_element_t __stdcall __std_minmax_element_2( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_both, _Minmax_traits_2>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_2>(_First, _Last, _Signed); } _Min_max_element_t __stdcall __std_minmax_element_4( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_both, _Minmax_traits_4>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_4>(_First, _Last, _Signed); } _Min_max_element_t __stdcall __std_minmax_element_8( const void* const _First, const void* const _Last, const bool _Signed) noexcept { - return __std_minmax_element_impl<_Mode_both, _Minmax_traits_8>(_First, _Last, _Signed); + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_8>(_First, _Last, _Signed); } -// __std_minmax_element_impl's "signature" comment explains `bool _Unused` +// __std_minmax_element_disp's "signature" comment explains `bool _Unused` _Min_max_element_t __stdcall __std_minmax_element_f( const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_impl<_Mode_both, _Minmax_traits_f>(_First, _Last, _Unused); + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_f>(_First, _Last, _Unused); } -// __std_minmax_element_impl's "signature" comment explains `bool _Unused` +// __std_minmax_element_disp's "signature" comment explains `bool _Unused` _Min_max_element_t __stdcall __std_minmax_element_d( const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_impl<_Mode_both, _Minmax_traits_d>(_First, _Last, _Unused); + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_d>(_First, _Last, _Unused); } __declspec(noalias) int8_t __stdcall __std_min_1i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_1, true>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_1, true>(_First, _Last); } __declspec(noalias) uint8_t __stdcall __std_min_1u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_1, false>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_1, false>(_First, _Last); } __declspec(noalias) int16_t __stdcall __std_min_2i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_2, true>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_2, true>(_First, _Last); } __declspec(noalias) uint16_t __stdcall __std_min_2u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_2, false>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_2, false>(_First, _Last); } __declspec(noalias) int32_t __stdcall __std_min_4i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_4, true>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_4, true>(_First, _Last); } __declspec(noalias) uint32_t __stdcall __std_min_4u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_4, false>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_4, false>(_First, _Last); } __declspec(noalias) int64_t __stdcall __std_min_8i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_8, true>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_8, true>(_First, _Last); } __declspec(noalias) uint64_t __stdcall __std_min_8u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_8, false>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_8, false>(_First, _Last); } __declspec(noalias) float __stdcall __std_min_f(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_f, true>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_f, true>(_First, _Last); } __declspec(noalias) double __stdcall __std_min_d(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_min, _Minmax_traits_d, true>(_First, _Last); + return __std_minmax_disp<_Mode_min, _Minmax_traits_d, true>(_First, _Last); } __declspec(noalias) int8_t __stdcall __std_max_1i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_1, true>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_1, true>(_First, _Last); } __declspec(noalias) uint8_t __stdcall __std_max_1u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_1, false>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_1, false>(_First, _Last); } __declspec(noalias) int16_t __stdcall __std_max_2i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_2, true>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_2, true>(_First, _Last); } __declspec(noalias) uint16_t __stdcall __std_max_2u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_2, false>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_2, false>(_First, _Last); } __declspec(noalias) int32_t __stdcall __std_max_4i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_4, true>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_4, true>(_First, _Last); } __declspec(noalias) uint32_t __stdcall __std_max_4u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_4, false>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_4, false>(_First, _Last); } __declspec(noalias) int64_t __stdcall __std_max_8i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_8, true>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_8, true>(_First, _Last); } __declspec(noalias) uint64_t __stdcall __std_max_8u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_8, false>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_8, false>(_First, _Last); } __declspec(noalias) float __stdcall __std_max_f(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_f, true>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_f, true>(_First, _Last); } __declspec(noalias) double __stdcall __std_max_d(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_max, _Minmax_traits_d, true>(_First, _Last); + return __std_minmax_disp<_Mode_max, _Minmax_traits_d, true>(_First, _Last); } __declspec(noalias) _Min_max_1i __stdcall __std_minmax_1i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_1, true>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_1, true>(_First, _Last); } __declspec(noalias) _Min_max_1u __stdcall __std_minmax_1u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_1, false>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_1, false>(_First, _Last); } __declspec(noalias) _Min_max_2i __stdcall __std_minmax_2i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_2, true>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_2, true>(_First, _Last); } __declspec(noalias) _Min_max_2u __stdcall __std_minmax_2u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_2, false>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_2, false>(_First, _Last); } __declspec(noalias) _Min_max_4i __stdcall __std_minmax_4i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_4, true>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_4, true>(_First, _Last); } __declspec(noalias) _Min_max_4u __stdcall __std_minmax_4u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_4, false>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_4, false>(_First, _Last); } __declspec(noalias) _Min_max_8i __stdcall __std_minmax_8i(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_8, true>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_8, true>(_First, _Last); } __declspec(noalias) _Min_max_8u __stdcall __std_minmax_8u(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_8, false>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_8, false>(_First, _Last); } __declspec(noalias) _Min_max_f __stdcall __std_minmax_f(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_f, true>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_f, true>(_First, _Last); } __declspec(noalias) _Min_max_d __stdcall __std_minmax_d(const void* const _First, const void* const _Last) noexcept { - return __std_minmax_impl<_Mode_both, _Minmax_traits_d, true>(_First, _Last); + return __std_minmax_disp<_Mode_both, _Minmax_traits_d, true>(_First, _Last); } } // extern "C" From ff019c135d355e1ac6d4b2050ee26ea665726c87 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 6 May 2024 21:18:52 +0300 Subject: [PATCH 02/23] additional dispatch does dispatch, scalar traits --- stl/src/vector_algorithms.cpp | 90 ++++++++++++++++++++++------------- 1 file changed, 56 insertions(+), 34 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 9a8d3aa0a12..552f4a00775 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -540,6 +540,14 @@ namespace { _Mode_both = _Mode_min | _Mode_max, }; + struct _Minmax_traits_scalar_base { + static constexpr bool _Vectorized = false; + }; + + struct _Minmax_traits_sse_base { + static constexpr bool _Vectorized = true; + }; + struct _Minmax_traits_1_base { static constexpr bool _Is_floating = false; @@ -558,8 +566,10 @@ namespace { #endif //_M_ARM64EC }; + struct _Minmax_traits_1_scalar : _Minmax_traits_1_base, _Minmax_traits_scalar_base {}; + #ifndef _M_ARM64EC - struct _Minmax_traits_1_sse : _Minmax_traits_1_base { + struct _Minmax_traits_1_sse : _Minmax_traits_1_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -663,8 +673,10 @@ namespace { #endif // !_M_ARM64EC }; + struct _Minmax_traits_2_scalar : _Minmax_traits_2_base, _Minmax_traits_scalar_base {}; + #ifndef _M_ARM64EC - struct _Minmax_traits_2_sse : _Minmax_traits_2_base { + struct _Minmax_traits_2_sse : _Minmax_traits_2_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -773,8 +785,10 @@ namespace { #endif // !_M_ARM64EC }; + struct _Minmax_traits_4_scalar : _Minmax_traits_4_base, _Minmax_traits_scalar_base {}; + #ifndef _M_ARM64EC - struct _Minmax_traits_4_sse : _Minmax_traits_4_base { + struct _Minmax_traits_4_sse : _Minmax_traits_4_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -874,8 +888,10 @@ namespace { #endif // !_M_ARM64EC }; + struct _Minmax_traits_8_scalar : _Minmax_traits_8_base, _Minmax_traits_scalar_base {}; + #ifndef _M_ARM64EC - struct _Minmax_traits_8_sse : _Minmax_traits_8_base { + struct _Minmax_traits_8_sse : _Minmax_traits_8_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); } @@ -988,8 +1004,10 @@ namespace { #endif // !_M_ARM64EC }; + struct _Minmax_traits_f_scalar : _Minmax_traits_f_base, _Minmax_traits_scalar_base {}; + #ifndef _M_ARM64EC - struct _Minmax_traits_f_sse : _Minmax_traits_f_base { + struct _Minmax_traits_f_sse : _Minmax_traits_f_base, _Minmax_traits_sse_base { static __m128 _Load(const void* _Src) noexcept { return _mm_loadu_ps(reinterpret_cast(_Src)); } @@ -1087,8 +1105,10 @@ namespace { #endif // !_M_ARM64EC }; + struct _Minmax_traits_d_scalar : _Minmax_traits_d_base, _Minmax_traits_scalar_base {}; + #ifndef _M_ARM64EC - struct _Minmax_traits_d_sse : _Minmax_traits_d_base { + struct _Minmax_traits_d_sse : _Minmax_traits_d_base, _Minmax_traits_sse_base { static __m128d _Load(const void* _Src) noexcept { return _mm_loadu_pd(reinterpret_cast(_Src)); } @@ -1179,42 +1199,42 @@ namespace { #endif // !_M_ARM64EC struct _Minmax_traits_1 { - using _Base = _Minmax_traits_1_base; + using _Scalar = _Minmax_traits_1_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_1_sse; #endif // !_M_ARM64EC }; struct _Minmax_traits_2 { - using _Base = _Minmax_traits_2_base; + using _Scalar = _Minmax_traits_2_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_2_sse; #endif // !_M_ARM64EC }; struct _Minmax_traits_4 { - using _Base = _Minmax_traits_4_base; + using _Scalar = _Minmax_traits_4_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_4_sse; #endif // !_M_ARM64EC }; struct _Minmax_traits_8 { - using _Base = _Minmax_traits_8_base; + using _Scalar = _Minmax_traits_8_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_8_sse; #endif // !_M_ARM64EC }; struct _Minmax_traits_f { - using _Base = _Minmax_traits_f_base; + using _Scalar = _Minmax_traits_f_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_f_sse; #endif // !_M_ARM64EC }; struct _Minmax_traits_d { - using _Base = _Minmax_traits_d_base; + using _Scalar = _Minmax_traits_d_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_d_sse; #endif // !_M_ARM64EC @@ -1230,10 +1250,11 @@ namespace { auto _Cur_min_val = _Traits::_Init_min_val; auto _Cur_max_val = _Traits::_Init_max_val; -#ifndef _M_ARM64EC - auto _Base = static_cast(_First); - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + if constexpr (_Traits::_Vectorized) { +#ifdef _M_ARM64EC + static_assert(false, "No vectorization for _M_ARM64EC yet"); +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv + auto _Base = static_cast(_First); size_t _Portion_byte_size = _Byte_length(_First, _Last) & ~size_t{0xF}; if constexpr (_Traits::_Has_portion_max) { @@ -1402,8 +1423,8 @@ namespace { } } } - } #endif // !_M_ARM64EC + } if constexpr (_Traits::_Is_floating) { if constexpr (_Mode == _Mode_min) { @@ -1445,12 +1466,12 @@ namespace { template <_Min_max_mode _Mode, class _Traits> auto __std_minmax_element_disp(const void* _First, const void* const _Last, const bool _Sign) noexcept { -#ifdef _M_ARM64EC - using _Inner_traits = _Traits::_Base; -#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv - using _Inner_traits = _Traits::_Sse; +#ifndef _M_ARM64EC + if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + return __std_minmax_element_impl<_Mode, typename _Traits::_Sse>(_First, _Last, _Sign); + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - return __std_minmax_element_impl<_Mode, _Inner_traits>(_First, _Last, _Sign); + return __std_minmax_element_impl<_Mode, typename _Traits::_Scalar>(_First, _Last, _Sign); } // __std_minmax_impl and __std_minmax_disp have exactly the same signature as the extern "C" functions @@ -1463,11 +1484,10 @@ namespace { _Ty _Cur_min_val; // initialized in both of the branches below _Ty _Cur_max_val; // initialized in both of the branches below -#ifndef _M_ARM64EC - // We don't have unsigned 64-bit stuff, so we'll use sign correction just for that case - constexpr bool _Sign_correction = sizeof(_Ty) == 8 && !_Sign; - - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + if constexpr (_Traits::_Vectorized) { +#ifdef _M_ARM64EC + static_assert(false, "No vectorization for _M_ARM64EC yet"); +#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv const size_t _Sse_byte_size = _Byte_length(_First, _Last) & ~size_t{0xF}; const void* _Stop_at = _First; @@ -1475,6 +1495,9 @@ namespace { auto _Cur_vals = _Traits::_Load(_First); + // We don't have unsigned 64-bit stuff, so we'll use sign correction just for that case + constexpr bool _Sign_correction = sizeof(_Ty) == 8 && !_Sign; + if constexpr (_Sign_correction) { _Cur_vals = _Traits::_Sign_correction(_Cur_vals, false); } @@ -1551,9 +1574,8 @@ namespace { break; } } - } else #endif // !_M_ARM64EC - { + } else { _Cur_min_val = *reinterpret_cast(_First); _Cur_max_val = *reinterpret_cast(_First); @@ -1591,12 +1613,12 @@ namespace { template <_Min_max_mode _Mode, class _Traits, bool _Sign> auto __stdcall __std_minmax_disp(const void* _First, const void* const _Last) noexcept { -#ifdef _M_ARM64EC - using _Inner_traits = _Traits::_Base; -#else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv - using _Inner_traits = _Traits::_Sse; +#ifndef _M_ARM64EC + if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + return __std_minmax_impl<_Mode, typename _Traits::_Sse, _Sign>(_First, _Last); + } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - return __std_minmax_impl<_Mode, _Inner_traits, _Sign>(_First, _Last); + return __std_minmax_impl<_Mode, typename _Traits::_Scalar, _Sign>(_First, _Last); } } // unnamed namespace From cd2b29fdd5b852e449a080e2d76ac49554527057 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 6 May 2024 22:35:40 +0300 Subject: [PATCH 03/23] move sse specifics to sse traits --- stl/src/vector_algorithms.cpp | 98 ++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 552f4a00775..96b3416fae0 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -546,6 +546,46 @@ namespace { struct _Minmax_traits_sse_base { static constexpr bool _Vectorized = true; + static constexpr size_t _Vec_size = 16; + static constexpr size_t _Vec_mask = 0xF; + + static __m128i _Zero() noexcept { + return _mm_setzero_si128(); + } + + static __m128i _All_ones() noexcept { + return _mm_set1_epi8(static_cast(0xFF)); + } + + static __m128i _Blend(const __m128i _Px1, const __m128i _Px2, const __m128i _Msk) noexcept { + return _mm_blendv_epi8(_Px1, _Px2, _Msk); + } + + static unsigned long _Mask(const __m128i _Val) noexcept { + return _mm_movemask_epi8(_Val); + } + }; + + struct _Minmax_traits_avx_base { + static constexpr bool _Vectorized = true; + static constexpr size_t _Vec_size = 32; + static constexpr size_t _Vec_mask = 0x1F; + + static __m256i _Zero() noexcept { + return _mm256_setzero_si256(); + } + + static __m256i _All_ones() noexcept { + return _mm256_set1_epi8(static_cast(0xFF)); + } + + static __m256i _Blend(const __m256i _Px1, const __m256i _Px2, const __m256i _Msk) noexcept { + return _mm256_blendv_epi8(_Px1, _Px2, _Msk); + } + + static unsigned long _Mask(const __m256i _Val) noexcept { + return _mm256_movemask_epi8(_Val); + } }; struct _Minmax_traits_1_base { @@ -1255,11 +1295,11 @@ namespace { static_assert(false, "No vectorization for _M_ARM64EC yet"); #else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv auto _Base = static_cast(_First); - size_t _Portion_byte_size = _Byte_length(_First, _Last) & ~size_t{0xF}; + size_t _Portion_byte_size = _Byte_length(_First, _Last) & ~_Traits::_Vec_mask; if constexpr (_Traits::_Has_portion_max) { // vector of indices will wrap around at exactly this size - constexpr size_t _Max_portion_byte_size = _Traits::_Portion_max * 16; + constexpr size_t _Max_portion_byte_size = _Traits::_Portion_max * _Traits::_Vec_size; if (_Portion_byte_size > _Max_portion_byte_size) { _Portion_byte_size = _Max_portion_byte_size; } @@ -1271,13 +1311,13 @@ namespace { // Load values and if unsigned adjust them to be signed (for signed vector comparisons) auto _Cur_vals = _Traits::_Sign_correction(_Traits::_Load(_First), _Sign); auto _Cur_vals_min = _Cur_vals; // vector of vertical minimum values - auto _Cur_idx_min = _mm_setzero_si128(); // vector of vertical minimum indices + auto _Cur_idx_min = _Traits::_Zero(); // vector of vertical minimum indices auto _Cur_vals_max = _Cur_vals; // vector of vertical maximum values - auto _Cur_idx_max = _mm_setzero_si128(); // vector of vertical maximum indices - auto _Cur_idx = _mm_setzero_si128(); // current vector of indices + auto _Cur_idx_max = _Traits::_Zero(); // vector of vertical maximum indices + auto _Cur_idx = _Traits::_Zero(); // current vector of indices for (;;) { - _Advance_bytes(_First, 16); + _Advance_bytes(_First, _Traits::_Vec_size); // Increment vertical indices. Will stop at exactly wrap around, if not reach the end before _Cur_idx = _Traits::_Inc(_Cur_idx); @@ -1291,7 +1331,7 @@ namespace { if constexpr ((_Mode & _Mode_min) != 0) { // Looking for the first occurrence of minimum, don't overwrite with newly found occurrences const auto _Is_less = _Traits::_Cmp_gt(_Cur_vals_min, _Cur_vals); // _Cur_vals < _Cur_vals_min - _Cur_idx_min = _mm_blendv_epi8( + _Cur_idx_min = _Traits::_Blend( _Cur_idx_min, _Cur_idx, _Traits::_Mask_cast(_Is_less)); // Remember their vertical indices _Cur_vals_min = _Traits::_Min(_Cur_vals_min, _Cur_vals, _Is_less); // Update the current minimum } @@ -1300,7 +1340,7 @@ namespace { // Looking for the first occurrence of maximum, don't overwrite with newly found occurrences const auto _Is_greater = _Traits::_Cmp_gt(_Cur_vals, _Cur_vals_max); // _Cur_vals > _Cur_vals_max - _Cur_idx_max = _mm_blendv_epi8(_Cur_idx_max, _Cur_idx, + _Cur_idx_max = _Traits::_Blend(_Cur_idx_max, _Cur_idx, _Traits::_Mask_cast(_Is_greater)); // Remember their vertical indices _Cur_vals_max = _Traits::_Max(_Cur_vals_max, _Cur_vals, _Is_greater); // Update the current maximum @@ -1308,7 +1348,7 @@ namespace { // Looking for the last occurrence of maximum, do overwrite with newly found occurrences const auto _Is_less = _Traits::_Cmp_gt(_Cur_vals_max, _Cur_vals); // !(_Cur_vals >= _Cur_vals_max) - _Cur_idx_max = _mm_blendv_epi8(_Cur_idx, _Cur_idx_max, + _Cur_idx_max = _Traits::_Blend(_Cur_idx, _Cur_idx_max, _Traits::_Mask_cast(_Is_less)); // Remember their vertical indices _Cur_vals_max = _Traits::_Max(_Cur_vals, _Cur_vals_max, _Is_less); // Update the current maximum } @@ -1324,22 +1364,22 @@ namespace { _Cur_min_val = _H_min_val; // update min const auto _Eq_mask = _Traits::_Cmp_eq(_H_min, _Cur_vals_min); // Mask of all elems eq to min - int _Mask = _mm_movemask_epi8(_Traits::_Mask_cast(_Eq_mask)); + unsigned long _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); // Indices of minimum elements or the greatest index if none - const auto _All_max = _mm_set1_epi8(static_cast(0xFF)); + const auto _All_max = _Traits::_All_ones(); const auto _Idx_min_val = - _mm_blendv_epi8(_All_max, _Cur_idx_min, _Traits::_Mask_cast(_Eq_mask)); + _Traits::_Blend(_All_max, _Cur_idx_min, _Traits::_Mask_cast(_Eq_mask)); auto _Idx_min = _Traits::_H_min_u(_Idx_min_val); // The smallest indices // Select the smallest vertical indices from the smallest element mask - _Mask &= _mm_movemask_epi8(_Traits::_Cmp_eq_idx(_Idx_min, _Idx_min_val)); + _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_min, _Idx_min_val)); unsigned long _H_pos; // Find the smallest horizontal index _BitScanForward(&_H_pos, _Mask); // lgtm [cpp/conditionallyuninitializedvariable] const auto _V_pos = _Traits::_Get_v_pos(_Cur_idx_min, _H_pos); // Extract its vertical index - _Res._Min = - _Base + static_cast(_V_pos) * 16 + _H_pos; // Finally, compute the pointer + // Finally, compute the pointer + _Res._Min = _Base + static_cast(_V_pos) * _Traits::_Vec_size + _H_pos; } } @@ -1354,17 +1394,17 @@ namespace { _Cur_max_val = _H_max_val; const auto _Eq_mask = _Traits::_Cmp_eq(_H_max, _Cur_vals_max); // Mask of all elems eq to max - int _Mask = _mm_movemask_epi8(_Traits::_Mask_cast(_Eq_mask)); + int _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); unsigned long _H_pos; if constexpr (_Mode == _Mode_both) { // Looking for the last occurrence of maximum // Indices of maximum elements or zero if none const auto _Idx_max_val = - _mm_blendv_epi8(_mm_setzero_si128(), _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); + _Traits::_Blend(_Traits::_Zero(), _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); const auto _Idx_max = _Traits::_H_max_u(_Idx_max_val); // The greatest indices // Select the greatest vertical indices from the largest element mask - _Mask &= _mm_movemask_epi8(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); + _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); // Find the largest horizontal index _BitScanReverse(&_H_pos, _Mask); // lgtm [cpp/conditionallyuninitializedvariable] @@ -1373,32 +1413,32 @@ namespace { } else { // Looking for the first occurrence of maximum // Indices of maximum elements or the greatest index if none - const auto _All_max = _mm_set1_epi8(static_cast(0xFF)); + const auto _All_max = _Traits::_All_ones(); const auto _Idx_max_val = - _mm_blendv_epi8(_All_max, _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); + _Traits::_Blend(_All_max, _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); const auto _Idx_max = _Traits::_H_min_u(_Idx_max_val); // The smallest indices // Select the smallest vertical indices from the largest element mask - _Mask &= _mm_movemask_epi8(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); + _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); // Find the smallest horizontal index _BitScanForward(&_H_pos, _Mask); // lgtm [cpp/conditionallyuninitializedvariable] } const auto _V_pos = _Traits::_Get_v_pos(_Cur_idx_max, _H_pos); // Extract its vertical index - _Res._Max = - _Base + static_cast(_V_pos) * 16 + _H_pos; // Finally, compute the pointer + // Finally, compute the pointer + _Res._Max = _Base + static_cast(_V_pos) * _Traits::_Vec_size + _H_pos; } } // Horizontal part done, results are saved, now need to see if there is another portion to process if constexpr (_Traits::_Has_portion_max) { // Either the last portion or wrapping point reached, need to determine - _Portion_byte_size = _Byte_length(_First, _Last) & ~size_t{0xF}; + _Portion_byte_size = _Byte_length(_First, _Last) & ~_Traits::_Vec_mask; if (_Portion_byte_size == 0) { break; // That was the last portion } // Start next portion to handle the wrapping indices. Assume _Cur_idx is zero - constexpr size_t _Max_portion_byte_size = _Traits::_Portion_max * 16; + constexpr size_t _Max_portion_byte_size = _Traits::_Portion_max * _Traits::_Vec_size; if (_Portion_byte_size > _Max_portion_byte_size) { _Portion_byte_size = _Max_portion_byte_size; } @@ -1411,12 +1451,12 @@ namespace { if constexpr ((_Mode & _Mode_min) != 0) { _Cur_vals_min = _Cur_vals; - _Cur_idx_min = _mm_setzero_si128(); + _Cur_idx_min = _Traits::_Zero(); } if constexpr ((_Mode & _Mode_max) != 0) { _Cur_vals_max = _Cur_vals; - _Cur_idx_max = _mm_setzero_si128(); + _Cur_idx_max = _Traits::_Zero(); } } else { break; // No wrapping, so it was the only portion @@ -1488,7 +1528,7 @@ namespace { #ifdef _M_ARM64EC static_assert(false, "No vectorization for _M_ARM64EC yet"); #else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv - const size_t _Sse_byte_size = _Byte_length(_First, _Last) & ~size_t{0xF}; + const size_t _Sse_byte_size = _Byte_length(_First, _Last) & ~_Traits::_Vec_mask; const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Sse_byte_size); @@ -1506,7 +1546,7 @@ namespace { auto _Cur_vals_max = _Cur_vals; // vector of vertical maximum values for (;;) { - _Advance_bytes(_First, 16); + _Advance_bytes(_First, _Traits::_Vec_size); if (_First != _Stop_at) { // This is the main part, finding vertical minimum/maximum From 560da3d8082932eeab9dae9a342e16f7c53a86ed Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 09:27:31 +0300 Subject: [PATCH 04/23] implement AVX optimization --- stl/src/vector_algorithms.cpp | 584 +++++++++++++++++++++++++++++++++- 1 file changed, 572 insertions(+), 12 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 96b3416fae0..1a62f2fc2fa 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -620,7 +620,7 @@ namespace { return _mm_sub_epi8(_Val, _mm_load_si128(reinterpret_cast(_Sign_corrections[_Sign]))); } - static __m128i _Inc(__m128i _Idx) noexcept { + static __m128i _Inc(const __m128i _Idx) noexcept { return _mm_add_epi8(_Idx, _mm_set1_epi8(1)); } @@ -689,7 +689,97 @@ namespace { return _mm_max_epu8(_First, _Second); } - static __m128i _Mask_cast(__m128i _Mask) noexcept { + static __m128i _Mask_cast(const __m128i _Mask) noexcept { + return _Mask; + } + }; + + struct _Minmax_traits_1_avx : _Minmax_traits_1_base, _Minmax_traits_avx_base { + static __m256i _Load(const void* _Src) noexcept { + return _mm256_loadu_si256(reinterpret_cast(_Src)); + } + + static __m256i _Sign_correction(const __m256i _Val, const bool _Sign) noexcept { + alignas(32) static constexpr _Unsigned_t _Sign_corrections[2][32] = { + {0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80}, + {}}; + return _mm256_sub_epi8(_Val, _mm256_load_si256(reinterpret_cast(_Sign_corrections[_Sign]))); + } + + static __m256i _Inc(const __m256i _Idx) noexcept { + return _mm256_add_epi8(_Idx, _mm256_set1_epi8(1)); + } + + template + static __m256i _H_func(const __m256i _Cur, _Fn _Funct) noexcept { + const __m128i _Shuf_bytes = _mm_set_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + const __m128i _Shuf_words = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + + __m256i _H_min_val = _Cur; + _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi8(_H_min_val, _mm256_broadcastsi128_si256(_Shuf_words))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi8(_H_min_val, _mm256_broadcastsi128_si256(_Shuf_bytes))); + return _H_min_val; + } + + static __m256i _H_min(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epi8(_Val1, _Val2); }); + } + + static __m256i _H_max(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epi8(_Val1, _Val2); }); + } + + static __m256i _H_min_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epu8(_Val1, _Val2); }); + } + + static __m256i _H_max_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epu8(_Val1, _Val2); }); + } + + static _Signed_t _Get_any(const __m256i _Cur) noexcept { + return static_cast<_Signed_t>(_mm256_cvtsi256_si32(_Cur)); + } + + static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { + const uint32_t _Part = _mm256_cvtsi256_si32( + _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); + return static_cast<_Unsigned_t>(_Part >> ((_H_pos & 0x3) << 3)); + } + + static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi8(_First, _Second); + } + + static __m256i _Cmp_gt(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpgt_epi8(_First, _Second); + } + + static __m256i _Cmp_eq_idx(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi8(_First, _Second); + } + + static __m256i _Min(const __m256i _First, const __m256i _Second, __m256i = _mm256_undefined_si256()) noexcept { + return _mm256_min_epi8(_First, _Second); + } + + static __m256i _Max(const __m256i _First, const __m256i _Second, __m256i = _mm256_undefined_si256()) noexcept { + return _mm256_max_epi8(_First, _Second); + } + + static __m256i _Min_u(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_min_epu8(_First, _Second); + } + + static __m256i _Max_u(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_max_epu8(_First, _Second); + } + + static __m256i _Mask_cast(const __m256i _Mask) noexcept { return _Mask; } }; @@ -727,7 +817,7 @@ namespace { return _mm_sub_epi16(_Val, _mm_load_si128(reinterpret_cast(_Sign_corrections[_Sign]))); } - static __m128i _Inc(__m128i _Idx) noexcept { + static __m128i _Inc(const __m128i _Idx) noexcept { return _mm_add_epi16(_Idx, _mm_set1_epi16(1)); } @@ -797,7 +887,96 @@ namespace { return _mm_max_epu16(_First, _Second); } - static __m128i _Mask_cast(__m128i _Mask) noexcept { + static __m128i _Mask_cast(const __m128i _Mask) noexcept { + return _Mask; + } + }; + + struct _Minmax_traits_2_avx : _Minmax_traits_2_base, _Minmax_traits_avx_base { + static __m256i _Load(const void* _Src) noexcept { + return _mm256_loadu_si256(reinterpret_cast(_Src)); + } + + static __m256i _Sign_correction(const __m256i _Val, const bool _Sign) noexcept { + alignas(32) static constexpr _Unsigned_t _Sign_corrections[2][16] = {0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, {}}; + return _mm256_sub_epi16( + _Val, _mm256_load_si256(reinterpret_cast(_Sign_corrections[_Sign]))); + } + + static __m256i _Inc(const __m256i _Idx) noexcept { + return _mm256_add_epi16(_Idx, _mm256_set1_epi16(1)); + } + + template + static __m256i _H_func(const __m256i _Cur, _Fn _Funct) noexcept { + const __m128i _Shuf_words = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + + __m256i _H_min_val = _Cur; + _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi8(_H_min_val, _mm256_broadcastsi128_si256(_Shuf_words))); + return _H_min_val; + } + + static __m256i _H_min(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epi16(_Val1, _Val2); }); + } + + static __m256i _H_max(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epi16(_Val1, _Val2); }); + } + + static __m256i _H_min_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epu16(_Val1, _Val2); }); + } + + static __m256i _H_max_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epu16(_Val1, _Val2); }); + } + + static _Signed_t _Get_any(const __m256i _Cur) noexcept { + return static_cast<_Signed_t>(_mm256_cvtsi256_si32(_Cur)); + } + + static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { + static constexpr _Unsigned_t _Shuf[] = {0x0100, 0x0302, 0x0504, 0x0706, 0x0908, 0x0B0A, 0x0D0C, 0x0F0E}; + + const uint32_t _Part = _mm256_cvtsi256_si32( + _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); + return static_cast<_Unsigned_t>(_Part >> ((_H_pos & 0x2) << 3)); + } + + static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi16(_First, _Second); + } + + static __m256i _Cmp_gt(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpgt_epi16(_First, _Second); + } + + static __m256i _Cmp_eq_idx(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi16(_First, _Second); + } + + static __m256i _Min(const __m256i _First, const __m256i _Second, __m256i = _mm256_undefined_si256()) noexcept { + return _mm256_min_epi16(_First, _Second); + } + + static __m256i _Max(const __m256i _First, const __m256i _Second, __m256i = _mm256_undefined_si256()) noexcept { + return _mm256_max_epi16(_First, _Second); + } + + static __m256i _Min_u(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_min_epu16(_First, _Second); + } + + static __m256i _Max_u(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_max_epu16(_First, _Second); + } + + static __m256i _Mask_cast(const __m256i _Mask) noexcept { return _Mask; } }; @@ -839,7 +1018,7 @@ namespace { return _mm_sub_epi32(_Val, _mm_load_si128(reinterpret_cast(_Sign_corrections[_Sign]))); } - static __m128i _Inc(__m128i _Idx) noexcept { + static __m128i _Inc(const __m128i _Idx) noexcept { return _mm_add_epi32(_Idx, _mm_set1_epi32(1)); } @@ -905,7 +1084,90 @@ namespace { return _mm_max_epu32(_First, _Second); } - static __m128i _Mask_cast(__m128i _Mask) noexcept { + static __m128i _Mask_cast(const __m128i _Mask) noexcept { + return _Mask; + } + }; + + struct _Minmax_traits_4_avx : _Minmax_traits_4_base, _Minmax_traits_avx_base { + static __m256i _Load(const void* _Src) noexcept { + return _mm256_loadu_si256(reinterpret_cast(_Src)); + } + + static __m256i _Sign_correction(const __m256i _Val, const bool _Sign) noexcept { + alignas(32) static constexpr _Unsigned_t _Sign_corrections[2][8] = {0x8000'0000UL, 0x8000'0000UL, + 0x8000'0000UL, 0x8000'0000UL, 0x8000'0000UL, 0x8000'0000UL, 0x8000'0000UL, 0x8000'0000UL, {}}; + return _mm256_sub_epi32( + _Val, _mm256_load_si256(reinterpret_cast(_Sign_corrections[_Sign]))); + } + + static __m256i _Inc(const __m256i _Idx) noexcept { + return _mm256_add_epi32(_Idx, _mm256_set1_epi32(1)); + } + + template + static __m256i _H_func(const __m256i _Cur, _Fn _Funct) noexcept { + __m256i _H_min_val = _Cur; + _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); + return _H_min_val; + } + + static __m256i _H_min(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epi32(_Val1, _Val2); }); + } + + static __m256i _H_max(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epi32(_Val1, _Val2); }); + } + + static __m256i _H_min_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epu32(_Val1, _Val2); }); + } + + static __m256i _H_max_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epu32(_Val1, _Val2); }); + } + + static _Signed_t _Get_any(const __m256i _Cur) noexcept { + return static_cast<_Signed_t>(_mm256_cvtsi256_si32(_Cur)); + } + + static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { + return _mm256_cvtsi256_si32( + _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); + } + + static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi32(_First, _Second); + } + + static __m256i _Cmp_gt(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpgt_epi32(_First, _Second); + } + + static __m256i _Cmp_eq_idx(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi32(_First, _Second); + } + + static __m256i _Min(const __m256i _First, const __m256i _Second, __m256i = _mm256_undefined_si256()) noexcept { + return _mm256_min_epi32(_First, _Second); + } + + static __m256i _Max(const __m256i _First, const __m256i _Second, __m256i = _mm256_undefined_si256()) noexcept { + return _mm256_max_epi32(_First, _Second); + } + + static __m256i _Min_u(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_min_epu32(_First, _Second); + } + + static __m256i _Max_u(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_max_epu32(_First, _Second); + } + + static __m256i _Mask_cast(const __m256i _Mask) noexcept { return _Mask; } }; @@ -942,7 +1204,7 @@ namespace { return _mm_sub_epi64(_Val, _mm_load_si128(reinterpret_cast(_Sign_corrections[_Sign]))); } - static __m128i _Inc(__m128i _Idx) noexcept { + static __m128i _Inc(const __m128i _Idx) noexcept { return _mm_add_epi64(_Idx, _mm_set1_epi64x(1)); } @@ -1016,7 +1278,109 @@ namespace { return _mm_blendv_epi8(_First, _Second, _Cmp_gt(_Second, _First)); } - static __m128i _Mask_cast(__m128i _Mask) noexcept { + static __m128i _Mask_cast(const __m128i _Mask) noexcept { + return _Mask; + } + }; + + struct _Minmax_traits_8_avx : _Minmax_traits_8_base, _Minmax_traits_avx_base { + static __m256i _Load(const void* _Src) noexcept { + return _mm256_loadu_si256(reinterpret_cast(_Src)); + } + + static __m256i _Sign_correction(const __m256i _Val, const bool _Sign) noexcept { + alignas(32) static constexpr _Unsigned_t _Sign_corrections[2][4] = {0x8000'0000'0000'0000ULL, + 0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL, 0x8000'0000'0000'0000ULL, {}}; + return _mm256_sub_epi64( + _Val, _mm256_load_si256(reinterpret_cast(_Sign_corrections[_Sign]))); + } + + static __m256i _Inc(const __m256i _Idx) noexcept { + return _mm256_add_epi64(_Idx, _mm256_set1_epi64x(1)); + } + + template + static __m256i _H_func(const __m256i _Cur, _Fn _Funct) noexcept { + alignas(32) _Signed_t _Array[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(_Array), _Cur); + + _Signed_t _H_min_v = _Array[0]; + + if (_Funct(_Array[1], _H_min_v)) { + _H_min_v = _Array[1]; + } + if (_Funct(_Array[2], _H_min_v)) { + _H_min_v = _Array[2]; + } + if (_Funct(_Array[3], _H_min_v)) { + _H_min_v = _Array[3]; + } + + return _mm256_set1_epi64x(_H_min_v); + } + + static __m256i _H_min(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](_Signed_t _Lhs, _Signed_t _Rhs) { return _Lhs < _Rhs; }); + } + + static __m256i _H_max(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](_Signed_t _Lhs, _Signed_t _Rhs) { return _Lhs > _Rhs; }); + } + + static __m256i _H_min_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](_Unsigned_t _Lhs, _Unsigned_t _Rhs) { return _Lhs < _Rhs; }); + } + + static __m256i _H_max_u(const __m256i _Cur) noexcept { + return _H_func(_Cur, [](_Unsigned_t _Lhs, _Unsigned_t _Rhs) { return _Lhs > _Rhs; }); + } + + static _Signed_t _Get_any(const __m256i _Cur) noexcept { + __m128i _Cur0 = _mm256_castsi256_si128(_Cur); +#ifdef _M_IX86 + return static_cast<_Signed_t>( + (static_cast<_Unsigned_t>(static_cast(_mm_extract_epi32(_Cur0, 1))) << 32) + | static_cast<_Unsigned_t>(static_cast(_mm_cvtsi128_si32(_Cur0)))); +#else // ^^^ x86 / x64 vvv + return static_cast<_Signed_t>(_mm_cvtsi128_si64(_Cur0)); +#endif // ^^^ x64 ^^^ + } + + static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { + _Unsigned_t _Array[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(&_Array), _Idx); + return _Array[_H_pos >> 3]; + } + + static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi64(_First, _Second); + } + + static __m256i _Cmp_gt(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpgt_epi64(_First, _Second); + } + + static __m256i _Cmp_eq_idx(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi64(_First, _Second); + } + + static __m256i _Min(const __m256i _First, const __m256i _Second, const __m256i _Mask) noexcept { + return _mm256_blendv_epi8(_First, _Second, _Mask); + } + + static __m256i _Max(const __m256i _First, const __m256i _Second, const __m256i _Mask) noexcept { + return _mm256_blendv_epi8(_First, _Second, _Mask); + } + + static __m256i _Min(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_blendv_epi8(_First, _Second, _Cmp_gt(_First, _Second)); + } + + static __m256i _Max(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_blendv_epi8(_First, _Second, _Cmp_gt(_Second, _First)); + } + + static __m256i _Mask_cast(const __m256i _Mask) noexcept { return _Mask; } }; @@ -1056,7 +1420,7 @@ namespace { return _Val; } - static __m128i _Inc(__m128i _Idx) noexcept { + static __m128i _Inc(const __m128i _Idx) noexcept { return _mm_add_epi32(_Idx, _mm_set1_epi32(1)); } @@ -1122,10 +1486,91 @@ namespace { return _mm_max_ps(_First, _Second); } - static __m128i _Mask_cast(__m128 _Mask) noexcept { + static __m128i _Mask_cast(const __m128 _Mask) noexcept { return _mm_castps_si128(_Mask); } }; + + struct _Minmax_traits_f_avx : _Minmax_traits_f_base, _Minmax_traits_avx_base { + static __m256 _Load(const void* _Src) noexcept { + return _mm256_loadu_ps(reinterpret_cast(_Src)); + } + + static __m256 _Sign_correction(const __m256 _Val, bool) noexcept { + return _Val; + } + + static __m256i _Inc(const __m256i _Idx) noexcept { + return _mm256_add_epi32(_Idx, _mm256_set1_epi32(1)); + } + + template + static __m256 _H_func(const __m256 _Cur, _Fn _Funct) noexcept { + __m256 _H_min_val = _Cur; + _H_min_val = _Funct(_H_min_val, _mm256_permute2f128_ps(_H_min_val, _H_min_val, _MM_SHUFFLE2(0, 1))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_ps(_H_min_val, _H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_ps(_H_min_val, _H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); + return _H_min_val; + } + + template + static __m256i _H_func_u(const __m256i _Cur, _Fn _Funct) noexcept { + __m256i _H_min_val = _Cur; + _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); + return _H_min_val; + } + + static __m256 _H_min(const __m256 _Cur) noexcept { + return _H_func(_Cur, [](__m256 _Val1, __m256 _Val2) { return _mm256_min_ps(_Val1, _Val2); }); + } + + static __m256 _H_max(const __m256 _Cur) noexcept { + return _H_func(_Cur, [](__m256 _Val1, __m256 _Val2) { return _mm256_max_ps(_Val1, _Val2); }); + } + + static __m256i _H_min_u(const __m256i _Cur) noexcept { + return _H_func_u(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epu32(_Val1, _Val2); }); + } + + static __m256i _H_max_u(const __m256i _Cur) noexcept { + return _H_func_u(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epu32(_Val1, _Val2); }); + } + + static float _Get_any(const __m256 _Cur) noexcept { + return _mm256_cvtss_f32(_Cur); + } + + static uint32_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { + return _mm256_cvtsi256_si32( + _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); + } + + static __m256 _Cmp_eq(const __m256 _First, const __m256 _Second) noexcept { + return _mm256_cmp_ps(_First, _Second, _CMP_EQ_OQ); + } + + static __m256 _Cmp_gt(const __m256 _First, const __m256 _Second) noexcept { + return _mm256_cmp_ps(_First, _Second, _CMP_GT_OQ); + } + + static __m256i _Cmp_eq_idx(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi32(_First, _Second); + } + + static __m256 _Min(const __m256 _First, const __m256 _Second, __m256 = _mm256_undefined_ps()) noexcept { + return _mm256_min_ps(_First, _Second); + } + + static __m256 _Max(const __m256 _First, const __m256 _Second, __m256 = _mm256_undefined_ps()) noexcept { + return _mm256_max_ps(_First, _Second); + } + + static __m256i _Mask_cast(const __m256 _Mask) noexcept { + return _mm256_castps_si256(_Mask); + } + }; #endif // !_M_ARM64EC struct _Minmax_traits_d_base { @@ -1157,7 +1602,7 @@ namespace { return _Val; } - static __m128i _Inc(__m128i _Idx) noexcept { + static __m128i _Inc(const __m128i _Idx) noexcept { return _mm_add_epi64(_Idx, _mm_set1_epi64x(1)); } @@ -1232,16 +1677,119 @@ namespace { return _mm_max_pd(_First, _Second); } - static __m128i _Mask_cast(__m128d _Mask) noexcept { + static __m128i _Mask_cast(const __m128d _Mask) noexcept { return _mm_castpd_si128(_Mask); } }; + + struct _Minmax_traits_d_avx : _Minmax_traits_d_base, _Minmax_traits_avx_base { + static __m256d _Load(const void* _Src) noexcept { + return _mm256_loadu_pd(reinterpret_cast(_Src)); + } + + static __m256d _Sign_correction(const __m256d _Val, bool) noexcept { + return _Val; + } + + static __m256i _Inc(const __m256i _Idx) noexcept { + return _mm256_add_epi64(_Idx, _mm256_set1_epi64x(1)); + } + + template + static __m256d _H_func(const __m256d _Cur, _Fn _Funct) noexcept { + __m256d _H_min_val = _Cur; + _H_min_val = _Funct(_H_min_val, _mm256_permute4x64_pd(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_shuffle_pd(_H_min_val, _H_min_val, 0b0101)); + return _H_min_val; + } + + template + static __m256i _H_func_u(const __m256i _Cur, _Fn _Funct) noexcept { + alignas(32) uint64_t _Array[4]; + _mm256_store_si256(reinterpret_cast<__m256i*>(_Array), _Cur); + + uint64_t _H_min_v = _Array[0]; + + if (_Funct(_Array[1], _H_min_v)) { + _H_min_v = _Array[1]; + } + if (_Funct(_Array[2], _H_min_v)) { + _H_min_v = _Array[2]; + } + if (_Funct(_Array[3], _H_min_v)) { + _H_min_v = _Array[3]; + } + + return _mm256_set1_epi64x(_H_min_v); + } + + static __m256d _H_min(const __m256d _Cur) noexcept { + return _H_func(_Cur, [](__m256d _Val1, __m256d _Val2) { return _mm256_min_pd(_Val1, _Val2); }); + } + + static __m256d _H_max(const __m256d _Cur) noexcept { + return _H_func(_Cur, [](__m256d _Val1, __m256d _Val2) { return _mm256_max_pd(_Val1, _Val2); }); + } + + static __m256i _H_min_u(const __m256i _Cur) noexcept { + return _H_func_u(_Cur, [](uint64_t _Lhs, uint64_t _Rhs) { return _Lhs < _Rhs; }); + } + + static __m256i _H_max_u(const __m256i _Cur) noexcept { + return _H_func_u(_Cur, [](uint64_t _Lhs, uint64_t _Rhs) { return _Lhs > _Rhs; }); + } + static double _Get_any(const __m256d _Cur) noexcept { + return _mm256_cvtsd_f64(_Cur); + } + + static uint64_t _Get_any_u(const __m256i _Cur) noexcept { + __m128i _Cur0 = _mm256_castsi256_si128(_Cur); +#ifdef _M_IX86 + return static_cast( + (static_cast(static_cast(_mm_extract_epi32(_Cur0, 1))) << 32) + | static_cast(static_cast(_mm_cvtsi128_si32(_Cur0)))); +#else // ^^^ x86 / x64 vvv + return static_cast(_mm_cvtsi128_si64(_Cur0)); +#endif // ^^^ x64 ^^^ + } + + static uint64_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { + uint64_t _Array[4]; + _mm256_storeu_si256(reinterpret_cast<__m256i*>(&_Array), _Idx); + return _Array[_H_pos >> 3]; + } + + static __m256d _Cmp_eq(const __m256d _First, const __m256d _Second) noexcept { + return _mm256_cmp_pd(_First, _Second, _CMP_EQ_OQ); + } + + static __m256d _Cmp_gt(const __m256d _First, const __m256d _Second) noexcept { + return _mm256_cmp_pd(_First, _Second, _CMP_GT_OQ); + } + + static __m256i _Cmp_eq_idx(const __m256i _First, const __m256i _Second) noexcept { + return _mm256_cmpeq_epi64(_First, _Second); + } + + static __m256d _Min(const __m256d _First, const __m256d _Second, __m256d = _mm256_undefined_pd()) noexcept { + return _mm256_min_pd(_First, _Second); + } + + static __m256d _Max(const __m256d _First, const __m256d _Second, __m256d = _mm256_undefined_pd()) noexcept { + return _mm256_max_pd(_First, _Second); + } + + static __m256i _Mask_cast(const __m256d _Mask) noexcept { + return _mm256_castpd_si256(_Mask); + } + }; #endif // !_M_ARM64EC struct _Minmax_traits_1 { using _Scalar = _Minmax_traits_1_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_1_sse; + using _Avx = _Minmax_traits_1_avx; #endif // !_M_ARM64EC }; @@ -1249,6 +1797,7 @@ namespace { using _Scalar = _Minmax_traits_2_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_2_sse; + using _Avx = _Minmax_traits_2_avx; #endif // !_M_ARM64EC }; @@ -1256,6 +1805,8 @@ namespace { using _Scalar = _Minmax_traits_4_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_4_sse; + using _Avx = _Minmax_traits_4_avx; + #endif // !_M_ARM64EC }; @@ -1263,6 +1814,7 @@ namespace { using _Scalar = _Minmax_traits_8_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_8_sse; + using _Avx = _Minmax_traits_8_avx; #endif // !_M_ARM64EC }; @@ -1270,6 +1822,7 @@ namespace { using _Scalar = _Minmax_traits_f_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_f_sse; + using _Avx = _Minmax_traits_f_avx; #endif // !_M_ARM64EC }; @@ -1277,6 +1830,7 @@ namespace { using _Scalar = _Minmax_traits_d_scalar; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_d_sse; + using _Avx = _Minmax_traits_d_avx; #endif // !_M_ARM64EC }; @@ -1507,6 +2061,9 @@ namespace { template <_Min_max_mode _Mode, class _Traits> auto __std_minmax_element_disp(const void* _First, const void* const _Last, const bool _Sign) noexcept { #ifndef _M_ARM64EC + if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + return __std_minmax_element_impl<_Mode, typename _Traits::_Avx>(_First, _Last, _Sign); + } if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { return __std_minmax_element_impl<_Mode, typename _Traits::_Sse>(_First, _Last, _Sign); } @@ -1654,6 +2211,9 @@ namespace { template <_Min_max_mode _Mode, class _Traits, bool _Sign> auto __stdcall __std_minmax_disp(const void* _First, const void* const _Last) noexcept { #ifndef _M_ARM64EC + if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + return __std_minmax_impl<_Mode, typename _Traits::_Avx, _Sign>(_First, _Last); + } if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { return __std_minmax_impl<_Mode, typename _Traits::_Sse, _Sign>(_First, _Last); } From a21d2dfbafc755eabf4c257d390cd06d7d687ac7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 09:38:04 +0300 Subject: [PATCH 05/23] float indices reuse integer indices --- stl/src/vector_algorithms.cpp | 98 +++++------------------------------ 1 file changed, 12 insertions(+), 86 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 1a62f2fc2fa..dfc0185c081 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1432,14 +1432,6 @@ namespace { return _H_min_val; } - template - static __m128i _H_func_u(const __m128i _Cur, _Fn _Funct) noexcept { - __m128i _H_min_val = _Cur; - _H_min_val = _Funct(_H_min_val, _mm_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); - _H_min_val = _Funct(_H_min_val, _mm_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); - return _H_min_val; - } - static __m128 _H_min(const __m128 _Cur) noexcept { return _H_func(_Cur, [](__m128 _Val1, __m128 _Val2) { return _mm_min_ps(_Val1, _Val2); }); } @@ -1449,11 +1441,11 @@ namespace { } static __m128i _H_min_u(const __m128i _Cur) noexcept { - return _H_func_u(_Cur, [](__m128i _Val1, __m128i _Val2) { return _mm_min_epu32(_Val1, _Val2); }); + return _Minmax_traits_4_sse::_H_min_u(_Cur); } static __m128i _H_max_u(const __m128i _Cur) noexcept { - return _H_func_u(_Cur, [](__m128i _Val1, __m128i _Val2) { return _mm_max_epu32(_Val1, _Val2); }); + return _Minmax_traits_4_sse::_H_max_u(_Cur); } static float _Get_any(const __m128 _Cur) noexcept { @@ -1461,9 +1453,7 @@ namespace { } static uint32_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - uint32_t _Array[4]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(&_Array), _Idx); - return _Array[_H_pos >> 2]; + return _Minmax_traits_4_sse::_Get_v_pos(_Idx, _H_pos); } static __m128 _Cmp_eq(const __m128 _First, const __m128 _Second) noexcept { @@ -1513,15 +1503,6 @@ namespace { return _H_min_val; } - template - static __m256i _H_func_u(const __m256i _Cur, _Fn _Funct) noexcept { - __m256i _H_min_val = _Cur; - _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); - _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); - _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); - return _H_min_val; - } - static __m256 _H_min(const __m256 _Cur) noexcept { return _H_func(_Cur, [](__m256 _Val1, __m256 _Val2) { return _mm256_min_ps(_Val1, _Val2); }); } @@ -1531,11 +1512,11 @@ namespace { } static __m256i _H_min_u(const __m256i _Cur) noexcept { - return _H_func_u(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_min_epu32(_Val1, _Val2); }); + return _Minmax_traits_4_avx::_H_min_u(_Cur); } static __m256i _H_max_u(const __m256i _Cur) noexcept { - return _H_func_u(_Cur, [](__m256i _Val1, __m256i _Val2) { return _mm256_max_epu32(_Val1, _Val2); }); + return _Minmax_traits_4_avx::_H_max_u(_Cur); } static float _Get_any(const __m256 _Cur) noexcept { @@ -1543,8 +1524,7 @@ namespace { } static uint32_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - return _mm256_cvtsi256_si32( - _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); + return _Minmax_traits_4_avx::_Get_v_pos(_Idx, _H_pos); } static __m256 _Cmp_eq(const __m256 _First, const __m256 _Second) noexcept { @@ -1613,16 +1593,6 @@ namespace { return _H_min_val; } - template - static __m128i _H_func_u(const __m128i _Cur, _Fn _Funct) noexcept { - uint64_t _H_min_a = _Get_any_u(_Cur); - uint64_t _H_min_b = _Get_any_u(_mm_bsrli_si128(_Cur, 8)); - if (_Funct(_H_min_b, _H_min_a)) { - _H_min_a = _H_min_b; - } - return _mm_set1_epi64x(_H_min_a); - } - static __m128d _H_min(const __m128d _Cur) noexcept { return _H_func(_Cur, [](__m128d _Val1, __m128d _Val2) { return _mm_min_pd(_Val1, _Val2); }); } @@ -1632,29 +1602,18 @@ namespace { } static __m128i _H_min_u(const __m128i _Cur) noexcept { - return _H_func_u(_Cur, [](uint64_t _Lhs, uint64_t _Rhs) { return _Lhs < _Rhs; }); + return _Minmax_traits_8_sse::_H_min_u(_Cur); } static __m128i _H_max_u(const __m128i _Cur) noexcept { - return _H_func_u(_Cur, [](uint64_t _Lhs, uint64_t _Rhs) { return _Lhs > _Rhs; }); + return _Minmax_traits_8_sse::_H_max_u(_Cur); } static double _Get_any(const __m128d _Cur) noexcept { return _mm_cvtsd_f64(_Cur); } - static uint64_t _Get_any_u(const __m128i _Cur) noexcept { -#ifdef _M_IX86 - return (static_cast(static_cast(_mm_extract_epi32(_Cur, 1))) << 32) - | static_cast(static_cast(_mm_cvtsi128_si32(_Cur))); -#else // ^^^ x86 / x64 vvv - return static_cast(_mm_cvtsi128_si64(_Cur)); -#endif // ^^^ x64 ^^^ - } - static uint64_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - uint64_t _Array[2]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(&_Array), _Idx); - return _Array[_H_pos >> 3]; + return _Minmax_traits_8_sse::_Get_v_pos(_Idx, _H_pos); } static __m128d _Cmp_eq(const __m128d _First, const __m128d _Second) noexcept { @@ -1703,26 +1662,6 @@ namespace { return _H_min_val; } - template - static __m256i _H_func_u(const __m256i _Cur, _Fn _Funct) noexcept { - alignas(32) uint64_t _Array[4]; - _mm256_store_si256(reinterpret_cast<__m256i*>(_Array), _Cur); - - uint64_t _H_min_v = _Array[0]; - - if (_Funct(_Array[1], _H_min_v)) { - _H_min_v = _Array[1]; - } - if (_Funct(_Array[2], _H_min_v)) { - _H_min_v = _Array[2]; - } - if (_Funct(_Array[3], _H_min_v)) { - _H_min_v = _Array[3]; - } - - return _mm256_set1_epi64x(_H_min_v); - } - static __m256d _H_min(const __m256d _Cur) noexcept { return _H_func(_Cur, [](__m256d _Val1, __m256d _Val2) { return _mm256_min_pd(_Val1, _Val2); }); } @@ -1732,31 +1671,18 @@ namespace { } static __m256i _H_min_u(const __m256i _Cur) noexcept { - return _H_func_u(_Cur, [](uint64_t _Lhs, uint64_t _Rhs) { return _Lhs < _Rhs; }); + return _Minmax_traits_8_avx::_H_min_u(_Cur); } static __m256i _H_max_u(const __m256i _Cur) noexcept { - return _H_func_u(_Cur, [](uint64_t _Lhs, uint64_t _Rhs) { return _Lhs > _Rhs; }); + return _Minmax_traits_8_avx::_H_max_u(_Cur); } static double _Get_any(const __m256d _Cur) noexcept { return _mm256_cvtsd_f64(_Cur); } - static uint64_t _Get_any_u(const __m256i _Cur) noexcept { - __m128i _Cur0 = _mm256_castsi256_si128(_Cur); -#ifdef _M_IX86 - return static_cast( - (static_cast(static_cast(_mm_extract_epi32(_Cur0, 1))) << 32) - | static_cast(static_cast(_mm_cvtsi128_si32(_Cur0)))); -#else // ^^^ x86 / x64 vvv - return static_cast(_mm_cvtsi128_si64(_Cur0)); -#endif // ^^^ x64 ^^^ - } - static uint64_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - uint64_t _Array[4]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(&_Array), _Idx); - return _Array[_H_pos >> 3]; + return _Minmax_traits_8_avx::_Get_v_pos(_Idx, _H_pos); } static __m256d _Cmp_eq(const __m256d _First, const __m256d _Second) noexcept { From 18a919f7bffa40a5210283b057e543f043bad432 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 11:40:27 +0300 Subject: [PATCH 06/23] template scalar traits --- stl/src/vector_algorithms.cpp | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index dfc0185c081..d10a32c256a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -540,7 +540,8 @@ namespace { _Mode_both = _Mode_min | _Mode_max, }; - struct _Minmax_traits_scalar_base { + template + struct _Minmax_traits_scalar : _Base { static constexpr bool _Vectorized = false; }; @@ -606,8 +607,6 @@ namespace { #endif //_M_ARM64EC }; - struct _Minmax_traits_1_scalar : _Minmax_traits_1_base, _Minmax_traits_scalar_base {}; - #ifndef _M_ARM64EC struct _Minmax_traits_1_sse : _Minmax_traits_1_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { @@ -803,8 +802,6 @@ namespace { #endif // !_M_ARM64EC }; - struct _Minmax_traits_2_scalar : _Minmax_traits_2_base, _Minmax_traits_scalar_base {}; - #ifndef _M_ARM64EC struct _Minmax_traits_2_sse : _Minmax_traits_2_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { @@ -1004,8 +1001,6 @@ namespace { #endif // !_M_ARM64EC }; - struct _Minmax_traits_4_scalar : _Minmax_traits_4_base, _Minmax_traits_scalar_base {}; - #ifndef _M_ARM64EC struct _Minmax_traits_4_sse : _Minmax_traits_4_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { @@ -1190,8 +1185,6 @@ namespace { #endif // !_M_ARM64EC }; - struct _Minmax_traits_8_scalar : _Minmax_traits_8_base, _Minmax_traits_scalar_base {}; - #ifndef _M_ARM64EC struct _Minmax_traits_8_sse : _Minmax_traits_8_base, _Minmax_traits_sse_base { static __m128i _Load(const void* _Src) noexcept { @@ -1408,8 +1401,6 @@ namespace { #endif // !_M_ARM64EC }; - struct _Minmax_traits_f_scalar : _Minmax_traits_f_base, _Minmax_traits_scalar_base {}; - #ifndef _M_ARM64EC struct _Minmax_traits_f_sse : _Minmax_traits_f_base, _Minmax_traits_sse_base { static __m128 _Load(const void* _Src) noexcept { @@ -1570,8 +1561,6 @@ namespace { #endif // !_M_ARM64EC }; - struct _Minmax_traits_d_scalar : _Minmax_traits_d_base, _Minmax_traits_scalar_base {}; - #ifndef _M_ARM64EC struct _Minmax_traits_d_sse : _Minmax_traits_d_base, _Minmax_traits_sse_base { static __m128d _Load(const void* _Src) noexcept { @@ -1712,7 +1701,7 @@ namespace { #endif // !_M_ARM64EC struct _Minmax_traits_1 { - using _Scalar = _Minmax_traits_1_scalar; + using _Scalar = _Minmax_traits_scalar<_Minmax_traits_1_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_1_sse; using _Avx = _Minmax_traits_1_avx; @@ -1720,7 +1709,7 @@ namespace { }; struct _Minmax_traits_2 { - using _Scalar = _Minmax_traits_2_scalar; + using _Scalar = _Minmax_traits_scalar<_Minmax_traits_2_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_2_sse; using _Avx = _Minmax_traits_2_avx; @@ -1728,7 +1717,7 @@ namespace { }; struct _Minmax_traits_4 { - using _Scalar = _Minmax_traits_4_scalar; + using _Scalar = _Minmax_traits_scalar<_Minmax_traits_4_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_4_sse; using _Avx = _Minmax_traits_4_avx; @@ -1737,7 +1726,7 @@ namespace { }; struct _Minmax_traits_8 { - using _Scalar = _Minmax_traits_8_scalar; + using _Scalar = _Minmax_traits_scalar<_Minmax_traits_8_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_8_sse; using _Avx = _Minmax_traits_8_avx; @@ -1745,7 +1734,7 @@ namespace { }; struct _Minmax_traits_f { - using _Scalar = _Minmax_traits_f_scalar; + using _Scalar = _Minmax_traits_scalar<_Minmax_traits_f_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_f_sse; using _Avx = _Minmax_traits_f_avx; @@ -1753,7 +1742,7 @@ namespace { }; struct _Minmax_traits_d { - using _Scalar = _Minmax_traits_d_scalar; + using _Scalar = _Minmax_traits_scalar<_Minmax_traits_d_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_d_sse; using _Avx = _Minmax_traits_d_avx; From 0ec33d9ff6e8a741d3589d3870bc7028158d6c2f Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 13:47:51 +0300 Subject: [PATCH 07/23] Drop the fuse, it no longer works --- stl/src/vector_algorithms.cpp | 49 ++++++++++++++--------------------- 1 file changed, 20 insertions(+), 29 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index d10a32c256a..929ae7b59a8 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1749,10 +1749,6 @@ namespace { #endif // !_M_ARM64EC }; - // __std_minmax_element_impl and __std_minmax_element_disp have exactly the same signature - // as the extern "C" functions (__std_min_element_N, __std_max_element_N, __std_minmax_element_N), - // up to calling convention. - // This makes sure the template specialization can be tail called without filling in the params. template <_Min_max_mode _Mode, class _Traits> auto __std_minmax_element_impl(const void* _First, const void* const _Last, const bool _Sign) noexcept { _Min_max_element_t _Res = {_First, _First}; @@ -1986,11 +1982,8 @@ namespace { return __std_minmax_element_impl<_Mode, typename _Traits::_Scalar>(_First, _Last, _Sign); } - // __std_minmax_impl and __std_minmax_disp have exactly the same signature as the extern "C" functions - // (__std_min_Nn, __std_max_Nn, __std_minmax_Nn), up to calling convention. - // This makes sure the template specialization can be tail called without filling in the params. template <_Min_max_mode _Mode, class _Traits, bool _Sign> - auto __stdcall __std_minmax_impl(const void* _First, const void* const _Last) noexcept { + auto __std_minmax_impl(const void* _First, const void* const _Last) noexcept { using _Ty = std::conditional_t<_Sign, typename _Traits::_Signed_t, typename _Traits::_Unsigned_t>; _Ty _Cur_min_val; // initialized in both of the branches below @@ -2124,7 +2117,7 @@ namespace { } template <_Min_max_mode _Mode, class _Traits, bool _Sign> - auto __stdcall __std_minmax_disp(const void* _First, const void* const _Last) noexcept { + auto __std_minmax_disp(const void* _First, const void* const _Last) noexcept { #ifndef _M_ARM64EC if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { return __std_minmax_impl<_Mode, typename _Traits::_Avx, _Sign>(_First, _Last); @@ -2160,14 +2153,14 @@ const void* __stdcall __std_min_element_8( return __std_minmax_element_disp<_Mode_min, _Minmax_traits_8>(_First, _Last, _Signed); } -const void* __stdcall __std_min_element_f( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` - const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_disp<_Mode_min, _Minmax_traits_f>(_First, _Last, _Unused); +// TRANSITION, ABI: remove unused `bool` +const void* __stdcall __std_min_element_f(const void* const _First, const void* const _Last, const bool) noexcept { + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_f>(_First, _Last, false); } -const void* __stdcall __std_min_element_d( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` - const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_disp<_Mode_min, _Minmax_traits_d>(_First, _Last, _Unused); +// TRANSITION, ABI: remove unused `bool` +const void* __stdcall __std_min_element_d(const void* const _First, const void* const _Last, const bool) noexcept { + return __std_minmax_element_disp<_Mode_min, _Minmax_traits_d>(_First, _Last, false); } const void* __stdcall __std_max_element_1( @@ -2190,14 +2183,14 @@ const void* __stdcall __std_max_element_8( return __std_minmax_element_disp<_Mode_max, _Minmax_traits_8>(_First, _Last, _Signed); } -const void* __stdcall __std_max_element_f( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` - const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_disp<_Mode_max, _Minmax_traits_f>(_First, _Last, _Unused); +// TRANSITION, ABI: remove unused `bool` +const void* __stdcall __std_max_element_f(const void* const _First, const void* const _Last, bool) noexcept { + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_f>(_First, _Last, false); } -const void* __stdcall __std_max_element_d( // __std_minmax_element_disp's "signature" comment explains `bool _Unused` - const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_disp<_Mode_max, _Minmax_traits_d>(_First, _Last, _Unused); +// TRANSITION, ABI: remove unused `bool` +const void* __stdcall __std_max_element_d(const void* const _First, const void* const _Last, bool) noexcept { + return __std_minmax_element_disp<_Mode_max, _Minmax_traits_d>(_First, _Last, false); } _Min_max_element_t __stdcall __std_minmax_element_1( @@ -2220,16 +2213,14 @@ _Min_max_element_t __stdcall __std_minmax_element_8( return __std_minmax_element_disp<_Mode_both, _Minmax_traits_8>(_First, _Last, _Signed); } -// __std_minmax_element_disp's "signature" comment explains `bool _Unused` -_Min_max_element_t __stdcall __std_minmax_element_f( - const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_disp<_Mode_both, _Minmax_traits_f>(_First, _Last, _Unused); +// TRANSITION, ABI: remove unused `bool` +_Min_max_element_t __stdcall __std_minmax_element_f(const void* const _First, const void* const _Last, bool) noexcept { + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_f>(_First, _Last, false); } -// __std_minmax_element_disp's "signature" comment explains `bool _Unused` -_Min_max_element_t __stdcall __std_minmax_element_d( - const void* const _First, const void* const _Last, const bool _Unused) noexcept { - return __std_minmax_element_disp<_Mode_both, _Minmax_traits_d>(_First, _Last, _Unused); +// TRANSITION, ABI: remove unused `bool` +_Min_max_element_t __stdcall __std_minmax_element_d(const void* const _First, const void* const _Last, bool) noexcept { + return __std_minmax_element_disp<_Mode_both, _Minmax_traits_d>(_First, _Last, false); } __declspec(noalias) int8_t __stdcall __std_min_1i(const void* const _First, const void* const _Last) noexcept { From e05e57f009650fdff3d382de80aff03879d53c46 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 14:05:36 +0300 Subject: [PATCH 08/23] -newline --- stl/src/vector_algorithms.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 929ae7b59a8..2172e8230b8 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2128,7 +2128,6 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ return __std_minmax_impl<_Mode, typename _Traits::_Scalar, _Sign>(_First, _Last); } - } // unnamed namespace extern "C" { From e860b8bec934a09eb60ed1273c71eb7006a9525e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 14:06:17 +0300 Subject: [PATCH 09/23] -const unused --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 2172e8230b8..dd389366cf2 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -2153,12 +2153,12 @@ const void* __stdcall __std_min_element_8( } // TRANSITION, ABI: remove unused `bool` -const void* __stdcall __std_min_element_f(const void* const _First, const void* const _Last, const bool) noexcept { +const void* __stdcall __std_min_element_f(const void* const _First, const void* const _Last, bool) noexcept { return __std_minmax_element_disp<_Mode_min, _Minmax_traits_f>(_First, _Last, false); } // TRANSITION, ABI: remove unused `bool` -const void* __stdcall __std_min_element_d(const void* const _First, const void* const _Last, const bool) noexcept { +const void* __stdcall __std_min_element_d(const void* const _First, const void* const _Last, bool) noexcept { return __std_minmax_element_disp<_Mode_min, _Minmax_traits_d>(_First, _Last, false); } From f7d813b893f2312f2b9391908aa446d8b1fd6562 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 14:33:49 +0300 Subject: [PATCH 10/23] This works for a wrong reason --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index dd389366cf2..bb576e0652d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1488,7 +1488,7 @@ namespace { template static __m256 _H_func(const __m256 _Cur, _Fn _Funct) noexcept { __m256 _H_min_val = _Cur; - _H_min_val = _Funct(_H_min_val, _mm256_permute2f128_ps(_H_min_val, _H_min_val, _MM_SHUFFLE2(0, 1))); + _H_min_val = _Funct(_H_min_val, _mm256_permute2f128_ps(_H_min_val, _mm256_undefined_ps(), 0x01)); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_ps(_H_min_val, _H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_ps(_H_min_val, _H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); return _H_min_val; From 2bf593182d16fdd13fb45fb73bd30228918050b5 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 14:34:11 +0300 Subject: [PATCH 11/23] More reuse! --- stl/src/vector_algorithms.cpp | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index bb576e0652d..afd01c62d4c 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1329,14 +1329,7 @@ namespace { } static _Signed_t _Get_any(const __m256i _Cur) noexcept { - __m128i _Cur0 = _mm256_castsi256_si128(_Cur); -#ifdef _M_IX86 - return static_cast<_Signed_t>( - (static_cast<_Unsigned_t>(static_cast(_mm_extract_epi32(_Cur0, 1))) << 32) - | static_cast<_Unsigned_t>(static_cast(_mm_cvtsi128_si32(_Cur0)))); -#else // ^^^ x86 / x64 vvv - return static_cast<_Signed_t>(_mm_cvtsi128_si64(_Cur0)); -#endif // ^^^ x64 ^^^ + return _Minmax_traits_8_sse::_Get_any(_mm256_castsi256_si128(_Cur)); } static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { From 5f045b2dc2f5e5596d9200e18d54449bc50dbbcd Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Tue, 7 May 2024 19:20:09 +0300 Subject: [PATCH 12/23] Nearly forgot that one again! --- stl/src/vector_algorithms.cpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index afd01c62d4c..3fe28a18113 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -565,6 +565,8 @@ namespace { static unsigned long _Mask(const __m128i _Val) noexcept { return _mm_movemask_epi8(_Val); } + + static void _Exit_vectorized() noexcept {} }; struct _Minmax_traits_avx_base { @@ -587,6 +589,10 @@ namespace { static unsigned long _Mask(const __m256i _Val) noexcept { return _mm256_movemask_epi8(_Val); } + + static void _Exit_vectorized() noexcept { + _mm256_zeroupper(); + } }; struct _Minmax_traits_1_base { @@ -1921,6 +1927,8 @@ namespace { } } } + + _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 #endif // !_M_ARM64EC } @@ -2072,6 +2080,8 @@ namespace { break; } } + + _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 #endif // !_M_ARM64EC } else { _Cur_min_val = *reinterpret_cast(_First); From 53809952c243f9757af957649dfc84d54f1e81ce Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 8 May 2024 08:24:14 +0300 Subject: [PATCH 13/23] fix ARM64EC build --- stl/src/vector_algorithms.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 3fe28a18113..509131b309e 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -545,6 +545,7 @@ namespace { static constexpr bool _Vectorized = false; }; +#ifndef _M_ARM64EC struct _Minmax_traits_sse_base { static constexpr bool _Vectorized = true; static constexpr size_t _Vec_size = 16; @@ -594,6 +595,7 @@ namespace { _mm256_zeroupper(); } }; +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_1_base { static constexpr bool _Is_floating = false; From 6bee3f030c908a14dfd5d88ffd99079200b95902 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 8 May 2024 08:32:40 +0300 Subject: [PATCH 14/23] make preprocessor comments consistent for at least minmax --- stl/src/vector_algorithms.cpp | 40 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 509131b309e..48096b00c7b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -612,7 +612,7 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 256; -#endif //_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; #ifndef _M_ARM64EC @@ -790,7 +790,7 @@ namespace { return _Mask; } }; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_2_base { static constexpr bool _Is_floating = false; @@ -807,7 +807,7 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 65536; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; #ifndef _M_ARM64EC @@ -985,7 +985,7 @@ namespace { return _Mask; } }; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_4_base { static constexpr bool _Is_floating = false; @@ -1006,7 +1006,7 @@ namespace { static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 0x1'0000'0000ULL; #endif // ^^^ 64-bit ^^^ -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; #ifndef _M_ARM64EC @@ -1174,7 +1174,7 @@ namespace { return _Mask; } }; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_8_base { static constexpr bool _Is_floating = false; @@ -1190,7 +1190,7 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = false; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; #ifndef _M_ARM64EC @@ -1378,7 +1378,7 @@ namespace { return _Mask; } }; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_f_base { static constexpr bool _Is_floating = true; @@ -1399,7 +1399,7 @@ namespace { static constexpr bool _Has_portion_max = true; static constexpr size_t _Portion_max = 0x1'0000'0000ULL; #endif // ^^^ 64-bit ^^^ -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; #ifndef _M_ARM64EC @@ -1543,7 +1543,7 @@ namespace { return _mm256_castps_si256(_Mask); } }; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_d_base { static constexpr bool _Is_floating = true; @@ -1559,7 +1559,7 @@ namespace { #ifndef _M_ARM64EC static constexpr bool _Has_portion_max = false; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; #ifndef _M_ARM64EC @@ -1699,14 +1699,14 @@ namespace { return _mm256_castpd_si256(_Mask); } }; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) struct _Minmax_traits_1 { using _Scalar = _Minmax_traits_scalar<_Minmax_traits_1_base>; #ifndef _M_ARM64EC using _Sse = _Minmax_traits_1_sse; using _Avx = _Minmax_traits_1_avx; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; struct _Minmax_traits_2 { @@ -1714,7 +1714,7 @@ namespace { #ifndef _M_ARM64EC using _Sse = _Minmax_traits_2_sse; using _Avx = _Minmax_traits_2_avx; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; struct _Minmax_traits_4 { @@ -1723,7 +1723,7 @@ namespace { using _Sse = _Minmax_traits_4_sse; using _Avx = _Minmax_traits_4_avx; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; struct _Minmax_traits_8 { @@ -1731,7 +1731,7 @@ namespace { #ifndef _M_ARM64EC using _Sse = _Minmax_traits_8_sse; using _Avx = _Minmax_traits_8_avx; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; struct _Minmax_traits_f { @@ -1739,7 +1739,7 @@ namespace { #ifndef _M_ARM64EC using _Sse = _Minmax_traits_f_sse; using _Avx = _Minmax_traits_f_avx; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; struct _Minmax_traits_d { @@ -1747,7 +1747,7 @@ namespace { #ifndef _M_ARM64EC using _Sse = _Minmax_traits_d_sse; using _Avx = _Minmax_traits_d_avx; -#endif // !_M_ARM64EC +#endif // !defined(_M_ARM64EC) }; template <_Min_max_mode _Mode, class _Traits> @@ -1931,7 +1931,7 @@ namespace { } _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 -#endif // !_M_ARM64EC +#endif // ^^^ !defined(_M_ARM64EC) ^^^ } if constexpr (_Traits::_Is_floating) { @@ -2084,7 +2084,7 @@ namespace { } _Traits::_Exit_vectorized(); // TRANSITION, DevCom-10331414 -#endif // !_M_ARM64EC +#endif // ^^^ !defined(_M_ARM64EC) ^^^ } else { _Cur_min_val = *reinterpret_cast(_First); _Cur_max_val = *reinterpret_cast(_First); From 08e84f881ee1e65d0d53f7992cd270471dd47210 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 13 May 2024 09:14:31 +0300 Subject: [PATCH 15/23] Avoid extra variable --- stl/src/vector_algorithms.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 48096b00c7b..aeb70e0a62d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1832,9 +1832,8 @@ namespace { _Traits::_Cmp_eq(_H_min, _Cur_vals_min); // Mask of all elems eq to min unsigned long _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); // Indices of minimum elements or the greatest index if none - const auto _All_max = _Traits::_All_ones(); const auto _Idx_min_val = - _Traits::_Blend(_All_max, _Cur_idx_min, _Traits::_Mask_cast(_Eq_mask)); + _Traits::_Blend(_Traits::_All_ones(), _Cur_idx_min, _Traits::_Mask_cast(_Eq_mask)); auto _Idx_min = _Traits::_H_min_u(_Idx_min_val); // The smallest indices // Select the smallest vertical indices from the smallest element mask _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_min, _Idx_min_val)); @@ -1879,9 +1878,8 @@ namespace { } else { // Looking for the first occurrence of maximum // Indices of maximum elements or the greatest index if none - const auto _All_max = _Traits::_All_ones(); const auto _Idx_max_val = - _Traits::_Blend(_All_max, _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); + _Traits::_Blend(_All_max, _Traits::_All_ones(), _Traits::_Mask_cast(_Eq_mask)); const auto _Idx_max = _Traits::_H_min_u(_Idx_max_val); // The smallest indices // Select the smallest vertical indices from the largest element mask _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); From 7aa4da043bea8437ebe4888d0b28fd9705580e25 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Mon, 13 May 2024 09:26:50 +0300 Subject: [PATCH 16/23] fix up previous change --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index aeb70e0a62d..e8a91544556 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1879,7 +1879,7 @@ namespace { // Looking for the first occurrence of maximum // Indices of maximum elements or the greatest index if none const auto _Idx_max_val = - _Traits::_Blend(_All_max, _Traits::_All_ones(), _Traits::_Mask_cast(_Eq_mask)); + _Traits::_Blend(_Traits::_All_ones(), _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); const auto _Idx_max = _Traits::_H_min_u(_Idx_max_val); // The smallest indices // Select the smallest vertical indices from the largest element mask _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); From 8a024d6d29cbda3796cf59b69c3051ab9d872da2 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 7 Jun 2024 14:10:19 +0300 Subject: [PATCH 17/23] right AVX2 vpermq --- stl/src/vector_algorithms.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 339512c759f..5fd552ef624 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -724,7 +724,7 @@ namespace { const __m128i _Shuf_words = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); __m256i _H_min_val = _Cur; - _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_permute4x64_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi8(_H_min_val, _mm256_broadcastsi128_si256(_Shuf_words))); @@ -918,7 +918,7 @@ namespace { const __m128i _Shuf_words = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); __m256i _H_min_val = _Cur; - _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_permute4x64_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi8(_H_min_val, _mm256_broadcastsi128_si256(_Shuf_words))); @@ -1111,7 +1111,7 @@ namespace { template static __m256i _H_func(const __m256i _Cur, _Fn _Funct) noexcept { __m256i _H_min_val = _Cur; - _H_min_val = _Funct(_H_min_val, _mm256_permutex_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); + _H_min_val = _Funct(_H_min_val, _mm256_permute4x64_epi64(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(1, 0, 3, 2))); _H_min_val = _Funct(_H_min_val, _mm256_shuffle_epi32(_H_min_val, _MM_SHUFFLE(2, 3, 0, 1))); return _H_min_val; From 6db0f490656714289ddcbcb4e5cd42588da5f896 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 7 Jun 2024 14:10:32 +0300 Subject: [PATCH 18/23] -newline --- stl/src/vector_algorithms.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 5fd552ef624..b1a6bd97331 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1718,7 +1718,6 @@ namespace { #ifndef _M_ARM64EC using _Sse = _Minmax_traits_4_sse; using _Avx = _Minmax_traits_4_avx; - #endif // !defined(_M_ARM64EC) }; From fbbad86705fe57f64f565835c77590c2e5ed7bf6 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 7 Jun 2024 14:11:36 +0300 Subject: [PATCH 19/23] unrolled and unchained --- stl/src/vector_algorithms.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b1a6bd97331..5a15dc4d55a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1306,9 +1306,11 @@ namespace { if (_Funct(_Array[1], _H_min_v)) { _H_min_v = _Array[1]; } + if (_Funct(_Array[2], _H_min_v)) { _H_min_v = _Array[2]; } + if (_Funct(_Array[3], _H_min_v)) { _H_min_v = _Array[3]; } From 97e670e43e3cc521461c3c9fdc15a4207caae909 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 7 Jun 2024 14:12:15 +0300 Subject: [PATCH 20/23] unused constant --- stl/src/vector_algorithms.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 5a15dc4d55a..802c29ac19c 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -946,8 +946,6 @@ namespace { } static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - static constexpr _Unsigned_t _Shuf[] = {0x0100, 0x0302, 0x0504, 0x0706, 0x0908, 0x0B0A, 0x0D0C, 0x0F0E}; - const uint32_t _Part = _mm256_cvtsi256_si32( _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); return static_cast<_Unsigned_t>(_Part >> ((_H_pos & 0x2) << 3)); From 567eb8c38966a4e0314d2dfa2d33c0cf5934afc8 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 7 Jun 2024 14:12:50 +0300 Subject: [PATCH 21/23] +newline --- stl/src/vector_algorithms.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 802c29ac19c..879dfbdac86 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1663,6 +1663,7 @@ namespace { static __m256i _H_max_u(const __m256i _Cur) noexcept { return _Minmax_traits_8_avx::_H_max_u(_Cur); } + static double _Get_any(const __m256d _Cur) noexcept { return _mm256_cvtsd_f64(_Cur); } From 7bf105da6dec71c1f1b7506e6a2b264501a05d7b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 7 Jun 2024 14:13:33 +0300 Subject: [PATCH 22/23] const --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 879dfbdac86..52b795c4f6c 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1967,7 +1967,7 @@ namespace { } template <_Min_max_mode _Mode, class _Traits> - auto __std_minmax_element_disp(const void* _First, const void* const _Last, const bool _Sign) noexcept { + auto __std_minmax_element_disp(const void* const _First, const void* const _Last, const bool _Sign) noexcept { #ifndef _M_ARM64EC if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { return __std_minmax_element_impl<_Mode, typename _Traits::_Avx>(_First, _Last, _Sign); @@ -2116,7 +2116,7 @@ namespace { } template <_Min_max_mode _Mode, class _Traits, bool _Sign> - auto __std_minmax_disp(const void* _First, const void* const _Last) noexcept { + auto __std_minmax_disp(const void* const _First, const void* const _Last) noexcept { #ifndef _M_ARM64EC if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { return __std_minmax_impl<_Mode, typename _Traits::_Avx, _Sign>(_First, _Last); From d8df8203eccaa5bc1ff024418c7a2debf92363b7 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Fri, 7 Jun 2024 05:36:48 -0700 Subject: [PATCH 23/23] Two more newlines. --- stl/src/vector_algorithms.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 52b795c4f6c..44cca169203 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -1972,6 +1972,7 @@ namespace { if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { return __std_minmax_element_impl<_Mode, typename _Traits::_Avx>(_First, _Last, _Sign); } + if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { return __std_minmax_element_impl<_Mode, typename _Traits::_Sse>(_First, _Last, _Sign); } @@ -2121,6 +2122,7 @@ namespace { if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { return __std_minmax_impl<_Mode, typename _Traits::_Avx, _Sign>(_First, _Last); } + if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { return __std_minmax_impl<_Mode, typename _Traits::_Sse, _Sign>(_First, _Last); }