From 09f40e228cf0fd3e0ee90fb352650e9fdd79eca2 Mon Sep 17 00:00:00 2001 From: Curtis Jacques Bezault Date: Sun, 20 Feb 2022 09:56:56 -0800 Subject: [PATCH 1/4] Implement byte reverse with neon --- stl/inc/xutility | 3 +- stl/src/vector_algorithms.cpp | 217 ++++++++++++++++++++++++++++------ 2 files changed, 183 insertions(+), 37 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index 155cc7eaadc..6bfae6d530e 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -21,7 +21,8 @@ _STL_DISABLE_CLANG_WARNINGS #pragma push_macro("new") #undef new -#if (defined(_M_IX86) || defined(_M_X64)) && !defined(_M_CEE_PURE) && !defined(_M_HYBRID) +#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) && !defined(_M_HYBRID) \ + && !defined(_M_ARM64EC) #ifndef _USE_STD_VECTOR_ALGORITHMS #define _USE_STD_VECTOR_ALGORITHMS 1 #endif // _USE_STD_VECTOR_ALGORITHMS diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index bfce5924326..ae3a833d6ec 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -10,15 +10,17 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) -#if defined(_M_ARM64EC) -#include -#else // defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) #include #include #include -#endif // defined(_M_ARM64EC) +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#include +#else // _M_ARM64 +#error Unsupported architecture +#endif #include extern "C" long __isa_enabled; @@ -54,7 +56,7 @@ static void _Advance_bytes(const void*& _Target, ptrdiff_t _Offset) noexcept { extern "C" { __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* _Last1, void* _First2) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) constexpr size_t _Mask_32 = ~((static_cast(1) << 5) - 1); if (_Byte_length(_First1, _Last1) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _First1; @@ -68,7 +70,7 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( _Advance_bytes(_First2, 32); } while (_First1 != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 constexpr size_t _Mask_16 = ~((static_cast(1) << 4) - 1); if (_Byte_length(_First1, _Last1) >= 16 @@ -79,16 +81,25 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_16); do { +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First1)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_First2)); _mm_storeu_si128(static_cast<__m128i*>(_First1), _Right); _mm_storeu_si128(static_cast<__m128i*>(_First2), _Left); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + const __n128 _Left = neon_ld1r_q8(static_cast<__int8*>(_First1)); + const __n128 _Right = neon_ld1r_q8(static_cast<__int8*>(_First2)); + neon_st1m_q8(static_cast<__int8*>(_First2), _Left); + neon_st1m_q8(static_cast<__int8*>(_First1), _Right); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First1, 16); _Advance_bytes(_First2, 16); } while (_First1 != _Stop_at); } -#if defined(_M_X64) // NOTE: UNALIGNED MEMORY ACCESSES +#if defined(_M_X64) || defined(_M_ARM64) // NOTE: UNALIGNED MEMORY ACCESSES constexpr size_t _Mask_8 = ~((static_cast(1) << 3) - 1); if (_Byte_length(_First1, _Last1) >= 8) { const void* _Stop_at = _First1; @@ -108,8 +119,8 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_4); do { - const unsigned long _Left = *static_cast(_First1); - const unsigned long _Right = *static_cast(_First2); + const unsigned long _Left = *static_cast(_First1); + const unsigned long _Right = *static_cast(_First2); *static_cast(_First1) = _Right; *static_cast(_First2) = _Left; _Advance_bytes(_First1, 4); @@ -137,7 +148,7 @@ void* __cdecl __std_swap_ranges_trivially_swappable(void* _First1, void* _Last1, } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // @@ -159,20 +170,42 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 - if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_M_X64) + && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_char_sse); // SSSE3 const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_char_sse); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q8(static_cast<__int8*>(_First)); + const __n128 _Right = neon_ld1r_q8(static_cast<__int8*>(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Left_dword_reversed = neon_rev64q_8(_Left); + const __n128 _Right_dword_reversed = neon_rev64q_8(_Right); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left_dword_reversed, _Left_dword_reversed, 1); + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q8(static_cast<__int8*>(_Last), _Left_reversed); + neon_st1m_q8(static_cast<__int8*>(_First), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -181,7 +214,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // @@ -201,20 +234,42 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 - if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_M_X64) + && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_short_sse); // SSSE3 const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_short_sse); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q16(static_cast<__int16*>(_First)); + const __n128 _Right = neon_ld1r_q16(static_cast<__int16*>(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Left_dword_reversed = neon_rev64q_16(_Left); + const __n128 _Right_dword_reversed = neon_rev64q_16(_Right); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left_dword_reversed, _Left_dword_reversed, 1); + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q16(static_cast<__int16*>(_Last), _Left_reversed); + neon_st1m_q16(static_cast<__int16*>(_First), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -223,7 +278,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); @@ -240,7 +295,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 if (_Byte_length(_First, _Last) >= 32 #ifdef _M_IX86 @@ -251,12 +306,29 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(0, 1, 2, 3)); const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(0, 1, 2, 3)); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q32(static_cast<__int32*>(_First)); + const __n128 _Right = neon_ld1r_q32(static_cast<__int32*>(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Left_dword_reversed = neon_rev64q_32(_Left); + const __n128 _Right_dword_reversed = neon_rev64q_32(_Right); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left_dword_reversed, _Left_dword_reversed, 1); + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q32(static_cast<__int32*>(_Last), _Left_reversed); + neon_st1m_q32(static_cast<__int32*>(_First), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error "Unsupported Architecture" +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -265,7 +337,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _First, void* _Last) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); @@ -280,7 +352,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 if (_Byte_length(_First, _Last) >= 32 #ifdef _M_IX86 @@ -291,12 +363,26 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(1, 0, 3, 2)); const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(1, 0, 3, 2)); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q64(static_cast<__int64*>(_First)); + const __n128 _Right = neon_ld1r_q64(static_cast<__int64*>(_Last)); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left, _Left, 1); + const __n128 _Right_reversed = neon_extq64(_Right, _Right, 1); + // 128-bit stores + neon_st1m_q64(static_cast<__int64*>(_Last), _Left_reversed); + neon_st1m_q64(static_cast<__int64*>(_First), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -306,7 +392,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const void* _First, const void* _Last, void* _Dest) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // @@ -322,17 +408,35 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 - if (_Byte_length(_First, _Last) >= 16 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { + if (_Byte_length(_First, _Last) >= 16 +#if defined(_M_IX86) || defined(_M_X64) + && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_char_sse); // SSSE3 _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q8(static_cast(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Right_dword_reversed = neon_rev64q_8(_Right); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q8(static_cast<__int8*>(_Dest), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -343,7 +447,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const void* _First, const void* _Last, void* _Dest) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // @@ -359,17 +463,35 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 - if (_Byte_length(_First, _Last) >= 16 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { + if (_Byte_length(_First, _Last) >= 16 +#if defined(_M_IX86) || defined(_M_X64) + && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_short_sse); // SSSE3 _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q16(static_cast(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Right_dword_reversed = neon_rev64q_16(_Right); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q16(static_cast<__int16*>(_Dest), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -380,7 +502,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const void* _First, const void* _Last, void* _Dest) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); @@ -393,7 +515,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 if (_Byte_length(_First, _Last) >= 16 #ifdef _M_IX86 @@ -404,9 +526,22 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(0, 1, 2, 3)); _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q32(static_cast(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Right_dword_reversed = neon_rev64q_32(_Right); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q32(static_cast<__int32*>(_Dest), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -417,7 +552,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( const void* _First, const void* _Last, void* _Dest) noexcept { -#if !defined(_M_ARM64EC) +#if defined(_M_IX86) || defined(_M_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); @@ -429,7 +564,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // !defined(_M_ARM64EC) +#endif // _M_IX86 || _M_X64 if (_Byte_length(_First, _Last) >= 16 #ifdef _M_IX86 @@ -440,9 +575,20 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_M_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(1, 0, 3, 2)); _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q64(static_cast(_Last)); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right, _Right, 1); + // 128-bit stores + neon_st1m_q64(static_cast<__int64*>(_Dest), _Right_reversed); +#else // ^^^ _M_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -451,7 +597,6 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( static_cast(_Dest)); } - } // extern "C" -#endif // defined(_M_IX86) || defined(_M_X64) +#endif // defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) From e72af53511a2b1aa73ab0155fb8e6b1248fb6b7e Mon Sep 17 00:00:00 2001 From: Curtis Jacques Bezault Date: Sun, 27 Feb 2022 20:44:03 -0800 Subject: [PATCH 2/4] Use _M_ARM64EC correctly --- stl/inc/xutility | 3 +- stl/src/vector_algorithms.cpp | 119 ++++++++++++++++++---------------- 2 files changed, 64 insertions(+), 58 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index 6bfae6d530e..2568a188749 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -21,8 +21,7 @@ _STL_DISABLE_CLANG_WARNINGS #pragma push_macro("new") #undef new -#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) && !defined(_M_HYBRID) \ - && !defined(_M_ARM64EC) +#if (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) && !defined(_M_HYBRID) #ifndef _USE_STD_VECTOR_ALGORITHMS #define _USE_STD_VECTOR_ALGORITHMS 1 #endif // _USE_STD_VECTOR_ALGORITHMS diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index ae3a833d6ec..dab5be17354 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -10,21 +10,28 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif +#if defined(_M_ARM64) || defined(_M_ARM64EC) +#define _VECTOR_ARM64 +#elif defined(_M_X64) && !defined(_M_ARM64EC) +#define _VECTOR_X64 +#endif + #if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) #include #include #include -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv -#include -#else // _M_ARM64 -#error Unsupported architecture -#endif #include extern "C" long __isa_enabled; +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv +#include +#else // _VECTOR_ARM64 +#error Unsupported architecture +#endif + template static void _Reverse_tail(_BidIt _First, _BidIt _Last) noexcept { for (; _First != _Last && _First != --_Last; ++_First) { @@ -56,7 +63,7 @@ static void _Advance_bytes(const void*& _Target, ptrdiff_t _Offset) noexcept { extern "C" { __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* _Last1, void* _First2) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) constexpr size_t _Mask_32 = ~((static_cast(1) << 5) - 1); if (_Byte_length(_First1, _Last1) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _First1; @@ -70,7 +77,7 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( _Advance_bytes(_First2, 32); } while (_First1 != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 constexpr size_t _Mask_16 = ~((static_cast(1) << 4) - 1); if (_Byte_length(_First1, _Last1) >= 16 @@ -81,17 +88,17 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_16); do { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First1)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_First2)); _mm_storeu_si128(static_cast<__m128i*>(_First1), _Right); _mm_storeu_si128(static_cast<__m128i*>(_First2), _Left); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv const __n128 _Left = neon_ld1r_q8(static_cast<__int8*>(_First1)); const __n128 _Right = neon_ld1r_q8(static_cast<__int8*>(_First2)); neon_st1m_q8(static_cast<__int8*>(_First2), _Left); neon_st1m_q8(static_cast<__int8*>(_First1), _Right); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_First1, 16); @@ -99,7 +106,7 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( } while (_First1 != _Stop_at); } -#if defined(_M_X64) || defined(_M_ARM64) // NOTE: UNALIGNED MEMORY ACCESSES +#if defined(_VECTOR_X64) || defined(_VECTOR_ARM64) // NOTE: UNALIGNED MEMORY ACCESSES constexpr size_t _Mask_8 = ~((static_cast(1) << 3) - 1); if (_Byte_length(_First1, _Last1) >= 8) { const void* _Stop_at = _First1; @@ -148,7 +155,7 @@ void* __cdecl __std_swap_ranges_trivially_swappable(void* _First1, void* _Last1, } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // @@ -170,10 +177,10 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); #else @@ -183,14 +190,14 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_char_sse); // SSSE3 const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_char_sse); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Left = neon_ld1r_q8(static_cast<__int8*>(_First)); const __n128 _Right = neon_ld1r_q8(static_cast<__int8*>(_Last)); @@ -203,7 +210,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs // 128-bit stores neon_st1m_q8(static_cast<__int8*>(_Last), _Left_reversed); neon_st1m_q8(static_cast<__int8*>(_First), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_First, 16); @@ -214,7 +221,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // @@ -234,10 +241,10 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -247,14 +254,14 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_short_sse); // SSSE3 const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_short_sse); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Left = neon_ld1r_q16(static_cast<__int16*>(_First)); const __n128 _Right = neon_ld1r_q16(static_cast<__int16*>(_Last)); @@ -267,7 +274,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs // 128-bit stores neon_st1m_q16(static_cast<__int16*>(_Last), _Left_reversed); neon_st1m_q16(static_cast<__int16*>(_First), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_First, 16); @@ -278,7 +285,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); @@ -295,7 +302,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 #ifdef _M_IX86 @@ -306,14 +313,14 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(0, 1, 2, 3)); const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(0, 1, 2, 3)); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Left = neon_ld1r_q32(static_cast<__int32*>(_First)); const __n128 _Right = neon_ld1r_q32(static_cast<__int32*>(_Last)); @@ -326,7 +333,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs // 128-bit stores neon_st1m_q32(static_cast<__int32*>(_Last), _Left_reversed); neon_st1m_q32(static_cast<__int32*>(_First), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error "Unsupported Architecture" #endif _Advance_bytes(_First, 16); @@ -337,7 +344,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _First, void* _Last) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); @@ -352,7 +359,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 #ifdef _M_IX86 @@ -363,14 +370,14 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(1, 0, 3, 2)); const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(1, 0, 3, 2)); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Left = neon_ld1r_q64(static_cast<__int64*>(_First)); const __n128 _Right = neon_ld1r_q64(static_cast<__int64*>(_Last)); @@ -380,7 +387,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs // 128-bit stores neon_st1m_q64(static_cast<__int64*>(_Last), _Left_reversed); neon_st1m_q64(static_cast<__int64*>(_First), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_First, 16); @@ -392,7 +399,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const void* _First, const void* _Last, void* _Dest) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // @@ -408,10 +415,10 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); #else @@ -421,11 +428,11 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_char_sse); // SSSE3 _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Right = neon_ld1r_q8(static_cast(_Last)); // Reverse the bytes of each 64-bit DWORDs @@ -434,7 +441,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); // 128-bit stores neon_st1m_q8(static_cast<__int8*>(_Dest), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_Dest, 16); @@ -447,7 +454,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const void* _First, const void* _Last, void* _Dest) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // @@ -463,10 +470,10 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -476,11 +483,11 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_short_sse); // SSSE3 _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Right = neon_ld1r_q16(static_cast(_Last)); // Reverse the bytes of each 64-bit DWORDs @@ -489,7 +496,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); // 128-bit stores neon_st1m_q16(static_cast<__int16*>(_Dest), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_Dest, 16); @@ -502,7 +509,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const void* _First, const void* _Last, void* _Dest) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); @@ -515,7 +522,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 #ifdef _M_IX86 @@ -526,11 +533,11 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(0, 1, 2, 3)); _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Right = neon_ld1r_q32(static_cast(_Last)); // Reverse the bytes of each 64-bit DWORDs @@ -539,7 +546,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); // 128-bit stores neon_st1m_q32(static_cast<__int32*>(_Dest), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_Dest, 16); @@ -552,7 +559,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( const void* _First, const void* _Last, void* _Dest) noexcept { -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); @@ -564,7 +571,7 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _M_X64 +#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 #ifdef _M_IX86 @@ -575,18 +582,18 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); -#if defined(_M_IX86) || defined(_M_X64) +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(1, 0, 3, 2)); _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); -#elif defined(_M_ARM64) // ^^^ _M_IX86 || _M_X64 ^^^ // vvv _M_ARM64 vvv +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv // 128-bit loads const __n128 _Right = neon_ld1r_q64(static_cast(_Last)); // Swap the 64-bit DWORDS const __n128 _Right_reversed = neon_extq64(_Right, _Right, 1); // 128-bit stores neon_st1m_q64(static_cast<__int64*>(_Dest), _Right_reversed); -#else // ^^^ _M_ARM64 ^^^ +#else // ^^^ _VECTOR_ARM64 ^^^ #error Unsupported architecture #endif _Advance_bytes(_Dest, 16); From aae92e43db6407e2a183dc3fe5bddb55ae669276 Mon Sep 17 00:00:00 2001 From: Curtis Jacques Bezault Date: Mon, 28 Feb 2022 19:56:40 -0800 Subject: [PATCH 3/4] Unroll NEON implementation once --- stl/src/vector_algorithms.cpp | 183 +++++++++++++++++++++++++++++----- 1 file changed, 160 insertions(+), 23 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index dab5be17354..efe7b206540 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -63,21 +63,32 @@ static void _Advance_bytes(const void*& _Target, ptrdiff_t _Offset) noexcept { extern "C" { __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* _Last1, void* _First2) noexcept { -#if defined(_M_IX86) || defined(_VECTOR_X64) constexpr size_t _Mask_32 = ~((static_cast(1) << 5) - 1); - if (_Byte_length(_First1, _Last1) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + if (_Byte_length(_First1, _Last1) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2) +#endif + ) { const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_32); do { +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First1)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_First2)); _mm256_storeu_si256(static_cast<__m256i*>(_First1), _Right); _mm256_storeu_si256(static_cast<__m256i*>(_First2), _Left); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q8(static_cast<__int8*>(_First1)); + const __n128x2 _Right = neon_ld1m2_q8(static_cast<__int8*>(_First2)); + neon_st1m2_q8(static_cast<__int8*>(_First2), _Left); + neon_st1m2_q8(static_cast<__int8*>(_First1), _Right); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First1, 32); _Advance_bytes(_First2, 32); } while (_First1 != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 constexpr size_t _Mask_16 = ~((static_cast(1) << 4) - 1); if (_Byte_length(_First1, _Last1) >= 16 @@ -155,15 +166,20 @@ void* __cdecl __std_swap_ranges_trivially_swappable(void* _First1, void* _Last1, } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { + if (_Byte_length(_First, _Last) >= 64 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) // vpermq to load left and right, and transpose the lanes const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); @@ -174,10 +190,25 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_char_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X65 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q8(static_cast<__int8*>(_First)); + const __n128x2 _Right = neon_ld1m2_q8(static_cast<__int8*>(_Last)); + const __n128 _Left_dword_reversed1 = neon_rev64q_8(_Left.val[0]); + const __n128 _Left_dword_reversed2 = neon_rev64q_8(_Left.val[1]); + const __n128 _Right_dword_reversed2 = neon_rev64q_8(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_8(_Right.val[1]); + const __n128 _Left_reversed1 = neon_extq64(_Left_dword_reversed1, _Left_dword_reversed1, 1); + const __n128 _Left_reversed2 = neon_extq64(_Left_dword_reversed2, _Left_dword_reversed2, 1); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q8(static_cast<__int8*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q8(static_cast<__int8*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 #if defined(_M_IX86) || defined(_VECTOR_X64) @@ -221,15 +252,20 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { + if (_Byte_length(_First, _Last) >= 64 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); @@ -238,10 +274,25 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_short_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q16(static_cast<__int16*>(_First)); + const __n128x2 _Right = neon_ld1m2_q16(static_cast<__int16*>(_Last)); + const __n128 _Left_dword_reversed1 = neon_rev64q_16(_Left.val[0]); + const __n128 _Left_dword_reversed2 = neon_rev64q_16(_Left.val[1]); + const __n128 _Right_dword_reversed2 = neon_rev64q_16(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_16(_Right.val[1]); + const __n128 _Left_reversed1 = neon_extq64(_Left_dword_reversed1, _Left_dword_reversed1, 1); + const __n128 _Left_reversed2 = neon_extq64(_Left_dword_reversed2, _Left_dword_reversed2, 1); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q16(static_cast<__int16*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q16(static_cast<__int16*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 #if defined(_M_IX86) || defined(_VECTOR_X64) @@ -285,12 +336,16 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { + if (_Byte_length(_First, _Last) >= 64 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2) +#endif + ) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); @@ -299,10 +354,25 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs const __m256i _Right_reversed = _mm256_shuffle_epi32(_Right_perm, _MM_SHUFFLE(0, 1, 2, 3)); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q32(static_cast<__int32*>(_First)); + const __n128x2 _Right = neon_ld1m2_q32(static_cast<__int32*>(_Last)); + const __n128 _Left_dword_reversed1 = neon_rev64q_32(_Left.val[0]); + const __n128 _Left_dword_reversed2 = neon_rev64q_32(_Left.val[1]); + const __n128 _Right_dword_reversed2 = neon_rev64q_32(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_32(_Right.val[1]); + const __n128 _Left_reversed1 = neon_extq64(_Left_dword_reversed1, _Left_dword_reversed1, 1); + const __n128 _Left_reversed2 = neon_extq64(_Left_dword_reversed2, _Left_dword_reversed2, 1); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q32(static_cast<__int32*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q32(static_cast<__int32*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 #ifdef _M_IX86 @@ -344,22 +414,37 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _First, void* _Last) noexcept { + if (_Byte_length(_First, _Last) >= 64 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2) +#endif + ) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); const __m256i _Left_reversed = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(0, 1, 2, 3)); const __m256i _Right_reversed = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(0, 1, 2, 3)); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q64(static_cast<__int64*>(_First)); + const __n128x2 _Right = neon_ld1m2_q64(static_cast<__int64*>(_Last)); + const __n128 _Left_reversed1 = neon_extq64(_Left.val[0], _Left.val[0], 1); + const __n128 _Left_reversed2 = neon_extq64(_Left.val[1], _Left.val[1], 1); + const __n128 _Right_reversed2 = neon_extq64(_Right.val[0], _Right.val[0], 1); + const __n128 _Right_reversed1 = neon_extq64(_Right.val[1], _Right.val[1], 1); + neon_st1m2_q64(static_cast<__int64*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q64(static_cast<__int64*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 32 #ifdef _M_IX86 @@ -399,23 +484,37 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const void* _First, const void* _Last, void* _Dest) noexcept { + if (_Byte_length(_First, _Last) >= 32 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_char_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q8(static_cast(_Last)); + const __n128 _Right_dword_reversed2 = neon_rev64q_8(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_8(_Right.val[1]); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q8(static_cast<__int8*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 #if defined(_M_IX86) || defined(_VECTOR_X64) @@ -454,23 +553,37 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const void* _First, const void* _Last, void* _Dest) noexcept { + if (_Byte_length(_First, _Last) >= 32 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_short_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q16(static_cast(_Last)); + const __n128 _Right_dword_reversed2 = neon_rev64q_16(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_16(_Right.val[1]); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q16(reinterpret_cast<__int16*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 #if defined(_M_IX86) || defined(_VECTOR_X64) @@ -509,20 +622,33 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const void* _First, const void* _Last, void* _Dest) noexcept { + if (_Byte_length(_First, _Last) >= 32 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2) +#endif + ) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); const __m256i _Block_reversed = _mm256_shuffle_epi32(_Block_permuted, _MM_SHUFFLE(0, 1, 2, 3)); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q32(static_cast(_Last)); + const __n128 _Right_dword_reversed2 = neon_rev64q_32(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_32(_Right.val[1]); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q32(static_cast<__int32*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 #ifdef _M_IX86 @@ -559,19 +685,30 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( const void* _First, const void* _Last, void* _Dest) noexcept { + if (_Byte_length(_First, _Last) >= 32 #if defined(_M_IX86) || defined(_VECTOR_X64) - if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { + && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2) +#endif + ) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_reversed = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(0, 1, 2, 3)); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q64(static_cast(_Last)); + const __n128 _Right_reversed2 = neon_extq64(_Right.val[0], _Right.val[0], 1); + const __n128 _Right_reversed1 = neon_extq64(_Right.val[1], _Right.val[1], 1); + neon_st1m2_q64(static_cast<__int64*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } -#endif // _M_IX86 || _VECTOR_X64 if (_Byte_length(_First, _Last) >= 16 #ifdef _M_IX86 From 4557dad2cb5aaee1171297e0a4ccf7f293164804 Mon Sep 17 00:00:00 2001 From: Charles Barto Date: Wed, 11 May 2022 17:25:29 -0700 Subject: [PATCH 4/4] clang-format --- stl/src/vector_algorithms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b948e9c2619..6cd93f0cc54 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -186,7 +186,7 @@ void* __cdecl __std_swap_ranges_trivially_swappable(void* _First1, void* _Last1, __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { if (_Byte_length(_First, _Last) >= 64 #if defined(_M_IX86) || defined(_VECTOR_X64) - && _Use_avx2()) { + && _Use_avx2()) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); @@ -272,7 +272,7 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { if (_Byte_length(_First, _Last) >= 64 #if defined(_M_IX86) || defined(_VECTOR_X64) - && _Use_avx2()) { + && _Use_avx2()) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14);