diff --git a/stl/inc/xutility b/stl/inc/xutility index 84a1848d9ce..ffeeb4384a4 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -31,7 +31,7 @@ _STL_DISABLE_CLANG_WARNINGS #elif _USE_STD_VECTOR_ALGORITHMS #error Vector algorithms are not supported when building msvcp140.dll, but _USE_STD_VECTOR_ALGORITHMS is set. #endif // _USE_STD_VECTOR_ALGORITHMS -#elif (defined(_M_IX86) || defined(_M_X64)) && !defined(_M_CEE_PURE) && !defined(_M_HYBRID) && !defined(_M_ARM64EC) +#elif (defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64)) && !defined(_M_CEE_PURE) && !defined(_M_HYBRID) #ifndef _USE_STD_VECTOR_ALGORITHMS #define _USE_STD_VECTOR_ALGORITHMS 1 #endif // _USE_STD_VECTOR_ALGORITHMS diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index e621ecbb248..6cd93f0cc54 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -10,8 +10,14 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif -#if (defined(_M_IX86) || defined(_M_X64)) && !defined(_M_ARM64EC) +#if defined(_M_ARM64) || defined(_M_ARM64EC) +#define _VECTOR_ARM64 +#elif defined(_M_X64) && !defined(_M_ARM64EC) +#define _VECTOR_X64 +#endif +#if defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) +#if defined(_M_IX86) || defined(_VECTOR_X64) #include #include #include @@ -20,6 +26,12 @@ extern "C" long __isa_enabled; +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv +#include +#else // _VECTOR_ARM64 +#error Unsupported architecture +#endif + #ifndef _DEBUG #pragma optimize("t", on) // Override /Os with /Ot for this TU #endif // !_DEBUG @@ -74,14 +86,27 @@ extern "C" { __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* _Last1, void* _First2) noexcept { constexpr size_t _Mask_32 = ~((static_cast(1) << 5) - 1); - if (_Byte_length(_First1, _Last1) >= 32 && _Use_avx2()) { + if (_Byte_length(_First1, _Last1) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2() +#endif + ) { const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_32); do { +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First1)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_First2)); _mm256_storeu_si256(static_cast<__m256i*>(_First1), _Right); _mm256_storeu_si256(static_cast<__m256i*>(_First2), _Left); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q8(static_cast<__int8*>(_First1)); + const __n128x2 _Right = neon_ld1m2_q8(static_cast<__int8*>(_First2)); + neon_st1m2_q8(static_cast<__int8*>(_First2), _Left); + neon_st1m2_q8(static_cast<__int8*>(_First1), _Right); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First1, 32); _Advance_bytes(_First2, 32); } while (_First1 != _Stop_at); @@ -92,16 +117,25 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( const void* _Stop_at = _First1; _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_16); do { +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First1)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_First2)); _mm_storeu_si128(static_cast<__m128i*>(_First1), _Right); _mm_storeu_si128(static_cast<__m128i*>(_First2), _Left); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128 _Left = neon_ld1r_q8(static_cast<__int8*>(_First1)); + const __n128 _Right = neon_ld1r_q8(static_cast<__int8*>(_First2)); + neon_st1m_q8(static_cast<__int8*>(_First2), _Left); + neon_st1m_q8(static_cast<__int8*>(_First1), _Right); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First1, 16); _Advance_bytes(_First2, 16); } while (_First1 != _Stop_at); } -#if defined(_M_X64) // NOTE: UNALIGNED MEMORY ACCESSES +#if defined(_VECTOR_X64) || defined(_VECTOR_ARM64) // NOTE: UNALIGNED MEMORY ACCESSES constexpr size_t _Mask_8 = ~((static_cast(1) << 3) - 1); if (_Byte_length(_First1, _Last1) >= 8) { const void* _Stop_at = _First1; @@ -150,14 +184,20 @@ void* __cdecl __std_swap_ranges_trivially_swappable(void* _First1, void* _Last1, } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 64 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2()) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) // vpermq to load left and right, and transpose the lanes const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); @@ -168,22 +208,60 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_char_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X65 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q8(static_cast<__int8*>(_First)); + const __n128x2 _Right = neon_ld1m2_q8(static_cast<__int8*>(_Last)); + const __n128 _Left_dword_reversed1 = neon_rev64q_8(_Left.val[0]); + const __n128 _Left_dword_reversed2 = neon_rev64q_8(_Left.val[1]); + const __n128 _Right_dword_reversed2 = neon_rev64q_8(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_8(_Right.val[1]); + const __n128 _Left_reversed1 = neon_extq64(_Left_dword_reversed1, _Left_dword_reversed1, 1); + const __n128 _Left_reversed2 = neon_extq64(_Left_dword_reversed2, _Left_dword_reversed2, 1); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q8(static_cast<__int8*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q8(static_cast<__int8*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_sse42()) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_char_sse); // SSSE3 const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_char_sse); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q8(static_cast<__int8*>(_First)); + const __n128 _Right = neon_ld1r_q8(static_cast<__int8*>(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Left_dword_reversed = neon_rev64q_8(_Left); + const __n128 _Right_dword_reversed = neon_rev64q_8(_Right); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left_dword_reversed, _Left_dword_reversed, 1); + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q8(static_cast<__int8*>(_Last), _Left_reversed); + neon_st1m_q8(static_cast<__int8*>(_First), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -192,14 +270,20 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _First, void* _Last) noexcept { - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 64 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2()) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); const __m256i _Left_perm = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(1, 0, 3, 2)); @@ -208,22 +292,60 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs const __m256i _Right_reversed = _mm256_shuffle_epi8(_Right_perm, _Reverse_short_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q16(static_cast<__int16*>(_First)); + const __n128x2 _Right = neon_ld1m2_q16(static_cast<__int16*>(_Last)); + const __n128 _Left_dword_reversed1 = neon_rev64q_16(_Left.val[0]); + const __n128 _Left_dword_reversed2 = neon_rev64q_16(_Left.val[1]); + const __n128 _Right_dword_reversed2 = neon_rev64q_16(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_16(_Right.val[1]); + const __n128 _Left_reversed1 = neon_extq64(_Left_dword_reversed1, _Left_dword_reversed1, 1); + const __n128 _Left_reversed2 = neon_extq64(_Left_dword_reversed2, _Left_dword_reversed2, 1); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q16(static_cast<__int16*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q16(static_cast<__int16*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse42()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_sse42()) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; +#else + ) { +#endif + const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_short_sse); // SSSE3 const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_short_sse); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q16(static_cast<__int16*>(_First)); + const __n128 _Right = neon_ld1r_q16(static_cast<__int16*>(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Left_dword_reversed = neon_rev64q_16(_Left); + const __n128 _Right_dword_reversed = neon_rev64q_16(_Right); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left_dword_reversed, _Left_dword_reversed, 1); + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q16(static_cast<__int16*>(_Last), _Left_reversed); + neon_st1m_q16(static_cast<__int16*>(_First), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -232,33 +354,75 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _First, void* _Last) noexcept { - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 64 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2() +#endif + ) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); const __m256i _Left_reversed = _mm256_permutevar8x32_epi32(_Left, _Shuf); const __m256i _Right_reversed = _mm256_permutevar8x32_epi32(_Right, _Shuf); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q32(static_cast<__int32*>(_First)); + const __n128x2 _Right = neon_ld1m2_q32(static_cast<__int32*>(_Last)); + const __n128 _Left_dword_reversed1 = neon_rev64q_32(_Left.val[0]); + const __n128 _Left_dword_reversed2 = neon_rev64q_32(_Left.val[1]); + const __n128 _Right_dword_reversed2 = neon_rev64q_32(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_32(_Right.val[1]); + const __n128 _Left_reversed1 = neon_extq64(_Left_dword_reversed1, _Left_dword_reversed1, 1); + const __n128 _Left_reversed2 = neon_extq64(_Left_dword_reversed2, _Left_dword_reversed2, 1); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q32(static_cast<__int32*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q32(static_cast<__int32*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } - if (_Byte_length(_First, _Last) >= 32 && _Use_sse2()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_sse2() +#endif + ) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(0, 1, 2, 3)); const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(0, 1, 2, 3)); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q32(static_cast<__int32*>(_First)); + const __n128 _Right = neon_ld1r_q32(static_cast<__int32*>(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Left_dword_reversed = neon_rev64q_32(_Left); + const __n128 _Right_dword_reversed = neon_rev64q_32(_Right); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left_dword_reversed, _Left_dword_reversed, 1); + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q32(static_cast<__int32*>(_Last), _Left_reversed); + neon_st1m_q32(static_cast<__int32*>(_First), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error "Unsupported Architecture" +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -267,17 +431,34 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_4(void* _Firs } __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _First, void* _Last) noexcept { - if (_Byte_length(_First, _Last) >= 64 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 64 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2() +#endif + ) { const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 6 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Left = _mm256_loadu_si256(static_cast<__m256i*>(_First)); const __m256i _Right = _mm256_loadu_si256(static_cast<__m256i*>(_Last)); const __m256i _Left_reversed = _mm256_permute4x64_epi64(_Left, _MM_SHUFFLE(0, 1, 2, 3)); const __m256i _Right_reversed = _mm256_permute4x64_epi64(_Right, _MM_SHUFFLE(0, 1, 2, 3)); _mm256_storeu_si256(static_cast<__m256i*>(_First), _Right_reversed); _mm256_storeu_si256(static_cast<__m256i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Left = neon_ld1m2_q64(static_cast<__int64*>(_First)); + const __n128x2 _Right = neon_ld1m2_q64(static_cast<__int64*>(_Last)); + const __n128 _Left_reversed1 = neon_extq64(_Left.val[0], _Left.val[0], 1); + const __n128 _Left_reversed2 = neon_extq64(_Left.val[1], _Left.val[1], 1); + const __n128 _Right_reversed2 = neon_extq64(_Right.val[0], _Right.val[0], 1); + const __n128 _Right_reversed1 = neon_extq64(_Right.val[1], _Right.val[1], 1); + neon_st1m2_q64(static_cast<__int64*>(_Last), __n128x2{_Left_reversed2, _Left_reversed1}); + neon_st1m2_q64(static_cast<__int64*>(_First), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 32); } while (_First != _Stop_at); } @@ -287,12 +468,26 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); const __m128i _Left_reversed = _mm_shuffle_epi32(_Left, _MM_SHUFFLE(1, 0, 3, 2)); const __m128i _Right_reversed = _mm_shuffle_epi32(_Right, _MM_SHUFFLE(1, 0, 3, 2)); _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Left = neon_ld1r_q64(static_cast<__int64*>(_First)); + const __n128 _Right = neon_ld1r_q64(static_cast<__int64*>(_Last)); + // Swap the 64-bit DWORDS + const __n128 _Left_reversed = neon_extq64(_Left, _Left, 1); + const __n128 _Right_reversed = neon_extq64(_Right, _Right, 1); + // 128-bit stores + neon_st1m_q64(static_cast<__int64*>(_Last), _Left_reversed); + neon_st1m_q64(static_cast<__int64*>(_First), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_First, 16); } while (_First != _Stop_at); } @@ -302,31 +497,65 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_8(void* _Firs __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( const void* _First, const void* _Last, void* _Dest) noexcept { - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2()) { const __m256i _Reverse_char_lanes_avx = _mm256_set_epi8( // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_char_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q8(static_cast(_Last)); + const __n128 _Right_dword_reversed2 = neon_rev64q_8(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_8(_Right.val[1]); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q8(static_cast<__int8*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + if (_Byte_length(_First, _Last) >= 16 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_sse42()) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_char_sse); // SSSE3 _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q8(static_cast(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Right_dword_reversed = neon_rev64q_8(_Right); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q8(static_cast<__int8*>(_Dest), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -337,31 +566,65 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( const void* _First, const void* _Last, void* _Dest) noexcept { - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2()) { const __m256i _Reverse_short_lanes_avx = _mm256_set_epi8( // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, // 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_permuted = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(1, 0, 3, 2)); const __m256i _Block_reversed = _mm256_shuffle_epi8(_Block_permuted, _Reverse_short_lanes_avx); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q16(static_cast(_Last)); + const __n128 _Right_dword_reversed2 = neon_rev64q_16(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_16(_Right.val[1]); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q16(reinterpret_cast<__int16*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } - if (_Byte_length(_First, _Last) >= 16 && _Use_sse42()) { + if (_Byte_length(_First, _Last) >= 16 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_sse42()) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _Dest; +#else + ) { +#endif + const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_short_sse); // SSSE3 _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q16(static_cast(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Right_dword_reversed = neon_rev64q_16(_Right); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q16(static_cast<__int16*>(_Dest), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -372,15 +635,30 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( const void* _First, const void* _Last, void* _Dest) noexcept { - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2() +#endif + ) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); const __m256i _Shuf = _mm256_set_epi32(0, 1, 2, 3, 4, 5, 6, 7); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_reversed = _mm256_permutevar8x32_epi32(_Block, _Shuf); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q32(static_cast(_Last)); + const __n128 _Right_dword_reversed2 = neon_rev64q_32(_Right.val[0]); + const __n128 _Right_dword_reversed1 = neon_rev64q_32(_Right.val[1]); + const __n128 _Right_reversed2 = neon_extq64(_Right_dword_reversed2, _Right_dword_reversed2, 1); + const __n128 _Right_reversed1 = neon_extq64(_Right_dword_reversed1, _Right_dword_reversed1, 1); + neon_st1m2_q32(static_cast<__int32*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } @@ -390,9 +668,22 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(0, 1, 2, 3)); _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q32(static_cast(_Last)); + // Reverse the bytes of each 64-bit DWORDs + const __n128 _Right_dword_reversed = neon_rev64q_32(_Right); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right_dword_reversed, _Right_dword_reversed, 1); + // 128-bit stores + neon_st1m_q32(static_cast<__int32*>(_Dest), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -403,14 +694,27 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_4( __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( const void* _First, const void* _Last, void* _Dest) noexcept { - if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { + if (_Byte_length(_First, _Last) >= 32 +#if defined(_M_IX86) || defined(_VECTOR_X64) + && _Use_avx2() +#endif + ) { const void* _Stop_at = _Dest; _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 5); do { _Advance_bytes(_Last, -32); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m256i _Block = _mm256_loadu_si256(static_cast(_Last)); const __m256i _Block_reversed = _mm256_permute4x64_epi64(_Block, _MM_SHUFFLE(0, 1, 2, 3)); _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + const __n128x2 _Right = neon_ld1m2_q64(static_cast(_Last)); + const __n128 _Right_reversed2 = neon_extq64(_Right.val[0], _Right.val[0], 1); + const __n128 _Right_reversed1 = neon_extq64(_Right.val[1], _Right.val[1], 1); + neon_st1m2_q64(static_cast<__int64*>(_Dest), __n128x2{_Right_reversed1, _Right_reversed2}); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 32); } while (_Dest != _Stop_at); } @@ -420,9 +724,20 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); do { _Advance_bytes(_Last, -16); +#if defined(_M_IX86) || defined(_VECTOR_X64) const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); const __m128i _Block_reversed = _mm_shuffle_epi32(_Block, _MM_SHUFFLE(1, 0, 3, 2)); _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); +#elif defined(_VECTOR_ARM64) // ^^^ _M_IX86 || _VECTOR_X64 ^^^ // vvv _VECTOR_ARM64 vvv + // 128-bit loads + const __n128 _Right = neon_ld1r_q64(static_cast(_Last)); + // Swap the 64-bit DWORDS + const __n128 _Right_reversed = neon_extq64(_Right, _Right, 1); + // 128-bit stores + neon_st1m_q64(static_cast<__int64*>(_Dest), _Right_reversed); +#else // ^^^ _VECTOR_ARM64 ^^^ +#error Unsupported architecture +#endif _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); } @@ -430,7 +745,6 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Reverse_copy_tail(static_cast(_First), static_cast(_Last), static_cast(_Dest)); } - } // extern "C" namespace {