From 30e5f45293f06cac586f45c4be621d8b15d153d0 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Fri, 10 Dec 2021 23:16:18 +0200 Subject: [PATCH 1/4] Optimize reverse for 8 and 16-bit trivial types faster reverse tail --- stl/src/vector_algorithms.cpp | 51 +++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index bfce5924326..b8664194582 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -20,6 +20,7 @@ #include #endif // defined(_M_ARM64EC) #include +#include extern "C" long __isa_enabled; @@ -177,6 +178,19 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs } while (_First != _Stop_at); } + if (_Byte_length(_First, _Last) >= 8) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 3 << 2); + do { + _Advance_bytes(_Last, -4); + unsigned long _Left = _byteswap_ulong(*static_cast(_First)); + unsigned long _Right = _byteswap_ulong(*static_cast(_Last)); + *static_cast(_First) = _Right; + *static_cast(_Last) = _Left; + _Advance_bytes(_First, 4); + } while (_First != _Stop_at); + } + _Reverse_tail(static_cast(_First), static_cast(_Last)); } @@ -219,6 +233,19 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs } while (_First != _Stop_at); } + if (_Byte_length(_First, _Last) >= 8) { + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 3 << 2); + do { + _Advance_bytes(_Last, -4); + unsigned long _Left = _rotl(*static_cast(_First), 16); + unsigned long _Right = _rotl(*static_cast(_Last), 16); + *static_cast(_First) = _Right; + *static_cast(_Last) = _Left; + _Advance_bytes(_First, 4); + } while (_First != _Stop_at); + } + _Reverse_tail(static_cast(_First), static_cast(_Last)); } @@ -337,6 +364,18 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( } while (_Dest != _Stop_at); } + if (_Byte_length(_First, _Last) >= 4) { + const void* _Stop_at = _Dest; + _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 2 << 2); + do { + _Advance_bytes(_Last, -4); + unsigned long _Block = *static_cast(_Last); + unsigned long _Block_reversed = _byteswap_ulong(_Block); + *static_cast(_Dest) = _Block_reversed; + _Advance_bytes(_Dest, 4); + } while (_Dest != _Stop_at); + } + _Reverse_copy_tail(static_cast(_First), static_cast(_Last), static_cast(_Dest)); } @@ -374,6 +413,18 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( } while (_Dest != _Stop_at); } + if (_Byte_length(_First, _Last) >= 4) { + const void* _Stop_at = _Dest; + _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 2 << 2); + do { + _Advance_bytes(_Last, -4); + unsigned long _Block = *static_cast(_Last); + unsigned long _Block_reversed = _rotl(_Block, 16); + *static_cast(_Dest) = _Block_reversed; + _Advance_bytes(_Dest, 4); + } while (_Dest != _Stop_at); + } + _Reverse_copy_tail(static_cast(_First), static_cast(_Last), static_cast(_Dest)); } From 4b58c19e6d30ec8658bd48ef081aa825cadf2bdd Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 11 Dec 2021 21:27:33 +0200 Subject: [PATCH 2/4] another optimization strategy --- stl/src/vector_algorithms.cpp | 126 ++++++++++++++++++++-------------- 1 file changed, 74 insertions(+), 52 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index b8664194582..40953d3f7d2 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -137,6 +137,10 @@ void* __cdecl __std_swap_ranges_trivially_swappable(void* _First1, void* _Last1, return static_cast(_First2) + (static_cast(_Last1) - static_cast(_First1)); } +struct alignas(16) _Final_lut_entry_sse { + const unsigned char _Lut[16]; +}; + __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _First, void* _Last) noexcept { #if !defined(_M_ARM64EC) if (_Byte_length(_First, _Last) >= 64 && _bittest(&__isa_enabled, __ISA_AVAILABLE_AVX2)) { @@ -164,8 +168,30 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); + static constexpr _Final_lut_entry_sse _Reverse_char_sse_final[16] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {1, 0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {2, 1, 0, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {3, 2, 1, 0, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {4, 3, 2, 1, 0, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {5, 4, 3, 2, 1, 0, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {6, 5, 4, 3, 2, 1, 0, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {7, 6, 5, 4, 3, 2, 1, 0, 8, 9, 10, 11, 12, 13, 14, 15}, + {8, 7, 6, 5, 4, 3, 2, 1, 0, 9, 10, 11, 12, 13, 14, 15}, + {9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 10, 11, 12, 13, 14, 15}, + {10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 11, 12, 13, 14, 15}, + {11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 12, 13, 14, 15}, + {12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 13, 14, 15}, + {13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 14, 15}, + {14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0, 15}, + }; + const void* _Stop_at = _First; + const size_t _Size = _Byte_length(_First, _Last); + const __m128i _Final_lut_char_sse = + _mm_loadu_si128(reinterpret_cast(&_Reverse_char_sse_final[_Size & 0xF])); + + _Advance_bytes(_Stop_at, _Size >> 5 << 4); do { _Advance_bytes(_Last, -16); const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); @@ -176,19 +202,22 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_1(void* _Firs _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); _Advance_bytes(_First, 16); } while (_First != _Stop_at); - } - if (_Byte_length(_First, _Last) >= 8) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 3 << 2); - do { - _Advance_bytes(_Last, -4); - unsigned long _Left = _byteswap_ulong(*static_cast(_First)); - unsigned long _Right = _byteswap_ulong(*static_cast(_Last)); - *static_cast(_First) = _Right; - *static_cast(_Last) = _Left; - _Advance_bytes(_First, 4); - } while (_First != _Stop_at); + if ((_Size & 0x10) == 0) { + const __m128i _Final = _mm_loadu_si128(static_cast<__m128i*>(_First)); + const __m128i _Final_reversed = _mm_shuffle_epi8(_Final, _Final_lut_char_sse); + _mm_storeu_si128(static_cast<__m128i*>(_First), _Final_reversed); + } else { + _Advance_bytes(_Last, -16); + const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); + const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); + const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_char_sse); + const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_char_sse); + _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); + _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); + } + + return; } _Reverse_tail(static_cast(_First), static_cast(_Last)); @@ -219,8 +248,22 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs if (_Byte_length(_First, _Last) >= 32 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 5 << 4); + static constexpr _Final_lut_entry_sse _Reverse_short_sse_final[8] = { + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {2, 3, 0, 1, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {4, 5, 2, 3, 0, 1, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, + {6, 7, 4, 5, 2, 3, 0, 1, 8, 9, 10, 11, 12, 13, 14, 15}, + {8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 10, 11, 12, 13, 14, 15}, + {10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 12, 13, 14, 15}, + {12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1, 14, 15}, + }; + const void* _Stop_at = _First; + const size_t _Size = _Byte_length(_First, _Last); + const __m128i _Final_lut_short_sse = + _mm_loadu_si128(reinterpret_cast(&_Reverse_short_sse_final[(_Size & 0xF) >> 1])); + + _Advance_bytes(_Stop_at, _Size >> 5 << 4); do { _Advance_bytes(_Last, -16); const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); @@ -231,19 +274,22 @@ __declspec(noalias) void __cdecl __std_reverse_trivially_swappable_2(void* _Firs _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); _Advance_bytes(_First, 16); } while (_First != _Stop_at); - } - if (_Byte_length(_First, _Last) >= 8) { - const void* _Stop_at = _First; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 3 << 2); - do { - _Advance_bytes(_Last, -4); - unsigned long _Left = _rotl(*static_cast(_First), 16); - unsigned long _Right = _rotl(*static_cast(_Last), 16); - *static_cast(_First) = _Right; - *static_cast(_Last) = _Left; - _Advance_bytes(_First, 4); - } while (_First != _Stop_at); + if ((_Size & 0x10) == 0) { + const __m128i _Final = _mm_loadu_si128(static_cast<__m128i*>(_First)); + const __m128i _Final_reversed = _mm_shuffle_epi8(_Final, _Final_lut_short_sse); + _mm_storeu_si128(static_cast<__m128i*>(_First), _Final_reversed); + } else { + _Advance_bytes(_Last, -16); + const __m128i _Left = _mm_loadu_si128(static_cast<__m128i*>(_First)); + const __m128i _Right = _mm_loadu_si128(static_cast<__m128i*>(_Last)); + const __m128i _Left_reversed = _mm_shuffle_epi8(_Left, _Reverse_short_sse); + const __m128i _Right_reversed = _mm_shuffle_epi8(_Right, _Reverse_short_sse); + _mm_storeu_si128(static_cast<__m128i*>(_First), _Right_reversed); + _mm_storeu_si128(static_cast<__m128i*>(_Last), _Left_reversed); + } + + return; } _Reverse_tail(static_cast(_First), static_cast(_Last)); @@ -364,18 +410,6 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( } while (_Dest != _Stop_at); } - if (_Byte_length(_First, _Last) >= 4) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 2 << 2); - do { - _Advance_bytes(_Last, -4); - unsigned long _Block = *static_cast(_Last); - unsigned long _Block_reversed = _byteswap_ulong(_Block); - *static_cast(_Dest) = _Block_reversed; - _Advance_bytes(_Dest, 4); - } while (_Dest != _Stop_at); - } - _Reverse_copy_tail(static_cast(_First), static_cast(_Last), static_cast(_Dest)); } @@ -413,18 +447,6 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( } while (_Dest != _Stop_at); } - if (_Byte_length(_First, _Last) >= 4) { - const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 2 << 2); - do { - _Advance_bytes(_Last, -4); - unsigned long _Block = *static_cast(_Last); - unsigned long _Block_reversed = _rotl(_Block, 16); - *static_cast(_Dest) = _Block_reversed; - _Advance_bytes(_Dest, 4); - } while (_Dest != _Stop_at); - } - _Reverse_copy_tail(static_cast(_First), static_cast(_Last), static_cast(_Dest)); } From df1d3cb9ece249fc1d6a65d554c4d4e6935cc589 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 11 Dec 2021 21:28:27 +0200 Subject: [PATCH 3/4] #include --- stl/src/vector_algorithms.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 40953d3f7d2..f32de3ae5c3 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -20,7 +20,6 @@ #include #endif // defined(_M_ARM64EC) #include -#include extern "C" long __isa_enabled; From 9d3d430f9602bcd028ee5dda1049414dde7d32be Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 11 Dec 2021 22:00:53 +0200 Subject: [PATCH 4/4] reverse copy --- stl/src/vector_algorithms.cpp | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f32de3ae5c3..b4d296b422f 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -399,7 +399,10 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( if (_Byte_length(_First, _Last) >= 16 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_char_sse = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); + const size_t _Size = _Byte_length(_First, _Last); + void* _Final_dest = _Dest; + _Advance_bytes(_Final_dest, _Size - 16); + _Advance_bytes(_Stop_at, _Size >> 4 << 4); do { _Advance_bytes(_Last, -16); const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); @@ -407,6 +410,12 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_1( _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); + + const __m128i _Block = _mm_loadu_si128(static_cast(_First)); + const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_char_sse); + _mm_storeu_si128(static_cast<__m128i*>(_Final_dest), _Block_reversed); + + return; } _Reverse_copy_tail(static_cast(_First), static_cast(_Last), @@ -436,7 +445,10 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( if (_Byte_length(_First, _Last) >= 16 && _bittest(&__isa_enabled, __ISA_AVAILABLE_SSE42)) { const __m128i _Reverse_short_sse = _mm_set_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); const void* _Stop_at = _Dest; - _Advance_bytes(_Stop_at, _Byte_length(_First, _Last) >> 4 << 4); + const size_t _Size = _Byte_length(_First, _Last); + void* _Final_dest = _Dest; + _Advance_bytes(_Final_dest, _Size - 16); + _Advance_bytes(_Stop_at, _Size >> 4 << 4); do { _Advance_bytes(_Last, -16); const __m128i _Block = _mm_loadu_si128(static_cast(_Last)); @@ -444,6 +456,12 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_2( _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Block_reversed); _Advance_bytes(_Dest, 16); } while (_Dest != _Stop_at); + + const __m128i _Block = _mm_loadu_si128(static_cast(_First)); + const __m128i _Block_reversed = _mm_shuffle_epi8(_Block, _Reverse_short_sse); + _mm_storeu_si128(static_cast<__m128i*>(_Final_dest), _Block_reversed); + + return; } _Reverse_copy_tail(static_cast(_First), static_cast(_Last),