diff --git a/stl/inc/xutility b/stl/inc/xutility index da3e663bdec..17feb4cd9c5 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -100,7 +100,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_ROTATE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_SEARCH _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_SEARCH_N _VECTORIZED_FOR_X64_X86 -#define _VECTORIZED_SWAP_RANGES _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_SWAP_RANGES _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_UNIQUE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_UNIQUE_COPY _VECTORIZED_FOR_X64_X86 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index ea38e0e39d7..ac2d743a32d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -5,26 +5,29 @@ #error _M_CEE_PURE should not be defined when compiling vector_algorithms.cpp. #endif -#if defined(_M_IX86) || defined(_M_X64) // NB: includes _M_ARM64EC +#ifndef _DEBUG +#pragma optimize("t", on) // TRANSITION, GH-2108: Override /Os with /Ot for this TU before any function definitions +#endif + #include <__msvc_minmax.hpp> #include #include #include #include -#ifndef _M_ARM64EC +#if !defined(_M_ARM64) && !defined(_M_ARM64EC) #include #include extern "C" long __isa_enabled; +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ -#ifndef _DEBUG -#pragma optimize("t", on) // Override /Os with /Ot for this TU -#endif // !defined(_DEBUG) -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +#ifdef _M_ARM64 +#include +#endif namespace { -#ifndef _M_ARM64EC +#if !defined(_M_ARM64) && !defined(_M_ARM64EC) bool _Use_avx2() noexcept { return __isa_enabled & (1 << __ISA_AVAILABLE_AVX2); } @@ -51,7 +54,7 @@ namespace { return _mm256_loadu_si256(reinterpret_cast( reinterpret_cast(_Tail_masks) + (32 - _Count_in_bytes))); } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ size_t _Byte_length(const void* const _First, const void* const _Last) noexcept { return static_cast(_Last) - static_cast(_First); @@ -78,6 +81,86 @@ namespace { extern "C" { +#ifdef _M_ARM64 +__declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( + void* _First1, void* const _Last1, void* _First2) noexcept { + if (_Byte_length(_First1, _Last1) >= 64) { + constexpr size_t _Mask_64 = ~((static_cast(1) << 6) - 1); + const void* _Stop_at = _First1; + _Advance_bytes(_Stop_at, _Byte_length(_First1, _Last1) & _Mask_64); + do { + const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0); + const uint8x16_t _Left2 = vld1q_u8(static_cast(_First1) + 16); + const uint8x16_t _Left3 = vld1q_u8(static_cast(_First1) + 32); + const uint8x16_t _Left4 = vld1q_u8(static_cast(_First1) + 48); + const uint8x16_t _Right1 = vld1q_u8(static_cast(_First2) + 0); + const uint8x16_t _Right2 = vld1q_u8(static_cast(_First2) + 16); + const uint8x16_t _Right3 = vld1q_u8(static_cast(_First2) + 32); + const uint8x16_t _Right4 = vld1q_u8(static_cast(_First2) + 48); + vst1q_u8(static_cast(_First1) + 0, _Right1); + vst1q_u8(static_cast(_First1) + 16, _Right2); + vst1q_u8(static_cast(_First1) + 32, _Right3); + vst1q_u8(static_cast(_First1) + 48, _Right4); + vst1q_u8(static_cast(_First2) + 0, _Left1); + vst1q_u8(static_cast(_First2) + 16, _Left2); + vst1q_u8(static_cast(_First2) + 32, _Left3); + vst1q_u8(static_cast(_First2) + 48, _Left4); + _Advance_bytes(_First1, 64); + _Advance_bytes(_First2, 64); + } while (_First1 != _Stop_at); + } + + if (_Byte_length(_First1, _Last1) >= 32) { + const uint8x16_t _Left1 = vld1q_u8(static_cast(_First1) + 0); + const uint8x16_t _Left2 = vld1q_u8(static_cast(_First1) + 16); + const uint8x16_t _Right1 = vld1q_u8(static_cast(_First2) + 0); + const uint8x16_t _Right2 = vld1q_u8(static_cast(_First2) + 16); + vst1q_u8(static_cast(_First1) + 0, _Right1); + vst1q_u8(static_cast(_First1) + 16, _Right2); + vst1q_u8(static_cast(_First2) + 0, _Left1); + vst1q_u8(static_cast(_First2) + 16, _Left2); + _Advance_bytes(_First1, 32); + _Advance_bytes(_First2, 32); + } + + if (_Byte_length(_First1, _Last1) >= 16) { + const uint8x16_t _Left = vld1q_u8(static_cast(_First1)); + const uint8x16_t _Right = vld1q_u8(static_cast(_First2)); + vst1q_u8(static_cast(_First1), _Right); + vst1q_u8(static_cast(_First2), _Left); + _Advance_bytes(_First1, 16); + _Advance_bytes(_First2, 16); + } + + if (_Byte_length(_First1, _Last1) >= 8) { + const uint8x8_t _Left = vld1_u8(static_cast(_First1)); + const uint8x8_t _Right = vld1_u8(static_cast(_First2)); + vst1_u8(static_cast(_First1), _Right); + vst1_u8(static_cast(_First2), _Left); + _Advance_bytes(_First1, 8); + _Advance_bytes(_First2, 8); + } + + if (_Byte_length(_First1, _Last1) >= 4) { + uint32x2_t _Left = vdup_n_u32(0); + uint32x2_t _Right = vdup_n_u32(0); + _Left = vld1_lane_u32(static_cast(_First1), _Left, 0); + _Right = vld1_lane_u32(static_cast(_First2), _Right, 0); + vst1_lane_u32(static_cast(_First1), _Right, 0); + vst1_lane_u32(static_cast(_First2), _Left, 0); + _Advance_bytes(_First1, 4); + _Advance_bytes(_First2, 4); + } + + auto _First1c = static_cast(_First1); + auto _First2c = static_cast(_First2); + for (; _First1c != _Last1; ++_First1c, ++_First2c) { + const unsigned char _Ch = *_First1c; + *_First1c = *_First2c; + *_First2c = _Ch; + } +} +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( void* _First1, void* const _Last1, void* _First2) noexcept { #ifndef _M_ARM64EC @@ -157,15 +240,17 @@ __declspec(noalias) void __cdecl __std_swap_ranges_trivially_swappable_noalias( } } -// TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility +// TRANSITION, ABI: __std_swap_ranges_trivially_swappable() is preserved for binary compatibility (x64/x86/ARM64EC) void* __cdecl __std_swap_ranges_trivially_swappable( void* const _First1, void* const _Last1, void* const _First2) noexcept { __std_swap_ranges_trivially_swappable_noalias(_First1, _Last1, _First2); return static_cast(_First2) + (static_cast(_Last1) - static_cast(_First1)); } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // extern "C" +#ifndef _M_ARM64 namespace { namespace _Rotating { void _Swap_3_ranges(void* _First1, void* const _Last1, void* _First2, void* _First3) noexcept { @@ -7694,4 +7779,4 @@ __declspec(noalias) bool __stdcall __std_bitset_from_string_2(void* const _Dest, } } // extern "C" -#endif // defined(_M_IX86) || defined(_M_X64) +#endif // ^^^ !defined(_M_ARM64) ^^^