diff --git a/stl/inc/atomic b/stl/inc/atomic index 5e229aefe28..11e584ae0a1 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -414,24 +414,45 @@ void _Atomic_wait_direct( } #endif // _HAS_CXX20 -#if 1 // TRANSITION, ABI +#if 1 // TRANSITION, ABI, GH-1151 inline void _Atomic_lock_acquire(long& _Spinlock) noexcept { - while (_InterlockedExchange(&_Spinlock, 1)) { - _YIELD_PROCESSOR(); +#if defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC)) + // Algorithm from Intel(R) 64 and IA-32 Architectures Optimization Reference Manual, May 2020 + // Example 2-4. Contended Locks with Increasing Back-off Example - Improved Version, page 2-22 + // The code in mentioned manual is covered by the 0BSD license. + int _Current_backoff = 1; + const int _Max_backoff = 64; + while (_InterlockedExchange(&_Spinlock, 1) != 0) { + while (__iso_volatile_load32(&reinterpret_cast(_Spinlock)) != 0) { + for (int _Count_down = _Current_backoff; _Count_down != 0; --_Count_down) { + _mm_pause(); + } + _Current_backoff = _Current_backoff < _Max_backoff ? _Current_backoff << 1 : _Max_backoff; + } } +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) + while (_InterlockedExchange(&_Spinlock, 1) != 0) { // TRANSITION, GH-1133: _InterlockedExchange_acq + while (__iso_volatile_load32(&reinterpret_cast(_Spinlock)) != 0) { + __yield(); + } + } +#else // ^^^ defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) ^^^ +#error Unsupported hardware +#endif } inline void _Atomic_lock_release(long& _Spinlock) noexcept { -#if defined(_M_ARM) || defined(_M_ARM64) +#if defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC)) + _InterlockedExchange(&_Spinlock, 0); // TRANSITION, GH-1133: same as ARM +#elif defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) _Memory_barrier(); __iso_volatile_store32(reinterpret_cast(&_Spinlock), 0); - _Memory_barrier(); -#else // ^^^ ARM32/ARM64 hardware / x86/x64 hardware vvv - _InterlockedExchange(&_Spinlock, 0); -#endif // hardware + _Memory_barrier(); // TRANSITION, GH-1133: remove +#else // ^^^ defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) ^^^ +#error Unsupported hardware +#endif } - inline void _Atomic_lock_acquire(_Smtx_t* _Spinlock) noexcept { _Smtx_lock_exclusive(_Spinlock); } diff --git a/stl/inc/xatomic.h b/stl/inc/xatomic.h index af41c5af67a..c5f97d76db9 100644 --- a/stl/inc/xatomic.h +++ b/stl/inc/xatomic.h @@ -28,7 +28,14 @@ _STL_DISABLE_CLANG_WARNINGS #define _INTRIN_ACQUIRE(x) x #define _INTRIN_RELEASE(x) x #define _INTRIN_ACQ_REL(x) x +#ifdef _M_CEE_PURE #define _YIELD_PROCESSOR() +#else // ^^^ _M_CEE_PURE / !_M_CEE_PURE vvv +#if 1 // TRANSITION, VS 2019 16.10 +extern "C" void _mm_pause(void); +#endif // TRANSITION, VS 2019 16.10 +#define _YIELD_PROCESSOR() _mm_pause() +#endif // ^^^ !_M_CEE_PURE ^^^ #elif defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) #define _INTRIN_RELAXED(x) _CONCAT(x, _nf) diff --git a/tests/std/include/test_atomic_wait.hpp b/tests/std/include/test_atomic_wait.hpp index a3d9b4471c2..7d2150b3d24 100644 --- a/tests/std/include/test_atomic_wait.hpp +++ b/tests/std/include/test_atomic_wait.hpp @@ -43,7 +43,7 @@ void test_atomic_wait_func_impl(UnderlyingType& old_value, const UnderlyingType // timing assumption that the main thread evaluates the `wait(old_value)` before this timeout expires std::this_thread::sleep_for(waiting_duration); add_seq('6'); -#endif // CAN_FAIL_ON_TIMING_ASSUMPTION +#endif }); a.wait(old_value);