From 3c7092a401ca76abc6b30026ae278c74641277fb Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 29 Jul 2020 22:49:59 +0300 Subject: [PATCH 01/10] Fix #1103 --- stl/inc/bit | 62 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 15 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index 0aa8f4451d1..4fc4eae9107 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -127,18 +127,9 @@ extern int __isa_available; } template -_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { +_NODISCARD int _Unchecked_x86_x64_countl_zero(const _Ty _Val) noexcept { constexpr int _Digits = numeric_limits<_Ty>::digits; -#ifndef __AVX2__ - const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; - // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero - // bsr has undefined output for zero - if (!_Have_lzcnt && _Val == 0) { - return _Digits; - } -#endif // __AVX2__ - // We use lzcnt (actually bsr if lzcnt is not supported) now that we know // we're not zero. We can do this because lzcnt and bsr share the same instruction // encoding. @@ -148,6 +139,18 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { return static_cast(__lzcnt(_Val)); } else { #ifdef _M_IX86 + static_assert(_Digits <= 32, "Should have handled this in _Checked_x86_x64_countl_zero"); +#else // ^^^ _M_IX86 / !_M_IX86 vvv + return static_cast(__lzcnt64(_Val)); +#endif // _M_IX86 + } + // note: we don't need to call a fallback here because + // all supported x86 processors at least have bsr/bsf +} + +template +_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { + if constexpr (sizeof(_Ty) > sizeof(void*)) { const unsigned int _High = _Val >> 32; const auto _Low = static_cast(_Val); if (_High == 0) { @@ -155,12 +158,41 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { } else { return _Checked_x86_x64_countl_zero(_High); } -#else // ^^^ _M_IX86 / !_M_IX86 vvv - return static_cast(__lzcnt64(_Val)); -#endif // _M_IX86 + } else { + int _Result = _Unchecked_x86_x64_countl_zero(_Val); +#ifndef __AVX2__ + static constexpr char _Have_lzcnt = 0; + static constexpr char _Dont_Have_lzcnt = 1; + static constexpr char _Deteting_lzcnt = 2; + + static char _Lzcnt_presence = _Deteting_lzcnt; + // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero + // bsr has undefined output for zero + auto _Lzcnt_presence_local = __iso_volatile_load8(&_Lzcnt_presence); + for (;;) { + if (_Lzcnt_presence_local == _Have_lzcnt) { + return _Result; + } else if (_Lzcnt_presence_local == _Dont_Have_lzcnt) { + if (_Val == 0) { + return 0; + } else { + if constexpr (_Digits < 16) { + _Result += (16 - _Digits) + } + // bsr counts from least significant bit, lzcnt counts from most significant bit, apply correction + constexpr int _Digits = numeric_limits<_Ty>::digits; + return _Digits - 1 - _Result; + } + } + + volatile unsigned _Test = 0x8000'0000; + _Lzcnt_presence_local = ((__lzcnt(_Test) == 0) ? _Have_lzcnt : _Dont_Have_lzcnt); + __iso_volatile_store8(&_Lzcnt_presence, _Lzcnt_presence_local); + } +#else // ^^^ !__AVX2__ / __AVX2__ vvv + return _Result; +#endif // ^^^ __AVX2__ ^^^ } - // note: we don't need to call a fallback here because - // all supported x86 processors at least have bsr/bsf } template From 0c67d0de63d6e12296ad37e38ff76d9b5ddc2ee7 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 29 Jul 2020 22:55:53 +0300 Subject: [PATCH 02/10] more volatiles to make compiler not optimize it --- stl/inc/bit | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index 4fc4eae9107..4da32af8cea 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -176,17 +176,16 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { if (_Val == 0) { return 0; } else { - if constexpr (_Digits < 16) { - _Result += (16 - _Digits) - } - // bsr counts from least significant bit, lzcnt counts from most significant bit, apply correction constexpr int _Digits = numeric_limits<_Ty>::digits; - return _Digits - 1 - _Result; + constexpr int _Digits_of_bsr = (_Digits < 16) ? 16 : _Digits; + // bsr counts from least significant bit, lzcnt counts from most significant bit, apply correction + return _Digits_of_bsr - 1 - _Result; } } - - volatile unsigned _Test = 0x8000'0000; - _Lzcnt_presence_local = ((__lzcnt(_Test) == 0) ? _Have_lzcnt : _Dont_Have_lzcnt); + + volatile unsigned _Test = 0x8000'0000; + volatile unsigned _Test_result = __lzcnt(_Test); + _Lzcnt_presence_local = ((_Test_result == 0) ? _Have_lzcnt : _Dont_Have_lzcnt); __iso_volatile_store8(&_Lzcnt_presence, _Lzcnt_presence_local); } #else // ^^^ !__AVX2__ / __AVX2__ vvv From f3bc66efe74a453e9aef313ea44d251acb0ab422 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Wed, 29 Jul 2020 22:57:37 +0300 Subject: [PATCH 03/10] clang format --- stl/inc/bit | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index 4da32af8cea..39fa64a3039 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -128,7 +128,7 @@ extern int __isa_available; template _NODISCARD int _Unchecked_x86_x64_countl_zero(const _Ty _Val) noexcept { - constexpr int _Digits = numeric_limits<_Ty>::digits; + constexpr int _Digits = numeric_limits<_Ty>::digits; // We use lzcnt (actually bsr if lzcnt is not supported) now that we know // we're not zero. We can do this because lzcnt and bsr share the same instruction @@ -159,11 +159,11 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { return _Checked_x86_x64_countl_zero(_High); } } else { - int _Result = _Unchecked_x86_x64_countl_zero(_Val); + int _Result = _Unchecked_x86_x64_countl_zero(_Val); #ifndef __AVX2__ - static constexpr char _Have_lzcnt = 0; + static constexpr char _Have_lzcnt = 0; static constexpr char _Dont_Have_lzcnt = 1; - static constexpr char _Deteting_lzcnt = 2; + static constexpr char _Deteting_lzcnt = 2; static char _Lzcnt_presence = _Deteting_lzcnt; // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero @@ -176,7 +176,7 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { if (_Val == 0) { return 0; } else { - constexpr int _Digits = numeric_limits<_Ty>::digits; + constexpr int _Digits = numeric_limits<_Ty>::digits; constexpr int _Digits_of_bsr = (_Digits < 16) ? 16 : _Digits; // bsr counts from least significant bit, lzcnt counts from most significant bit, apply correction return _Digits_of_bsr - 1 - _Result; From 20f51ae742e1fceaa56f921eb1132f955abea26b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 06:04:18 +0300 Subject: [PATCH 04/10] Start over again, will use BSR --- stl/inc/bit | 63 ++++++++++++++--------------------------------------- 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index 39fa64a3039..0aa8f4451d1 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -127,8 +127,17 @@ extern int __isa_available; } template -_NODISCARD int _Unchecked_x86_x64_countl_zero(const _Ty _Val) noexcept { - constexpr int _Digits = numeric_limits<_Ty>::digits; +_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { + constexpr int _Digits = numeric_limits<_Ty>::digits; + +#ifndef __AVX2__ + const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; + // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero + // bsr has undefined output for zero + if (!_Have_lzcnt && _Val == 0) { + return _Digits; + } +#endif // __AVX2__ // We use lzcnt (actually bsr if lzcnt is not supported) now that we know // we're not zero. We can do this because lzcnt and bsr share the same instruction @@ -139,18 +148,6 @@ _NODISCARD int _Unchecked_x86_x64_countl_zero(const _Ty _Val) noexcept { return static_cast(__lzcnt(_Val)); } else { #ifdef _M_IX86 - static_assert(_Digits <= 32, "Should have handled this in _Checked_x86_x64_countl_zero"); -#else // ^^^ _M_IX86 / !_M_IX86 vvv - return static_cast(__lzcnt64(_Val)); -#endif // _M_IX86 - } - // note: we don't need to call a fallback here because - // all supported x86 processors at least have bsr/bsf -} - -template -_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { - if constexpr (sizeof(_Ty) > sizeof(void*)) { const unsigned int _High = _Val >> 32; const auto _Low = static_cast(_Val); if (_High == 0) { @@ -158,40 +155,12 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { } else { return _Checked_x86_x64_countl_zero(_High); } - } else { - int _Result = _Unchecked_x86_x64_countl_zero(_Val); -#ifndef __AVX2__ - static constexpr char _Have_lzcnt = 0; - static constexpr char _Dont_Have_lzcnt = 1; - static constexpr char _Deteting_lzcnt = 2; - - static char _Lzcnt_presence = _Deteting_lzcnt; - // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero - // bsr has undefined output for zero - auto _Lzcnt_presence_local = __iso_volatile_load8(&_Lzcnt_presence); - for (;;) { - if (_Lzcnt_presence_local == _Have_lzcnt) { - return _Result; - } else if (_Lzcnt_presence_local == _Dont_Have_lzcnt) { - if (_Val == 0) { - return 0; - } else { - constexpr int _Digits = numeric_limits<_Ty>::digits; - constexpr int _Digits_of_bsr = (_Digits < 16) ? 16 : _Digits; - // bsr counts from least significant bit, lzcnt counts from most significant bit, apply correction - return _Digits_of_bsr - 1 - _Result; - } - } - - volatile unsigned _Test = 0x8000'0000; - volatile unsigned _Test_result = __lzcnt(_Test); - _Lzcnt_presence_local = ((_Test_result == 0) ? _Have_lzcnt : _Dont_Have_lzcnt); - __iso_volatile_store8(&_Lzcnt_presence, _Lzcnt_presence_local); - } -#else // ^^^ !__AVX2__ / __AVX2__ vvv - return _Result; -#endif // ^^^ __AVX2__ ^^^ +#else // ^^^ _M_IX86 / !_M_IX86 vvv + return static_cast(__lzcnt64(_Val)); +#endif // _M_IX86 } + // note: we don't need to call a fallback here because + // all supported x86 processors at least have bsr/bsf } template From 04928b8b9a83fe7b0483272f3aea1aeb47a81096 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 06:26:08 +0300 Subject: [PATCH 05/10] use BSR --- stl/inc/bit | 62 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index 0aa8f4451d1..e0ba61e0d12 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -127,21 +127,9 @@ extern int __isa_available; } template -_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { +_NODISCARD int _x86_x64_countl_zero_lzcnt(const _Ty _Val) noexcept { constexpr int _Digits = numeric_limits<_Ty>::digits; -#ifndef __AVX2__ - const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; - // lzcnt (when it doesn't fall back to bsr) is defined correctly for zero - // bsr has undefined output for zero - if (!_Have_lzcnt && _Val == 0) { - return _Digits; - } -#endif // __AVX2__ - - // We use lzcnt (actually bsr if lzcnt is not supported) now that we know - // we're not zero. We can do this because lzcnt and bsr share the same instruction - // encoding. if constexpr (_Digits <= 16) { return static_cast(__lzcnt16(_Val) - (16 - _Digits)); } else if constexpr (_Digits == 32) { @@ -151,16 +139,56 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { const unsigned int _High = _Val >> 32; const auto _Low = static_cast(_Val); if (_High == 0) { - return 32 + _Checked_x86_x64_countl_zero(_Low); + return 32 + _x86_x64_countl_zero_lzcnt(_Low); } else { - return _Checked_x86_x64_countl_zero(_High); + return _x86_x64_countl_zero_lzcnt(_High); } #else // ^^^ _M_IX86 / !_M_IX86 vvv return static_cast(__lzcnt64(_Val)); #endif // _M_IX86 } - // note: we don't need to call a fallback here because - // all supported x86 processors at least have bsr/bsf +} + +template +_NODISCARD int _x86_x64_countl_zero_bsr(const _Ty _Val) noexcept { + constexpr int _Digits = numeric_limits<_Ty>::digits; + + unsigned long _Result; + if constexpr (_Digits <= 32) { + if (!_BitScanReverse(&_Result, _Val)) { + _Result = _Digits; + } + } else { +#ifdef _M_IX86 + const unsigned int _High = _Val >> 32; + if (!_BitScanReverse(&_Result, _High)) { + const auto _Low = static_cast(_Val); + if (!_BitScanReverse(&_Result, _Low)) { + _Result = _Digits; + } + return static_cast(_Digits - _Result + 32); + } +#else // ^^^ _M_IX86 / !_M_IX86 vvv + if (!_BitScanReverse64(&_Result, _Val)) { + _Result = _Digits; + } +#endif // _M_IX86 + } + return static_cast(_Digits - _Result); +} + +template +_NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { +#ifdef __AVX2__ + return _x86_x64_countl_zero_lzcnt(_Val); +#else + const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; + if (_Have_lzcnt) { + return _x86_x64_countl_zero_lzcnt(_Val); + } else { + return _x86_x64_countl_zero_bsr(_Val); + } +#endif // __AVX2__ } template From c967665b53058d08f3fc134b07706124d629bd3e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 06:46:00 +0300 Subject: [PATCH 06/10] fix BSR branch --- stl/inc/bit | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index e0ba61e0d12..950d112d109 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -156,25 +156,26 @@ _NODISCARD int _x86_x64_countl_zero_bsr(const _Ty _Val) noexcept { unsigned long _Result; if constexpr (_Digits <= 32) { if (!_BitScanReverse(&_Result, _Val)) { - _Result = _Digits; + return _Digits; } } else { #ifdef _M_IX86 const unsigned int _High = _Val >> 32; - if (!_BitScanReverse(&_Result, _High)) { - const auto _Low = static_cast(_Val); - if (!_BitScanReverse(&_Result, _Low)) { - _Result = _Digits; - } - return static_cast(_Digits - _Result + 32); + if (_BitScanReverse(&_Result, _High)) { + return static_cast(31 - _Result); + } + + const auto _Low = static_cast(_Val); + if (!_BitScanReverse(&_Result, _Low)) { + return _Digits; } #else // ^^^ _M_IX86 / !_M_IX86 vvv if (!_BitScanReverse64(&_Result, _Val)) { - _Result = _Digits; + return _Digits; } #endif // _M_IX86 } - return static_cast(_Digits - _Result); + return static_cast(_Digits - 1 - _Result); } template From f88dd8695b31b0f50b14779e753f60a84b5b7dcd Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 06:46:27 +0300 Subject: [PATCH 07/10] clang format --- stl/inc/bit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/bit b/stl/inc/bit index 950d112d109..44e38eeb713 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -128,7 +128,7 @@ extern int __isa_available; template _NODISCARD int _x86_x64_countl_zero_lzcnt(const _Ty _Val) noexcept { - constexpr int _Digits = numeric_limits<_Ty>::digits; + constexpr int _Digits = numeric_limits<_Ty>::digits; if constexpr (_Digits <= 16) { return static_cast(__lzcnt16(_Val) - (16 - _Digits)); From 4857dd799e8acb89de88eb71261de93a41b3685e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 10:49:00 +0300 Subject: [PATCH 08/10] Update stl/inc/bit Co-authored-by: Stephan T. Lavavej --- stl/inc/bit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/bit b/stl/inc/bit index 44e38eeb713..9a0fe75284c 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -182,7 +182,7 @@ template _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { #ifdef __AVX2__ return _x86_x64_countl_zero_lzcnt(_Val); -#else +#else // __AVX2__ const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; if (_Have_lzcnt) { return _x86_x64_countl_zero_lzcnt(_Val); From 9b014eabd5eba71216ed9b7685e693f4cd16b06a Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 10:54:39 +0300 Subject: [PATCH 09/10] names --- stl/inc/bit | 22 +++++++++++----------- stl/inc/limits | 4 ++-- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/stl/inc/bit b/stl/inc/bit index 44e38eeb713..753799d950e 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -127,7 +127,7 @@ extern int __isa_available; } template -_NODISCARD int _x86_x64_countl_zero_lzcnt(const _Ty _Val) noexcept { +_NODISCARD int _Countl_zero_lzcnt(const _Ty _Val) noexcept { constexpr int _Digits = numeric_limits<_Ty>::digits; if constexpr (_Digits <= 16) { @@ -139,9 +139,9 @@ _NODISCARD int _x86_x64_countl_zero_lzcnt(const _Ty _Val) noexcept { const unsigned int _High = _Val >> 32; const auto _Low = static_cast(_Val); if (_High == 0) { - return 32 + _x86_x64_countl_zero_lzcnt(_Low); + return 32 + _Countl_zero_lzcnt(_Low); } else { - return _x86_x64_countl_zero_lzcnt(_High); + return _Countl_zero_lzcnt(_High); } #else // ^^^ _M_IX86 / !_M_IX86 vvv return static_cast(__lzcnt64(_Val)); @@ -150,7 +150,7 @@ _NODISCARD int _x86_x64_countl_zero_lzcnt(const _Ty _Val) noexcept { } template -_NODISCARD int _x86_x64_countl_zero_bsr(const _Ty _Val) noexcept { +_NODISCARD int _Countl_zero_bsr(const _Ty _Val) noexcept { constexpr int _Digits = numeric_limits<_Ty>::digits; unsigned long _Result; @@ -181,13 +181,13 @@ _NODISCARD int _x86_x64_countl_zero_bsr(const _Ty _Val) noexcept { template _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { #ifdef __AVX2__ - return _x86_x64_countl_zero_lzcnt(_Val); + return _Countl_zero_lzcnt(_Val); #else - const bool _Have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; - if (_Have_lzcnt) { - return _x86_x64_countl_zero_lzcnt(_Val); + const bool _Definitely_have_lzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; + if (_Definitely_have_lzcnt) { + return _Countl_zero_lzcnt(_Val); } else { - return _x86_x64_countl_zero_bsr(_Val); + return _Countl_zero_bsr(_Val); } #endif // __AVX2__ } @@ -196,8 +196,8 @@ template _NODISCARD int _Checked_x86_x64_popcount(const _Ty _Val) noexcept { constexpr int _Digits = numeric_limits<_Ty>::digits; #ifndef __AVX__ - const bool _Have_popcnt = __isa_available >= __ISA_AVAILABLE_SSE42; - if (!_Have_popcnt) { + const bool _Definitely_have_popcnt = __isa_available >= __ISA_AVAILABLE_SSE42; + if (!_Definitely_have_popcnt) { return _Popcount_fallback(_Val); } #endif // !defined(__AVX__) diff --git a/stl/inc/limits b/stl/inc/limits index 67a18dd7173..0b78a1b0396 100644 --- a/stl/inc/limits +++ b/stl/inc/limits @@ -1061,8 +1061,8 @@ _NODISCARD int _Checked_x86_x64_countr_zero(const _Ty _Val) noexcept { constexpr _Ty _Max = (numeric_limits<_Ty>::max)(); #ifndef __AVX2__ - const bool _Have_tzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; - if (!_Have_tzcnt && _Val == 0) { + const bool _Definitely_have_tzcnt = __isa_available >= __ISA_AVAILABLE_AVX2; + if (!_Definitely_have_tzcnt && _Val == 0) { return _Digits; } #endif // __AVX2__ From aeecd23d1efbabd106b58577336db363db5b4775 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 30 Jul 2020 10:56:56 +0300 Subject: [PATCH 10/10] clang format --- stl/inc/bit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/bit b/stl/inc/bit index 291e2fc91ca..9983b889003 100644 --- a/stl/inc/bit +++ b/stl/inc/bit @@ -194,7 +194,7 @@ _NODISCARD int _Checked_x86_x64_countl_zero(const _Ty _Val) noexcept { template _NODISCARD int _Checked_x86_x64_popcount(const _Ty _Val) noexcept { - constexpr int _Digits = numeric_limits<_Ty>::digits; + constexpr int _Digits = numeric_limits<_Ty>::digits; #ifndef __AVX__ const bool _Definitely_have_popcnt = __isa_available >= __ISA_AVAILABLE_SSE42; if (!_Definitely_have_popcnt) {