From cbe2ee99caf122555a305d18f69e57c75b3fe5ec Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Fri, 12 Dec 2025 11:08:16 -0800 Subject: [PATCH 01/19] Update tests for upcoming compiler changes to `consteval` (#5960) --- .../tests/P1208R6_source_location/test.cpp | 22 ++++++------------- .../VSO_1775715_user_defined_modules/user.ixx | 5 +++++ 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/tests/std/tests/P1208R6_source_location/test.cpp b/tests/std/tests/P1208R6_source_location/test.cpp index c09c70432d8..128f4a1abeb 100644 --- a/tests/std/tests/P1208R6_source_location/test.cpp +++ b/tests/std/tests/P1208R6_source_location/test.cpp @@ -96,19 +96,15 @@ constexpr void sloc_constructor_test() { assert(x.loc.column() == 13); #endif // ^^^ !defined(__EDG__) ^^^ #if _USE_DETAILED_FUNCTION_NAME_IN_SOURCE_LOCATION -#if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 - if (is_constant_evaluated()) { - assert(x.loc.function_name() == "int __cdecl main(void)"sv); - } else +#if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 (fixed in MSVC Compiler 19.51) + if (!is_constant_evaluated()) #endif // ^^^ workaround ^^^ { assert(x.loc.function_name() == "void __cdecl sloc_constructor_test(void)"sv); } #else // ^^^ detailed / basic vvv -#if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 - if (is_constant_evaluated()) { - assert(x.loc.function_name() == "main"sv); - } else +#if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 (fixed in MSVC Compiler 19.51) + if (!is_constant_evaluated()) #endif // ^^^ workaround ^^^ { assert(x.loc.function_name() == "sloc_constructor_test"sv); @@ -146,19 +142,15 @@ constexpr void sub_member_test() { assert(s.x.loc.column() == 14); #endif // ^^^ !defined(__EDG__) ^^^ #if _USE_DETAILED_FUNCTION_NAME_IN_SOURCE_LOCATION -#if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 - if (is_constant_evaluated()) { - assert(s.x.loc.function_name() == "int __cdecl main(void)"sv); - } else +#if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 (fixed in MSVC Compiler 19.51) + if (!is_constant_evaluated()) #endif // ^^^ workaround ^^^ { assert(s.x.loc.function_name() == "void __cdecl sub_member_test(void)"sv); } #else // ^^^ detailed / basic vvv #if !defined(__clang__) && !defined(__EDG__) // TRANSITION, VSO-1285783 - if (is_constant_evaluated()) { - assert(s.x.loc.function_name() == "main"sv); - } else + if (!is_constant_evaluated()) #endif // ^^^ workaround ^^^ { assert(s.x.loc.function_name() == "sub_member_test"sv); diff --git a/tests/std/tests/VSO_1775715_user_defined_modules/user.ixx b/tests/std/tests/VSO_1775715_user_defined_modules/user.ixx index bfe86c63f97..3e0e3e7eb82 100644 --- a/tests/std/tests/VSO_1775715_user_defined_modules/user.ixx +++ b/tests/std/tests/VSO_1775715_user_defined_modules/user.ixx @@ -16,6 +16,11 @@ namespace user { assert(test_environment_preparer.succeeded()); } + // TRANSITION, VSO-2649620 reachability problem in modules + // The compiler does not export the bindings for `operator==` for a string + // and character pointer unless we force it with this export. + export using std::operator==; + // DevCom-10313766 VSO-1775715 "Using std::format in a module // requires including header in .cpp files using that module" export template From 641410dbe53e00d9671a96485a5110c5fb62f01e Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 8 Jan 2026 18:52:33 +0200 Subject: [PATCH 02/19] Use division by 100 in `to_string` for integers (#5691) Co-authored-by: Stephan T. Lavavej --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/integer_to_string.cpp | 100 ++++++++++++++++++ stl/inc/xmemory | 56 ++++++++-- .../tests/Dev11_0835323_to_string/test.cpp | 28 +++++ 4 files changed, 176 insertions(+), 9 deletions(-) create mode 100644 benchmarks/src/integer_to_string.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 59d84e3aec0..ce912d68979 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -113,6 +113,7 @@ add_benchmark(find_and_count src/find_and_count.cpp) add_benchmark(find_first_of src/find_first_of.cpp) add_benchmark(has_single_bit src/has_single_bit.cpp) add_benchmark(includes src/includes.cpp) +add_benchmark(integer_to_string src/integer_to_string.cpp) add_benchmark(iota src/iota.cpp) add_benchmark(is_sorted_until src/is_sorted_until.cpp) add_benchmark(locale_classic src/locale_classic.cpp) diff --git a/benchmarks/src/integer_to_string.cpp b/benchmarks/src/integer_to_string.cpp new file mode 100644 index 00000000000..8466545ba2d --- /dev/null +++ b/benchmarks/src/integer_to_string.cpp @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace std; + +template +auto generate_array() { + array a; + + mt19937_64 gen; + lognormal_distribution dis(M, S); + auto get_clamped_value = [&] { + for (;;) { + const double dbl = floor(dis(gen)); + constexpr auto max_val = static_cast(numeric_limits::max()); + if (dbl <= max_val) { + return static_cast(dbl); + } + } + }; + ranges::generate(a, get_clamped_value); + + if constexpr (is_signed_v) { + bernoulli_distribution b(0.5); + ranges::for_each(a, [&](T& v) { v *= (b(gen) ? -1 : 1); }); + } + + return a; +} + +template +void internal_integer_to_buff(benchmark::State& state) { + auto a = generate_array(); + + CharT buff[20]; // can hold -2^63 and 2^64 - 1 + auto buff_end = end(buff); + + auto it = a.begin(); + for (auto _ : state) { + auto i = *it; + benchmark::DoNotOptimize(i); + auto s = std::_UIntegral_to_buff(buff_end, i); + benchmark::DoNotOptimize(s); + + ++it; + if (it == a.end()) { + it = a.begin(); + } + } +} + +template +void integer_to_string(benchmark::State& state) { + auto a = generate_array(); + + auto it = a.begin(); + for (auto _ : state) { + auto i = *it; + benchmark::DoNotOptimize(i); + auto s = to_string(i); + benchmark::DoNotOptimize(s); + + ++it; + if (it == a.end()) { + it = a.begin(); + } + } +} + +BENCHMARK(internal_integer_to_buff); +BENCHMARK(internal_integer_to_buff); +BENCHMARK(internal_integer_to_buff); +BENCHMARK(internal_integer_to_buff); + +BENCHMARK(internal_integer_to_buff); +BENCHMARK(internal_integer_to_buff); +BENCHMARK(internal_integer_to_buff); +BENCHMARK(internal_integer_to_buff); + +BENCHMARK(integer_to_string); +BENCHMARK(integer_to_string); +BENCHMARK(integer_to_string); +BENCHMARK(integer_to_string); + +BENCHMARK(integer_to_string); +BENCHMARK(integer_to_string); +BENCHMARK(integer_to_string); +BENCHMARK(integer_to_string); + +BENCHMARK_MAIN(); diff --git a/stl/inc/xmemory b/stl/inc/xmemory index fcd92f799af..2e86091d9d0 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -2762,6 +2762,24 @@ namespace ranges { } // namespace ranges #endif // _HAS_CXX23 +template +struct _Digit_pair_table { + _Elem _Data[100][2]; + + constexpr explicit _Digit_pair_table() : _Data{} { + for (int _Idx = 0; _Idx != 100; ++_Idx) { + _Data[_Idx][0] = static_cast<_Elem>('0' + _Idx / 10); + _Data[_Idx][1] = static_cast<_Elem>('0' + _Idx % 10); + } + } + + _Digit_pair_table(const _Digit_pair_table&) = delete; + _Digit_pair_table& operator=(const _Digit_pair_table&) = delete; +}; + +template +constexpr _Digit_pair_table<_Elem> _Digit_pairs{}; + template _NODISCARD _Elem* _UIntegral_to_buff(_Elem* _RNext, _UTy _UVal) { // used by both to_string and thread::id output // format _UVal into buffer *ending at* _RNext @@ -2773,23 +2791,43 @@ _NODISCARD _Elem* _UIntegral_to_buff(_Elem* _RNext, _UTy _UVal) { // used by bot if constexpr (sizeof(_UTy) > 4) { // For 64-bit numbers, work in chunks to avoid 64-bit divisions. while (_UVal > 0xFFFFFFFFU) { - auto _UVal_chunk = static_cast(_UVal % 1000000000); - _UVal /= 1000000000; - - for (int _Idx = 0; _Idx != 9; ++_Idx) { - *--_RNext = static_cast<_Elem>('0' + _UVal_chunk % 10); - _UVal_chunk /= 10; + auto _UVal_chunk = static_cast(_UVal % 100000000); + _UVal /= 100000000; + + for (int _Idx = 0; _Idx != 3; ++_Idx) { + const unsigned long _UVal_chunk_part = _UVal_chunk % 100; + _UVal_chunk /= 100; + _RNext -= 2; + _CSTD memcpy(_RNext, _Digit_pairs<_Elem>._Data[_UVal_chunk_part], 2 * sizeof(_Elem)); } + + _RNext -= 2; + _CSTD memcpy(_RNext, _Digit_pairs<_Elem>._Data[_UVal_chunk], 2 * sizeof(_Elem)); } } auto _UVal_trunc = static_cast(_UVal); #endif // ^^^ !defined(_WIN64) ^^^ + // If we have a single digit, print [0, 9] and return. (This is necessary to correctly handle 0.) + if (_UVal_trunc < 10) { + *--_RNext = static_cast<_Elem>('0' + _UVal_trunc); + return _RNext; + } + + // Print one or more pairs of digits. do { - *--_RNext = static_cast<_Elem>('0' + _UVal_trunc % 10); - _UVal_trunc /= 10; - } while (_UVal_trunc != 0); + const unsigned long _UVal_trunc_part = _UVal_trunc % 100; + _UVal_trunc /= 100; + _RNext -= 2; + _CSTD memcpy(_RNext, _Digit_pairs<_Elem>._Data[_UVal_trunc_part], 2 * sizeof(_Elem)); + } while (_UVal_trunc >= 10); + + // If we have an unpaired digit, print it. For example, 1729 is printed as 17 29, and 19937 is printed as 1 99 37. + if (_UVal_trunc != 0) { + *--_RNext = static_cast<_Elem>('0' + _UVal_trunc); + } + return _RNext; } _STD_END diff --git a/tests/std/tests/Dev11_0835323_to_string/test.cpp b/tests/std/tests/Dev11_0835323_to_string/test.cpp index 10d52236e79..fa270b4cfa0 100644 --- a/tests/std/tests/Dev11_0835323_to_string/test.cpp +++ b/tests/std/tests/Dev11_0835323_to_string/test.cpp @@ -144,6 +144,34 @@ int main() { L"2304583236903222948165808559332123348274797826204144723168738177180919299881250404026184124858368.000000"); assert(to_wstring(numeric_limits::infinity()) == L"inf"); + // Exercise the logic for printing digit pairs, and the optimization for printing 64-bit values on 32-bit targets. + assert(to_string(0ULL) == "0"); + assert(to_string(1ULL) == "1"); + assert(to_string(12ULL) == "12"); + assert(to_string(123ULL) == "123"); + assert(to_string(1234ULL) == "1234"); + assert(to_string(12345ULL) == "12345"); + assert(to_string(123456ULL) == "123456"); + assert(to_string(1234567ULL) == "1234567"); + assert(to_string(12345678ULL) == "12345678"); + assert(to_string(123456789ULL) == "123456789"); + assert(to_string(1234567890ULL) == "1234567890"); + assert(to_string(4294967295ULL) == "4294967295"); // 2^32-1 + assert(to_string(4294967296ULL) == "4294967296"); + assert(to_string(12345678901ULL) == "12345678901"); + assert(to_string(123456789012ULL) == "123456789012"); + assert(to_string(1234567890123ULL) == "1234567890123"); + assert(to_string(12345678901234ULL) == "12345678901234"); + assert(to_string(123456789012345ULL) == "123456789012345"); + assert(to_string(1234567890123456ULL) == "1234567890123456"); + assert(to_string(12345678901234567ULL) == "12345678901234567"); + assert(to_string(123456789012345678ULL) == "123456789012345678"); + assert(to_string(429496729599999999ULL) == "429496729599999999"); // 2^32-1 followed by 8 digits + assert(to_string(429496729600000000ULL) == "429496729600000000"); + assert(to_string(1234567890123456789ULL) == "1234567890123456789"); + assert(to_string(4294967295999999999ULL) == "4294967295999999999"); // 2^32-1 followed by 9 digits + assert(to_string(4294967296000000000ULL) == "4294967296000000000"); + assert(to_string(12345678901234567890ULL) == "12345678901234567890"); // Also test DevDiv-875295 ": std::stof returns 1.#INF instead of throwing out_of_range [libcxx]". From 7364b422d8c4d2bf8c5149aa8ac81167ebf87b73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20Michal?= Date: Thu, 8 Jan 2026 18:00:22 +0100 Subject: [PATCH 03/19] Implement P3016R6 Resolve Inconsistencies In `begin`/`end` For `valarray` And Braced Initializer Lists (#5847) Co-authored-by: Stephan T. Lavavej --- stl/inc/initializer_list | 17 ++-- stl/inc/valarray | 52 +++++------- stl/inc/xutility | 61 +++++--------- stl/inc/yvals_core.h | 3 + tests/libcxx/expected_results.txt | 3 + tests/std/test.lst | 1 + .../tests/Dev11_1074023_constexpr/test.cpp | 2 + .../P3016R6_inconsistent_begin_end/env.lst | 4 + .../P3016R6_inconsistent_begin_end/test.cpp | 83 +++++++++++++++++++ .../test.compile.pass.cpp | 4 + 10 files changed, 146 insertions(+), 84 deletions(-) create mode 100644 tests/std/tests/P3016R6_inconsistent_begin_end/env.lst create mode 100644 tests/std/tests/P3016R6_inconsistent_begin_end/test.cpp diff --git a/stl/inc/initializer_list b/stl/inc/initializer_list index b5d6ae7370e..9a6279a4a2f 100644 --- a/stl/inc/initializer_list +++ b/stl/inc/initializer_list @@ -51,20 +51,19 @@ public: return static_cast(_Last - _First); } + _NODISCARD constexpr bool empty() const noexcept { + return _First == _Last; + } + + _NODISCARD constexpr const _Elem* data() const noexcept { + return _First; + } + private: const _Elem* _First; const _Elem* _Last; }; -_EXPORT_STD template -_NODISCARD constexpr const _Elem* begin(initializer_list<_Elem> _Ilist) noexcept { - return _Ilist.begin(); -} - -_EXPORT_STD template -_NODISCARD constexpr const _Elem* end(initializer_list<_Elem> _Ilist) noexcept { - return _Ilist.end(); -} _STD_END // TRANSITION, non-_Ugly attribute tokens diff --git a/stl/inc/valarray b/stl/inc/valarray index a6a0909d9a8..73a6a0e8fa8 100644 --- a/stl/inc/valarray +++ b/stl/inc/valarray @@ -59,19 +59,9 @@ class valarray { // store array with various indexing options public: friend _Tidy_deallocate_guard; - template - friend _Ty2* begin(valarray<_Ty2>& _Array) noexcept /* strengthened */; - - template - friend const _Ty2* begin(const valarray<_Ty2>& _Array) noexcept /* strengthened */; - - template - friend _Ty2* end(valarray<_Ty2>& _Array) noexcept /* strengthened */; - - template - friend const _Ty2* end(const valarray<_Ty2>& _Array) noexcept /* strengthened */; - - using value_type = _Ty; + using value_type = _Ty; + using iterator = _Ty*; + using const_iterator = const _Ty*; valarray() = default; // construct empty valarray @@ -411,6 +401,22 @@ public: return _Mysize; } + _NODISCARD iterator begin() noexcept /* strengthened */ { + return _Myptr; + } + + _NODISCARD const_iterator begin() const noexcept /* strengthened */ { + return _Myptr; + } + + _NODISCARD iterator end() noexcept /* strengthened */ { + return _Myptr + _Mysize; + } + + _NODISCARD const_iterator end() const noexcept /* strengthened */ { + return _Myptr + _Mysize; + } + _NODISCARD const _Ty& operator[](size_t _Off) const noexcept /* strengthened */ { #if _MSVC_STL_HARDENING_VALARRAY || _ITERATOR_DEBUG_LEVEL != 0 _STL_VERIFY(_Off < _Mysize, "valarray subscript out of range"); @@ -612,26 +618,6 @@ void swap(valarray<_Ty>& _Left, valarray<_Ty>& _Right) noexcept { _Left.swap(_Right); } -_EXPORT_STD template -_NODISCARD _Ty* begin(valarray<_Ty>& _Array) noexcept /* strengthened */ { - return _Array._Myptr; -} - -_EXPORT_STD template -_NODISCARD const _Ty* begin(const valarray<_Ty>& _Array) noexcept /* strengthened */ { - return _Array._Myptr; -} - -_EXPORT_STD template -_NODISCARD _Ty* end(valarray<_Ty>& _Array) noexcept /* strengthened */ { - return _Array._Myptr + _Array.size(); -} - -_EXPORT_STD template -_NODISCARD const _Ty* end(const valarray<_Ty>& _Array) noexcept /* strengthened */ { - return _Array._Myptr + _Array.size(); -} - _EXPORT_STD template _NODISCARD valarray<_Ty> operator*(const valarray<_Ty>& _Left, const typename valarray<_Ty>::value_type& _Right) { const size_t _Size = _Left.size(); diff --git a/stl/inc/xutility b/stl/inc/xutility index 53a64c6c787..b88e899ea2b 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -2142,26 +2142,23 @@ constexpr bool disable_sized_sentinel_for, reverse_ite #endif // _HAS_CXX20 _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto begin(_Container& _Cont) noexcept(noexcept(_Cont.begin())) /* strengthened */ - -> decltype(_Cont.begin()) { +_NODISCARD _CONSTEXPR17 auto begin(_Container& _Cont) noexcept(noexcept(_Cont.begin())) -> decltype(_Cont.begin()) { return _Cont.begin(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto begin(const _Container& _Cont) noexcept(noexcept(_Cont.begin())) /* strengthened */ +_NODISCARD _CONSTEXPR17 auto begin(const _Container& _Cont) noexcept(noexcept(_Cont.begin())) -> decltype(_Cont.begin()) { return _Cont.begin(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto end(_Container& _Cont) noexcept(noexcept(_Cont.end())) /* strengthened */ - -> decltype(_Cont.end()) { +_NODISCARD _CONSTEXPR17 auto end(_Container& _Cont) noexcept(noexcept(_Cont.end())) -> decltype(_Cont.end()) { return _Cont.end(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto end(const _Container& _Cont) noexcept(noexcept(_Cont.end())) /* strengthened */ - -> decltype(_Cont.end()) { +_NODISCARD _CONSTEXPR17 auto end(const _Container& _Cont) noexcept(noexcept(_Cont.end())) -> decltype(_Cont.end()) { return _Cont.end(); } @@ -2188,66 +2185,60 @@ _NODISCARD constexpr auto cend(const _Container& _Cont) noexcept(noexcept(_STD e } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto rbegin(_Container& _Cont) noexcept(noexcept(_Cont.rbegin())) /* strengthened */ - -> decltype(_Cont.rbegin()) { +_NODISCARD _CONSTEXPR17 auto rbegin(_Container& _Cont) noexcept(noexcept(_Cont.rbegin())) -> decltype(_Cont.rbegin()) { return _Cont.rbegin(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto rbegin(const _Container& _Cont) noexcept(noexcept(_Cont.rbegin())) /* strengthened */ +_NODISCARD _CONSTEXPR17 auto rbegin(const _Container& _Cont) noexcept(noexcept(_Cont.rbegin())) -> decltype(_Cont.rbegin()) { return _Cont.rbegin(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto rend(_Container& _Cont) noexcept(noexcept(_Cont.rend())) /* strengthened */ - -> decltype(_Cont.rend()) { +_NODISCARD _CONSTEXPR17 auto rend(_Container& _Cont) noexcept(noexcept(_Cont.rend())) -> decltype(_Cont.rend()) { return _Cont.rend(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto rend(const _Container& _Cont) noexcept(noexcept(_Cont.rend())) /* strengthened */ - -> decltype(_Cont.rend()) { +_NODISCARD _CONSTEXPR17 auto rend(const _Container& _Cont) noexcept(noexcept(_Cont.rend())) -> decltype(_Cont.rend()) { return _Cont.rend(); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 reverse_iterator<_Ty*> rbegin(_Ty (&_Array)[_Size]) noexcept /* strengthened */ { +_NODISCARD _CONSTEXPR17 reverse_iterator<_Ty*> rbegin(_Ty (&_Array)[_Size]) noexcept { return reverse_iterator<_Ty*>(_Array + _Size); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 reverse_iterator<_Ty*> rend(_Ty (&_Array)[_Size]) noexcept /* strengthened */ { +_NODISCARD _CONSTEXPR17 reverse_iterator<_Ty*> rend(_Ty (&_Array)[_Size]) noexcept { return reverse_iterator<_Ty*>(_Array); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 reverse_iterator rbegin(initializer_list<_Elem> _Ilist) noexcept -/* strengthened */ { +_NODISCARD _CONSTEXPR17 reverse_iterator rbegin(initializer_list<_Elem> _Ilist) noexcept { return reverse_iterator(_Ilist.end()); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 reverse_iterator rend(initializer_list<_Elem> _Ilist) noexcept -/* strengthened */ { +_NODISCARD _CONSTEXPR17 reverse_iterator rend(initializer_list<_Elem> _Ilist) noexcept { return reverse_iterator(_Ilist.begin()); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto crbegin(const _Container& _Cont) noexcept(noexcept(_STD rbegin(_Cont))) /* strengthened */ +_NODISCARD _CONSTEXPR17 auto crbegin(const _Container& _Cont) noexcept(noexcept(_STD rbegin(_Cont))) -> decltype(_STD rbegin(_Cont)) { return _STD rbegin(_Cont); } _EXPORT_STD template -_NODISCARD _CONSTEXPR17 auto crend(const _Container& _Cont) noexcept(noexcept(_STD rend(_Cont))) /* strengthened */ +_NODISCARD _CONSTEXPR17 auto crend(const _Container& _Cont) noexcept(noexcept(_STD rend(_Cont))) -> decltype(_STD rend(_Cont)) { return _STD rend(_Cont); } _EXPORT_STD template -_NODISCARD constexpr auto size(const _Container& _Cont) noexcept(noexcept(_Cont.size())) /* strengthened */ - -> decltype(_Cont.size()) { +_NODISCARD constexpr auto size(const _Container& _Cont) noexcept(noexcept(_Cont.size())) -> decltype(_Cont.size()) { return _Cont.size(); } @@ -2258,8 +2249,7 @@ _NODISCARD constexpr size_t size(const _Ty (&)[_Size]) noexcept { #if _HAS_CXX20 _EXPORT_STD template -_NODISCARD constexpr auto ssize(const _Container& _Cont) noexcept(noexcept( - static_cast>>(_Cont.size()))) /* strengthened */ +_NODISCARD constexpr auto ssize(const _Container& _Cont) noexcept(noexcept(_Cont.size())) -> common_type_t> { using _Common = common_type_t>; return static_cast<_Common>(_Cont.size()); @@ -2272,8 +2262,7 @@ _NODISCARD constexpr ptrdiff_t ssize(const _Ty (&)[_Size]) noexcept { #endif // _HAS_CXX20 _EXPORT_STD template -_NODISCARD_EMPTY_NON_MEMBER constexpr auto empty(const _Container& _Cont) - noexcept(noexcept(_Cont.empty())) /* strengthened */ +_NODISCARD_EMPTY_NON_MEMBER constexpr auto empty(const _Container& _Cont) noexcept(noexcept(_Cont.empty())) -> decltype(_Cont.empty()) { return _Cont.empty(); } @@ -2283,20 +2272,13 @@ _NODISCARD_EMPTY_NON_MEMBER constexpr bool empty(const _Ty (&)[_Size]) noexcept return false; } -_EXPORT_STD template -_NODISCARD_EMPTY_NON_MEMBER constexpr bool empty(initializer_list<_Elem> _Ilist) noexcept { - return _Ilist.size() == 0; -} - _EXPORT_STD template -_NODISCARD constexpr auto data(_Container& _Cont) noexcept(noexcept(_Cont.data())) /* strengthened */ - -> decltype(_Cont.data()) { +_NODISCARD constexpr auto data(_Container& _Cont) noexcept(noexcept(_Cont.data())) -> decltype(_Cont.data()) { return _Cont.data(); } _EXPORT_STD template -_NODISCARD constexpr auto data(const _Container& _Cont) noexcept(noexcept(_Cont.data())) /* strengthened */ - -> decltype(_Cont.data()) { +_NODISCARD constexpr auto data(const _Container& _Cont) noexcept(noexcept(_Cont.data())) -> decltype(_Cont.data()) { return _Cont.data(); } @@ -2305,11 +2287,6 @@ _NODISCARD constexpr _Ty* data(_Ty (&_Array)[_Size]) noexcept { return _Array; } -_EXPORT_STD template -_NODISCARD constexpr const _Elem* data(initializer_list<_Elem> _Ilist) noexcept { - return _Ilist.begin(); -} - #if _HAS_CXX20 #if _HAS_CXX23 _EXPORT_STD template diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index ea67cde8d2a..c28df8db1c6 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -80,6 +80,7 @@ // (__cpp_lib_freestanding_algorithm and __cpp_lib_freestanding_array only) // P2937R0 Freestanding Library: Remove strtok // P2968R2 Make std::ignore A First-Class Object +// P3016R6 Resolve Inconsistencies In begin/end For valarray And Braced Initializer Lists // P3223R2 Making istream::ignore() Less Surprising // P3323R1 Forbid atomic, Specify atomic_ref // (for atomic) @@ -1578,6 +1579,7 @@ _EMIT_STL_ERROR(STL1004, "C++98 unexpected() is incompatible with C++23 unexpect #define __cpp_lib_freestanding_tuple 202306L #define __cpp_lib_freestanding_utility 202306L #define __cpp_lib_generic_associative_lookup 201304L +#define __cpp_lib_initializer_list 202511L #define __cpp_lib_integer_sequence 201304L #define __cpp_lib_integral_constant_callable 201304L #define __cpp_lib_is_final 201402L @@ -1595,6 +1597,7 @@ _EMIT_STL_ERROR(STL1004, "C++98 unexpected() is incompatible with C++23 unexpect #define __cpp_lib_transformation_trait_aliases 201304L #define __cpp_lib_tuple_element_t 201402L #define __cpp_lib_tuples_by_type 201304L +#define __cpp_lib_valarray 202511L // C++17 #define __cpp_lib_addressof_constexpr 201603L diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 171c3869c57..2eb83e0cf0a 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -136,6 +136,9 @@ std/language.support/support.limits/support.limits.general/mdspan.version.compil # libc++ has not implemented P2937R0: "Freestanding Library: Remove strtok" std/language.support/support.limits/support.limits.general/cstring.version.compile.pass.cpp FAIL +# libc++ has not implemented P3016R6: "Resolve Inconsistencies In begin/end For valarray And Braced Initializer Lists" +std/language.support/support.initlist/support.initlist.range/begin_end.pass.cpp FAIL + # libc++ has not implemented P3323R1: "Forbid atomic, Specify atomic_ref" std/atomics/atomics.ref/member_types.compile.pass.cpp FAIL diff --git a/tests/std/test.lst b/tests/std/test.lst index c44a4197217..934f31f1d5e 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -725,6 +725,7 @@ tests\P2693R1_text_formatting_header_stacktrace tests\P2693R1_text_formatting_header_thread tests\P2693R1_text_formatting_stacktrace tests\P2693R1_text_formatting_thread_id +tests\P3016R6_inconsistent_begin_end tests\P3107R5_enabled_specializations tests\P3349R1_contiguous_iterators_to_pointers tests\P3503R3_packaged_task_promise_with_allocator diff --git a/tests/std/tests/Dev11_1074023_constexpr/test.cpp b/tests/std/tests/Dev11_1074023_constexpr/test.cpp index d9b87f87247..47092a0a47e 100644 --- a/tests/std/tests/Dev11_1074023_constexpr/test.cpp +++ b/tests/std/tests/Dev11_1074023_constexpr/test.cpp @@ -53,7 +53,9 @@ constexpr auto int64_max = numeric_limits::max(); constexpr initializer_list il; STATIC_ASSERT(il.size() == 0); STATIC_ASSERT(il.begin() == il.end()); +#if _HAS_CXX17 STATIC_ASSERT(begin(il) == end(il)); +#endif // _HAS_CXX17 // TRANSITION, // constexpr error_category() noexcept; diff --git a/tests/std/tests/P3016R6_inconsistent_begin_end/env.lst b/tests/std/tests/P3016R6_inconsistent_begin_end/env.lst new file mode 100644 index 00000000000..19f025bd0e6 --- /dev/null +++ b/tests/std/tests/P3016R6_inconsistent_begin_end/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_matrix.lst diff --git a/tests/std/tests/P3016R6_inconsistent_begin_end/test.cpp b/tests/std/tests/P3016R6_inconsistent_begin_end/test.cpp new file mode 100644 index 00000000000..2b666b769b0 --- /dev/null +++ b/tests/std/tests/P3016R6_inconsistent_begin_end/test.cpp @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include + +#define STATIC_ASSERT(...) static_assert(__VA_ARGS__, #__VA_ARGS__) + +namespace my { + template + void begin(std::initializer_list); + + template + void end(std::initializer_list); + + template + void empty(std::initializer_list); + + template + void data(std::initializer_list); +} // namespace my + +_CONSTEXPR17 bool test_initializer_list() { + // Check that free functions in std still can be invoked on std::initializer_list + std::initializer_list il = {1, 2, 3}; + + using namespace std; + (void) begin(il); + (void) cbegin(il); + (void) end(il); + (void) cend(il); + (void) size(il); + (void) empty(il); + (void) data(il); + + return true; +} + +// test_valarray() is not constexpr because valarray is not a literal type +bool test_valarray() { + // Check that free functions in std can be invoked on std::valarray + std::valarray v{1}; + + using namespace std; + (void) begin(v); + (void) cbegin(v); // Did not compile before P3016R6 + (void) end(v); + (void) cend(v); // Did not compile before P3016R6 + (void) size(v); + // There are no members 'empty' and 'data' of valarray + + return true; +} + +constexpr bool test_braced_init_list_iteration() { + int sum = 0; + for (int n : {1, 2, 3, 4}) { + sum += n; + } + return sum == 10; +} + +int main() { + // Check that free functions in std can't be invoked with braced-initializer-list. + // If they could be invoked, the following expressions would be ambiguous between std:: and my:: + using namespace std; + using namespace my; + STATIC_ASSERT(std::is_same_v); + STATIC_ASSERT(std::is_same_v); + STATIC_ASSERT(std::is_same_v); + STATIC_ASSERT(std::is_same_v); + + test_initializer_list(); + test_valarray(); + test_braced_init_list_iteration(); + +#if _HAS_CXX17 + STATIC_ASSERT(test_initializer_list()); +#endif // _HAS_CXX17 + STATIC_ASSERT(test_braced_init_list_iteration()); +} diff --git a/tests/std/tests/VSO_0157762_feature_test_macros/test.compile.pass.cpp b/tests/std/tests/VSO_0157762_feature_test_macros/test.compile.pass.cpp index 3fe53dca792..aaf81e98164 100644 --- a/tests/std/tests/VSO_0157762_feature_test_macros/test.compile.pass.cpp +++ b/tests/std/tests/VSO_0157762_feature_test_macros/test.compile.pass.cpp @@ -491,6 +491,8 @@ STATIC_ASSERT(__cpp_lib_hypot == 201603L); STATIC_ASSERT(__cpp_lib_incomplete_container_elements == 201505L); +STATIC_ASSERT(__cpp_lib_initializer_list == 202511L); + #if _HAS_CXX20 STATIC_ASSERT(__cpp_lib_int_pow2 == 202002L); #elif defined(__cpp_lib_int_pow2) @@ -1037,6 +1039,8 @@ STATIC_ASSERT(__cpp_lib_unwrap_ref == 201811L); #error __cpp_lib_unwrap_ref is defined #endif +STATIC_ASSERT(__cpp_lib_valarray == 202511L); + #if _HAS_CXX20 STATIC_ASSERT(__cpp_lib_variant == 202106L); #elif _HAS_CXX17 From 58d2910c3bcf2b13beb5f86f02e9a4a8554c32df Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 8 Jan 2026 17:04:33 +0000 Subject: [PATCH 04/19] Enable vectorized minmax_element using Neon on ARM64 (#5949) Co-authored-by: Stephan T. Lavavej --- stl/inc/algorithm | 6 + stl/inc/xutility | 27 +- stl/src/vector_algorithms.cpp | 760 +++++++++++++++++++++++++++++----- 3 files changed, 685 insertions(+), 108 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 3fc1b239309..85838bca9cc 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -64,7 +64,9 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( _Min_max_element_t __stdcall __std_minmax_element_1(const void* _First, const void* _Last, bool _Signed) noexcept; _Min_max_element_t __stdcall __std_minmax_element_2(const void* _First, const void* _Last, bool _Signed) noexcept; _Min_max_element_t __stdcall __std_minmax_element_4(const void* _First, const void* _Last, bool _Signed) noexcept; +#ifndef _M_ARM64 _Min_max_element_t __stdcall __std_minmax_element_8(const void* _First, const void* _Last, bool _Signed) noexcept; +#endif // ^^^ !defined(_M_ARM64) ^^^ _Min_max_element_t __stdcall __std_minmax_element_f(const void* _First, const void* _Last, bool _Unused) noexcept; _Min_max_element_t __stdcall __std_minmax_element_d(const void* _First, const void* _Last, bool _Unused) noexcept; #endif // ^^^ _VECTORIZED_MINMAX_ELEMENT ^^^ @@ -212,7 +214,11 @@ pair<_Ty*, _Ty*> _Minmax_element_vectorized(_Ty* const _First, _Ty* const _Last) } else if constexpr (sizeof(_Ty) == 4) { _Res = ::__std_minmax_element_4(_First, _Last, _Signed); } else if constexpr (sizeof(_Ty) == 8) { +#ifdef _M_ARM64 + static_assert(false, "unexpected size; 64-bit integers on ARM64 should not take this codepath"); +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv _Res = ::__std_minmax_element_8(_First, _Last, _Signed); +#endif // ^^^ !defined(_M_ARM64) ^^^ } else { static_assert(false, "unexpected size"); } diff --git a/stl/inc/xutility b/stl/inc/xutility index b88e899ea2b..ca847ac5015 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -90,7 +90,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_INCLUDES _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_IS_SORTED_UNTIL _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_MINMAX _VECTORIZED_FOR_X64_X86 -#define _VECTORIZED_MINMAX_ELEMENT _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_MINMAX_ELEMENT _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_MISMATCH _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REMOVE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REMOVE_COPY _VECTORIZED_FOR_X64_X86 @@ -207,14 +207,18 @@ const void* __stdcall __std_find_end_8( const void* __stdcall __std_min_element_1(const void* _First, const void* _Last, bool _Signed) noexcept; const void* __stdcall __std_min_element_2(const void* _First, const void* _Last, bool _Signed) noexcept; const void* __stdcall __std_min_element_4(const void* _First, const void* _Last, bool _Signed) noexcept; +#ifndef _M_ARM64 const void* __stdcall __std_min_element_8(const void* _First, const void* _Last, bool _Signed) noexcept; +#endif // ^^^ !defined(_M_ARM64) ^^^ const void* __stdcall __std_min_element_f(const void* _First, const void* _Last, bool _Unused) noexcept; const void* __stdcall __std_min_element_d(const void* _First, const void* _Last, bool _Unused) noexcept; const void* __stdcall __std_max_element_1(const void* _First, const void* _Last, bool _Signed) noexcept; const void* __stdcall __std_max_element_2(const void* _First, const void* _Last, bool _Signed) noexcept; const void* __stdcall __std_max_element_4(const void* _First, const void* _Last, bool _Signed) noexcept; +#ifndef _M_ARM64 const void* __stdcall __std_max_element_8(const void* _First, const void* _Last, bool _Signed) noexcept; +#endif // ^^^ !defined(_M_ARM64) ^^^ const void* __stdcall __std_max_element_f(const void* _First, const void* _Last, bool _Unused) noexcept; const void* __stdcall __std_max_element_d(const void* _First, const void* _Last, bool _Unused) noexcept; #endif // ^^^ _VECTORIZED_MINMAX_ELEMENT ^^^ @@ -416,7 +420,11 @@ _Ty* _Min_element_vectorized(_Ty* const _First, _Ty* const _Last) noexcept { } else if constexpr (sizeof(_Ty) == 4) { return const_cast<_Ty*>(static_cast(::__std_min_element_4(_First, _Last, _Signed))); } else if constexpr (sizeof(_Ty) == 8) { +#ifdef _M_ARM64 + static_assert(false, "unexpected size; 64-bit integers on ARM64 should not take this codepath"); +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv return const_cast<_Ty*>(static_cast(::__std_min_element_8(_First, _Last, _Signed))); +#endif // ^^^ !defined(_M_ARM64) ^^^ } else { static_assert(false, "unexpected size"); } @@ -437,7 +445,11 @@ _Ty* _Max_element_vectorized(_Ty* const _First, _Ty* const _Last) noexcept { } else if constexpr (sizeof(_Ty) == 4) { return const_cast<_Ty*>(static_cast(::__std_max_element_4(_First, _Last, _Signed))); } else if constexpr (sizeof(_Ty) == 8) { +#ifdef _M_ARM64 + static_assert(false, "unexpected size; 64-bit integers on ARM64 should not take this codepath"); +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv return const_cast<_Ty*>(static_cast(::__std_max_element_8(_First, _Last, _Signed))); +#endif // ^^^ !defined(_M_ARM64) ^^^ } else { static_assert(false, "unexpected size"); } @@ -7258,9 +7270,20 @@ constexpr bool _Is_predicate_greater = _Is_any_of_v<_Pr, #endif // _HAS_CXX20 greater<>, greater<_Iter_value_t<_Iter>>>; +#ifdef _M_ARM64 +// We choose not to vectorize minmax_element for 64-bit integers on ARM64 +// as this does not improve performance over the scalar code. +template +_INLINE_VAR constexpr bool _Is_64bit_int_on_arm64_v = sizeof(_Ty) == 8 && !is_floating_point_v<_Ty>; +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv +template +_INLINE_VAR constexpr bool _Is_64bit_int_on_arm64_v = false; +#endif // ^^^ !defined(_M_ARM64) ^^^ + template constexpr bool _Is_min_max_optimization_safe = // Activate the vector algorithms for min_/max_element? - _Is_min_max_iterators_safe<_Iter> && _Is_predicate_less<_Iter, _Pr>; + _Is_min_max_iterators_safe<_Iter> && _Is_predicate_less<_Iter, _Pr> + && !_Is_64bit_int_on_arm64_v<_Iter_value_t<_Iter>>; // Unlike the position-based vectorized implementation, the value-based vectorized implementation // does not always produce the expected results for floating-point types. diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cb8bbbdbca0..ae96d1d5e4b 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -15,16 +15,14 @@ #include #include -#if !defined(_M_ARM64) && !defined(_M_ARM64EC) +#ifdef _M_ARM64 +#include +#elif !defined(_M_ARM64EC) #include #include extern "C" long __isa_enabled; -#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ - -#ifdef _M_ARM64 -#include -#endif +#endif // ^^^ !defined(_M_ARM64EC) ^^^ namespace { #if !defined(_M_ARM64) && !defined(_M_ARM64EC) @@ -923,7 +921,6 @@ __declspec(noalias) void __cdecl __std_reverse_copy_trivially_copyable_8( } // extern "C" -#ifndef _M_ARM64 namespace { namespace _Sorting { enum _Min_max_mode { @@ -934,17 +931,46 @@ namespace { template struct _Traits_scalar : _Base { - static constexpr bool _Vectorized = false; - static constexpr size_t _Tail_mask = 0; + static constexpr bool _Vectorized = false; + static constexpr size_t _Tail_mask = 0; + static constexpr bool _Has_unsigned_cmp = false; }; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + struct _Traits_neon_base { + using _Guard = char; + static constexpr bool _Vectorized = true; + static constexpr size_t _Vec_size = 16; + static constexpr size_t _Vec_mask = 0xF; + static constexpr size_t _Tail_mask = 0; + static constexpr bool _Has_unsigned_cmp = true; + + static int8x16_t _Zero() noexcept { + return vdupq_n_s8(0); + } + + static int8x16_t _All_ones() noexcept { + return vdupq_n_s8(static_cast(0xFF)); + } + + static int8x16_t _Blend(const int8x16_t _Px1, const int8x16_t _Px2, const int8x16_t _Msk) noexcept { + return vbslq_s8(vreinterpretq_u8_s8(_Msk), _Px2, _Px1); + } + + static int8x16_t _Sign_correction(const int8x16_t _Val, bool) noexcept { + return _Val; + } + + static void _Exit_vectorized() noexcept {} + }; +#elif !defined(_M_ARM64EC) struct _Traits_sse_base { - using _Guard = char; - static constexpr bool _Vectorized = true; - static constexpr size_t _Vec_size = 16; - static constexpr size_t _Vec_mask = 0xF; - static constexpr size_t _Tail_mask = 0; + using _Guard = char; + static constexpr bool _Vectorized = true; + static constexpr size_t _Vec_size = 16; + static constexpr size_t _Vec_mask = 0xF; + static constexpr size_t _Tail_mask = 0; + static constexpr bool _Has_unsigned_cmp = false; static __m128i _Zero() noexcept { return _mm_setzero_si128(); @@ -963,13 +989,28 @@ namespace { } static void _Exit_vectorized() noexcept {} + + static unsigned long _Get_first_h_pos(const unsigned long _Mask) noexcept { + unsigned long _H_pos; + // CodeQL [SM02313] _H_pos is always initialized: element exists, so _Mask != 0. + _BitScanForward(&_H_pos, _Mask); + return _H_pos; + } + + static unsigned long _Get_last_h_pos(const unsigned long _Mask) noexcept { + unsigned long _H_pos; + // CodeQL [SM02313] _H_pos is always initialized: element exists, so _Mask != 0. + _BitScanReverse(&_H_pos, _Mask); + return _H_pos; + } }; struct _Traits_avx_base { - using _Guard = _Zeroupper_on_exit; - static constexpr bool _Vectorized = true; - static constexpr size_t _Vec_size = 32; - static constexpr size_t _Vec_mask = 0x1F; + using _Guard = _Zeroupper_on_exit; + static constexpr bool _Vectorized = true; + static constexpr size_t _Vec_size = 32; + static constexpr size_t _Vec_mask = 0x1F; + static constexpr bool _Has_unsigned_cmp = false; static __m256i _Zero() noexcept { return _mm256_setzero_si256(); @@ -990,6 +1031,14 @@ namespace { static void _Exit_vectorized() noexcept { _mm256_zeroupper(); } + + static unsigned long _Get_first_h_pos(const unsigned long _Mask) noexcept { + return _tzcnt_u32(_Mask); + } + + static unsigned long _Get_last_h_pos(const unsigned long _Mask) noexcept { + return 31 - _lzcnt_u32(_Mask); + } }; struct _Traits_avx_i_base : _Traits_avx_base { @@ -1023,7 +1072,97 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + struct _Traits_1_neon : _Traits_1_base, _Traits_neon_base { + using _Vec_t = int8x16_t; + + // Compresses a 128-bit Mask of 16 8-bit values into a 64-bit Mask of 16 4-bit values. + static uint64_t _Mask(const _Vec_t _Val) noexcept { + const uint8x8_t _Res = vshrn_n_u16(vreinterpretq_u16_s8(_Val), 4); + return vget_lane_u64(vreinterpret_u64_u8(_Res), 0); + } + + static unsigned long _Get_first_h_pos(const uint64_t _Mask) noexcept { + return _CountTrailingZeros64(_Mask) >> 2; + } + + static unsigned long _Get_last_h_pos(const uint64_t _Mask) noexcept { + return 15 - (_CountLeadingZeros64(_Mask) >> 2); + } + + static _Vec_t _Load(const void* const _Src) noexcept { + return vld1q_s8(reinterpret_cast(_Src)); + } + + static _Vec_t _Inc(const _Vec_t _Idx) noexcept { + return vaddq_s8(_Idx, vdupq_n_s8(1)); + } + + static _Vec_t _H_min(const _Vec_t _Cur) noexcept { + return vdupq_n_s8(vminvq_s8(_Cur)); + } + + static _Vec_t _H_max(const _Vec_t _Cur) noexcept { + return vdupq_n_s8(vmaxvq_s8(_Cur)); + } + + static _Vec_t _H_min_u(const _Vec_t _Cur) noexcept { + const uint8_t _Mx = vminvq_u8(vreinterpretq_u8_s8(_Cur)); + return vreinterpretq_s8_u8(vdupq_n_u8(_Mx)); + } + + static _Vec_t _H_max_u(const _Vec_t _Cur) noexcept { + const uint8_t _Mx = vmaxvq_u8(vreinterpretq_u8_s8(_Cur)); + return vreinterpretq_s8_u8(vdupq_n_u8(_Mx)); + } + + static _Signed_t _Get_any(const _Vec_t _Cur) noexcept { + return static_cast<_Signed_t>(vgetq_lane_s8(_Cur, 0)); + } + + static _Unsigned_t _Get_v_pos(const _Vec_t _Cur) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Cur)); + } + + static _Vec_t _Cmp_eq(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s8_u8(vceqq_s8(_First, _Second)); + } + + static _Vec_t _Cmp_gt(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s8_u8(vcgtq_s8(_First, _Second)); + } + + static _Vec_t _Cmp_gt_u(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s8_u8(vcgtq_u8(vreinterpretq_u8_s8(_First), vreinterpretq_u8_s8(_Second))); + } + + static _Vec_t _Cmp_eq_idx(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Cmp_eq(_First, _Second); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s8(0)) noexcept { + return vminq_s8(_First, _Second); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s8(0)) noexcept { + return vmaxq_s8(_First, _Second); + } + + static _Vec_t _Min_u(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s8(0)) noexcept { + const uint8x16_t _Rx = vminq_u8(vreinterpretq_u8_s8(_First), vreinterpretq_u8_s8(_Second)); + return vreinterpretq_s8_u8(_Rx); + } + + static _Vec_t _Max_u(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s8(0)) noexcept { + const uint8x16_t _Rx = vmaxq_u8(vreinterpretq_u8_s8(_First), vreinterpretq_u8_s8(_Second)); + return vreinterpretq_s8_u8(_Rx); + } + + static _Vec_t _Mask_cast(const _Vec_t _Mask) noexcept { + return _Mask; + } + }; +#elif !defined(_M_ARM64EC) struct _Traits_1_sse : _Traits_1_base, _Traits_sse_base { static __m128i _Load(const void* const _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); @@ -1077,8 +1216,8 @@ namespace { return static_cast<_Signed_t>(_mm_cvtsi128_si32(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - return static_cast<_Unsigned_t>(_mm_cvtsi128_si32(_mm_shuffle_epi8(_Idx, _mm_cvtsi32_si128(_H_pos)))); + static _Unsigned_t _Get_v_pos(const __m128i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m128i _Cmp_eq(const __m128i _First, const __m128i _Second) noexcept { @@ -1170,10 +1309,8 @@ namespace { return static_cast<_Signed_t>(_mm256_cvtsi256_si32(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - const uint32_t _Part = _mm256_cvtsi256_si32( - _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); - return static_cast<_Unsigned_t>(_Part >> ((_H_pos & 0x3) << 3)); + static _Unsigned_t _Get_v_pos(const __m256i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { @@ -1230,7 +1367,97 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + struct _Traits_2_neon : _Traits_2_base, _Traits_neon_base { + using _Vec_t = int16x8_t; + + // Compresses a 128-bit Mask of 8 16-bit values into a 64-bit Mask of 8 8-bit values. + static uint64_t _Mask(const _Vec_t _Val) noexcept { + const uint16x4_t _Res = vshrn_n_u32(vreinterpretq_u32_s16(_Val), 8); + return vget_lane_u64(vreinterpret_u64_u16(_Res), 0); + } + + static unsigned long _Get_first_h_pos(const uint64_t _Mask) noexcept { + return _CountTrailingZeros64(_Mask) >> 2; + } + + static unsigned long _Get_last_h_pos(const uint64_t _Mask) noexcept { + return 15 - (_CountLeadingZeros64(_Mask) >> 2); + } + + static _Vec_t _Load(const void* const _Src) noexcept { + return vld1q_s16(reinterpret_cast(_Src)); + } + + static _Vec_t _Inc(const _Vec_t _Idx) noexcept { + return vaddq_s16(_Idx, vdupq_n_s16(1)); + } + + static _Vec_t _H_min(const _Vec_t _Cur) noexcept { + return vdupq_n_s16(vminvq_s16(_Cur)); + } + + static _Vec_t _H_max(const _Vec_t _Cur) noexcept { + return vdupq_n_s16(vmaxvq_s16(_Cur)); + } + + static _Vec_t _H_min_u(const _Vec_t _Cur) noexcept { + const uint16_t _Mx = vminvq_u16(vreinterpretq_u16_s16(_Cur)); + return vreinterpretq_s16_u16(vdupq_n_u16(_Mx)); + } + + static _Vec_t _H_max_u(const _Vec_t _Cur) noexcept { + const uint16_t _Mx = vmaxvq_u16(vreinterpretq_u16_s16(_Cur)); + return vreinterpretq_s16_u16(vdupq_n_u16(_Mx)); + } + + static _Signed_t _Get_any(const _Vec_t _Cur) noexcept { + return static_cast<_Signed_t>(vgetq_lane_s16(_Cur, 0)); + } + + static _Unsigned_t _Get_v_pos(const _Vec_t _Cur) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Cur)); + } + + static _Vec_t _Cmp_eq(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s16_u16(vceqq_s16(_First, _Second)); + } + + static _Vec_t _Cmp_gt(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s16_u16(vcgtq_s16(_First, _Second)); + } + + static _Vec_t _Cmp_gt_u(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s16_u16(vcgtq_u16(vreinterpretq_u16_s16(_First), vreinterpretq_u16_s16(_Second))); + } + + static _Vec_t _Cmp_eq_idx(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Cmp_eq(_First, _Second); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s16(0)) noexcept { + return vminq_s16(_First, _Second); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s16(0)) noexcept { + return vmaxq_s16(_First, _Second); + } + + static _Vec_t _Min_u(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s16(0)) noexcept { + const uint16x8_t _Rx = vminq_u16(vreinterpretq_u16_s16(_First), vreinterpretq_u16_s16(_Second)); + return vreinterpretq_s16_u16(_Rx); + } + + static _Vec_t _Max_u(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s16(0)) noexcept { + const uint16x8_t _Rx = vmaxq_u16(vreinterpretq_u16_s16(_First), vreinterpretq_u16_s16(_Second)); + return vreinterpretq_s16_u16(_Rx); + } + + static _Vec_t _Mask_cast(const _Vec_t _Mask) noexcept { + return _Mask; + } + }; +#elif !defined(_M_ARM64EC) struct _Traits_2_sse : _Traits_2_base, _Traits_sse_base { static __m128i _Load(const void* const _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); @@ -1281,11 +1508,8 @@ namespace { return static_cast<_Signed_t>(_mm_cvtsi128_si32(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - static constexpr _Unsigned_t _Shuf[] = {0x0100, 0x0302, 0x0504, 0x0706, 0x0908, 0x0B0A, 0x0D0C, 0x0F0E}; - - return static_cast<_Unsigned_t>( - _mm_cvtsi128_si32(_mm_shuffle_epi8(_Idx, _mm_cvtsi32_si128(_Shuf[_H_pos >> 1])))); + static _Unsigned_t _Get_v_pos(const __m128i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m128i _Cmp_eq(const __m128i _First, const __m128i _Second) noexcept { @@ -1373,10 +1597,8 @@ namespace { return static_cast<_Signed_t>(_mm256_cvtsi256_si32(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - const uint32_t _Part = _mm256_cvtsi256_si32( - _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); - return static_cast<_Unsigned_t>(_Part >> ((_H_pos & 0x2) << 3)); + static _Unsigned_t _Get_v_pos(const __m256i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { @@ -1437,7 +1659,97 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + struct _Traits_4_neon : _Traits_4_base, _Traits_neon_base { + using _Vec_t = int32x4_t; + + // Compresses a 128-bit Mask of 4 32-bit values into a 64-bit Mask of 4 16-bit values. + static uint64_t _Mask(const int32x4_t _Val) noexcept { + const uint32x2_t _Res = vshrn_n_u64(vreinterpretq_u64_s32(_Val), 16); + return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); + } + + static unsigned long _Get_first_h_pos(const uint64_t _Mask) noexcept { + return _CountTrailingZeros64(_Mask) >> 2; + } + + static unsigned long _Get_last_h_pos(const uint64_t _Mask) noexcept { + return 15 - (_CountLeadingZeros64(_Mask) >> 2); + } + + static _Vec_t _Load(const void* const _Src) noexcept { + return vld1q_s32(reinterpret_cast(_Src)); + } + + static _Vec_t _Inc(const _Vec_t _Idx) noexcept { + return vaddq_s32(_Idx, vdupq_n_s32(1)); + } + + static _Vec_t _H_min(const _Vec_t _Cur) noexcept { + return vdupq_n_s32(vminvq_s32(_Cur)); + } + + static _Vec_t _H_max(const _Vec_t _Cur) noexcept { + return vdupq_n_s32(vmaxvq_s32(_Cur)); + } + + static _Vec_t _H_min_u(const _Vec_t _Cur) noexcept { + const uint32_t _Mx = vminvq_u32(vreinterpretq_u32_s32(_Cur)); + return vreinterpretq_s32_u32(vdupq_n_u32(_Mx)); + } + + static _Vec_t _H_max_u(const _Vec_t _Cur) noexcept { + const uint32_t _Mx = vmaxvq_u32(vreinterpretq_u32_s32(_Cur)); + return vreinterpretq_s32_u32(vdupq_n_u32(_Mx)); + } + + static _Signed_t _Get_any(const _Vec_t _Cur) noexcept { + return static_cast<_Signed_t>(vgetq_lane_s32(_Cur, 0)); + } + + static _Unsigned_t _Get_v_pos(const _Vec_t _Cur) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Cur)); + } + + static _Vec_t _Cmp_eq(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s32_u32(vceqq_s32(_First, _Second)); + } + + static _Vec_t _Cmp_gt(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s32_u32(vcgtq_s32(_First, _Second)); + } + + static _Vec_t _Cmp_gt_u(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s32_u32(vcgtq_u32(vreinterpretq_u32_s32(_First), vreinterpretq_u32_s32(_Second))); + } + + static _Vec_t _Cmp_eq_idx(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Cmp_eq(_First, _Second); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s32(0)) noexcept { + return vminq_s32(_First, _Second); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s32(0)) noexcept { + return vmaxq_s32(_First, _Second); + } + + static _Vec_t _Min_u(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s32(0)) noexcept { + const uint32x4_t _Rx = vminq_u32(vreinterpretq_u32_s32(_First), vreinterpretq_u32_s32(_Second)); + return vreinterpretq_s32_u32(_Rx); + } + + static _Vec_t _Max_u(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_s32(0)) noexcept { + const uint32x4_t _Rx = vmaxq_u32(vreinterpretq_u32_s32(_First), vreinterpretq_u32_s32(_Second)); + return vreinterpretq_s32_u32(_Rx); + } + + static _Vec_t _Mask_cast(const _Vec_t _Mask) noexcept { + return _Mask; + } + }; +#elif !defined(_M_ARM64EC) struct _Traits_4_sse : _Traits_4_base, _Traits_sse_base { static __m128i _Load(const void* const _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); @@ -1485,10 +1797,8 @@ namespace { return static_cast<_Signed_t>(_mm_cvtsi128_si32(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - _Unsigned_t _Array[4]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(&_Array), _Idx); - return _Array[_H_pos >> 2]; + static _Unsigned_t _Get_v_pos(const __m128i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m128i _Cmp_eq(const __m128i _First, const __m128i _Second) noexcept { @@ -1573,9 +1883,8 @@ namespace { return static_cast<_Signed_t>(_mm256_cvtsi256_si32(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - return _mm256_cvtsi256_si32( - _mm256_permutevar8x32_epi32(_Idx, _mm256_castsi128_si256(_mm_cvtsi32_si128(_H_pos >> 2)))); + static _Unsigned_t _Get_v_pos(const __m256i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { @@ -1631,7 +1940,7 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64EC +#if !defined(_M_ARM64) && !defined(_M_ARM64EC) struct _Traits_8_sse : _Traits_8_base, _Traits_sse_base { static __m128i _Load(const void* const _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); @@ -1678,13 +1987,13 @@ namespace { static _Signed_t _Get_any(const __m128i _Cur) noexcept { // With optimizations enabled, compiles into register movement, rather than an actual stack spill. // Works around the absence of _mm_cvtsi128_si64 on 32-bit. - return static_cast<_Signed_t>(_Get_v_pos(_Cur, 0)); + _Signed_t _Array[2]; + _mm_storeu_si128(reinterpret_cast<__m128i*>(&_Array), _Cur); + return _Array[0]; } - static _Unsigned_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - _Unsigned_t _Array[2]; - _mm_storeu_si128(reinterpret_cast<__m128i*>(&_Array), _Idx); - return _Array[_H_pos >> 3]; + static _Unsigned_t _Get_v_pos(const __m128i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m128i _Cmp_eq(const __m128i _First, const __m128i _Second) noexcept { @@ -1780,10 +2089,8 @@ namespace { return _Traits_8_sse::_Get_any(_mm256_castsi256_si128(_Cur)); } - static _Unsigned_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - _Unsigned_t _Array[4]; - _mm256_storeu_si256(reinterpret_cast<__m256i*>(&_Array), _Idx); - return _Array[_H_pos >> 3]; + static _Unsigned_t _Get_v_pos(const __m256i _Idx) noexcept { + return static_cast<_Unsigned_t>(_Get_any(_Idx)); } static __m256i _Cmp_eq(const __m256i _First, const __m256i _Second) noexcept { @@ -1818,7 +2125,7 @@ namespace { return _Mask; } }; -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ struct _Traits_f_base { static constexpr bool _Is_floating = true; @@ -1842,7 +2149,81 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + struct _Traits_f_neon : _Traits_f_base, _Traits_neon_base { + using _Vec_t = float32x4_t; + using _Idx_t = int32x4_t; + static constexpr bool _Has_unsigned_cmp = false; + + static uint64_t _Mask(const _Idx_t _Val) noexcept { + return _Traits_4_neon::_Mask(_Val); + } + + static unsigned long _Get_first_h_pos(const uint64_t _Mask) noexcept { + return _Traits_4_neon::_Get_first_h_pos(_Mask); + } + + static unsigned long _Get_last_h_pos(const uint64_t _Mask) noexcept { + return _Traits_4_neon::_Get_last_h_pos(_Mask); + } + + static _Vec_t _Load(const void* const _Src) noexcept { + return vld1q_f32(reinterpret_cast(_Src)); + } + + static _Idx_t _Inc(const _Idx_t _Idx) noexcept { + return _Traits_4_neon::_Inc(_Idx); + } + + static _Vec_t _H_min(const _Vec_t _Cur) noexcept { + return vdupq_n_f32(vminvq_f32(_Cur)); + } + + static _Vec_t _H_max(const _Vec_t _Cur) noexcept { + return vdupq_n_f32(vmaxvq_f32(_Cur)); + } + + static _Idx_t _H_min_u(const _Idx_t _Cur) noexcept { + return _Traits_4_neon::_H_min_u(_Cur); + } + + static _Idx_t _H_max_u(const _Idx_t _Cur) noexcept { + return _Traits_4_neon::_H_max_u(_Cur); + } + + static float _Get_any(const _Vec_t _Cur) noexcept { + return vgetq_lane_f32(_Cur, 0); + } + + static _Traits_4_neon::_Unsigned_t _Get_v_pos(const _Idx_t _Cur) noexcept { + return _Traits_4_neon::_Get_v_pos(_Cur); + } + + static _Idx_t _Cmp_eq(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s32_u32(vceqq_f32(_First, _Second)); + } + + static _Idx_t _Cmp_gt(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s32_u32(vcgtq_f32(_First, _Second)); + } + + static _Idx_t _Cmp_eq_idx(const _Idx_t _First, const _Idx_t _Second) noexcept { + return _Traits_4_neon::_Cmp_eq_idx(_First, _Second); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f32(0)) noexcept { + return vminq_f32(_First, _Second); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f32(0)) noexcept { + return vmaxq_f32(_First, _Second); + } + + static _Idx_t _Mask_cast(const _Vec_t _Mask) noexcept { + return vreinterpretq_s32_f32(_Mask); + } + }; +#elif !defined(_M_ARM64EC) struct _Traits_f_sse : _Traits_f_base, _Traits_sse_base { static __m128 _Load(const void* const _Src) noexcept { return _mm_loadu_ps(reinterpret_cast(_Src)); @@ -1886,8 +2267,8 @@ namespace { return _mm_cvtss_f32(_Cur); } - static uint32_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - return _Traits_4_sse::_Get_v_pos(_Idx, _H_pos); + static uint32_t _Get_v_pos(const __m128i _Idx) noexcept { + return _Traits_4_sse::_Get_v_pos(_Idx); } static __m128 _Cmp_eq(const __m128 _First, const __m128 _Second) noexcept { @@ -1969,8 +2350,8 @@ namespace { return _mm256_cvtss_f32(_Cur); } - static uint32_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - return _Traits_4_avx::_Get_v_pos(_Idx, _H_pos); + static uint32_t _Get_v_pos(const __m256i _Idx) noexcept { + return _Traits_4_avx::_Get_v_pos(_Idx); } static __m256 _Cmp_eq(const __m256 _First, const __m256 _Second) noexcept { @@ -2016,7 +2397,89 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + struct _Traits_d_neon : _Traits_d_base, _Traits_neon_base { + using _Vec_t = float64x2_t; + using _Idx_t = int64x2_t; + static constexpr bool _Has_unsigned_cmp = false; + + // Compresses a 128-bit Mask of 2 64-bit values into a 64-bit Mask of 2 32-bit values. + static uint64_t _Mask(const int64x2_t _Val) noexcept { + const uint32x2_t _Res = vreinterpret_u32_s32(vmovn_s64(_Val)); + return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); + } + + static unsigned long _Get_first_h_pos(const uint64_t _Mask) noexcept { + return _CountTrailingZeros64(_Mask) >> 2; + } + + static unsigned long _Get_last_h_pos(const uint64_t _Mask) noexcept { + return 15 - (_CountLeadingZeros64(_Mask) >> 2); + } + + static _Vec_t _Load(const void* const _Src) noexcept { + return vld1q_f64(reinterpret_cast(_Src)); + } + + static _Idx_t _Inc(const _Idx_t _Idx) noexcept { + return vaddq_s64(_Idx, vdupq_n_s64(1)); + } + + static _Vec_t _H_min(const _Vec_t _Cur) noexcept { + return vdupq_n_f64(vminvq_f64(_Cur)); + } + + static _Vec_t _H_max(const _Vec_t _Cur) noexcept { + return vdupq_n_f64(vmaxvq_f64(_Cur)); + } + + static _Idx_t _H_min_u(const _Idx_t _Cur) noexcept { + const uint64x2_t _Cur_u = vreinterpretq_u64_s64(_Cur); + const uint64x2_t _Swapped = vextq_u64(_Cur_u, _Cur_u, 1); + const uint64x2_t _Mask_lt = vcltq_u64(_Swapped, _Cur_u); + return vreinterpretq_s64_u64(vbslq_u64(_Mask_lt, _Swapped, _Cur_u)); + } + + static _Idx_t _H_max_u(const _Idx_t _Cur) noexcept { + const uint64x2_t _Cur_u = vreinterpretq_u64_s64(_Cur); + const uint64x2_t _Swapped = vextq_u64(_Cur_u, _Cur_u, 1); + const uint64x2_t _Mask_gt = vcgtq_u64(_Swapped, _Cur_u); + return vreinterpretq_s64_u64(vbslq_u64(_Mask_gt, _Swapped, _Cur_u)); + } + + static double _Get_any(const _Vec_t _Cur) noexcept { + return vgetq_lane_f64(_Cur, 0); + } + + static _Traits_8_base::_Unsigned_t _Get_v_pos(const _Idx_t _Cur) noexcept { + return static_cast<_Traits_8_base::_Unsigned_t>(vgetq_lane_s64(_Cur, 0)); + } + + static _Idx_t _Cmp_eq(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s64_u64(vceqq_f64(_First, _Second)); + } + + static _Idx_t _Cmp_gt(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s64_u64(vcgtq_f64(_First, _Second)); + } + + static _Idx_t _Cmp_eq_idx(const _Idx_t _First, const _Idx_t _Second) noexcept { + return vreinterpretq_s64_u64(vceqq_s64(_First, _Second)); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f64(0)) noexcept { + return vminq_f64(_First, _Second); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f64(0)) noexcept { + return vmaxq_f64(_First, _Second); + } + + static _Idx_t _Mask_cast(const _Vec_t _Mask) noexcept { + return vreinterpretq_s64_f64(_Mask); + } + }; +#elif !defined(_M_ARM64EC) struct _Traits_d_sse : _Traits_d_base, _Traits_sse_base { static __m128d _Load(const void* const _Src) noexcept { return _mm_loadu_pd(reinterpret_cast(_Src)); @@ -2058,8 +2521,8 @@ namespace { return _mm_cvtsd_f64(_Cur); } - static uint64_t _Get_v_pos(const __m128i _Idx, const unsigned long _H_pos) noexcept { - return _Traits_8_sse::_Get_v_pos(_Idx, _H_pos); + static uint64_t _Get_v_pos(const __m128i _Idx) noexcept { + return _Traits_8_sse::_Get_v_pos(_Idx); } static __m128d _Cmp_eq(const __m128d _First, const __m128d _Second) noexcept { @@ -2140,8 +2603,8 @@ namespace { return _mm256_cvtsd_f64(_Cur); } - static uint64_t _Get_v_pos(const __m256i _Idx, const unsigned long _H_pos) noexcept { - return _Traits_8_avx::_Get_v_pos(_Idx, _H_pos); + static uint64_t _Get_v_pos(const __m256i _Idx) noexcept { + return _Traits_8_avx::_Get_v_pos(_Idx); } static __m256d _Cmp_eq(const __m256d _First, const __m256d _Second) noexcept { @@ -2172,7 +2635,9 @@ namespace { struct _Traits_1 { using _Scalar = _Traits_scalar<_Traits_1_base>; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + using _Neon = _Traits_1_neon; +#elif !defined(_M_ARM64EC) using _Sse = _Traits_1_sse; using _Avx = _Traits_1_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ @@ -2180,7 +2645,9 @@ namespace { struct _Traits_2 { using _Scalar = _Traits_scalar<_Traits_2_base>; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + using _Neon = _Traits_2_neon; +#elif !defined(_M_ARM64EC) using _Sse = _Traits_2_sse; using _Avx = _Traits_2_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ @@ -2188,12 +2655,15 @@ namespace { struct _Traits_4 { using _Scalar = _Traits_scalar<_Traits_4_base>; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + using _Neon = _Traits_4_neon; +#elif !defined(_M_ARM64EC) using _Sse = _Traits_4_sse; using _Avx = _Traits_4_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; +#ifndef _M_ARM64 struct _Traits_8 { using _Scalar = _Traits_scalar<_Traits_8_base>; #ifndef _M_ARM64EC @@ -2201,10 +2671,13 @@ namespace { using _Avx = _Traits_8_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; +#endif // ^^^ !defined(_M_ARM64) ^^^ struct _Traits_f { using _Scalar = _Traits_scalar<_Traits_f_base>; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + using _Neon = _Traits_f_neon; +#elif !defined(_M_ARM64EC) using _Sse = _Traits_f_sse; using _Avx = _Traits_f_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ @@ -2212,7 +2685,9 @@ namespace { struct _Traits_d { using _Scalar = _Traits_scalar<_Traits_d_base>; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + using _Neon = _Traits_d_neon; +#elif !defined(_M_ARM64EC) using _Sse = _Traits_d_sse; using _Avx = _Traits_d_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ @@ -2262,8 +2737,13 @@ namespace { return _Res; } +#ifdef _M_ARM64 + template <_Min_max_mode _Mode, class _Traits, bool _Sign> + auto _Minmax_element_impl(const void* _First, const void* const _Last) noexcept { +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv template <_Min_max_mode _Mode, class _Traits> auto _Minmax_element_impl(const void* _First, const void* const _Last, const bool _Sign) noexcept { +#endif // ^^^ !defined(_M_ARM64) ^^^ _Min_max_element_t _Res = {_First, _First}; auto _Cur_min_val = _Traits::_Init_min_val; auto _Cur_max_val = _Traits::_Init_max_val; @@ -2294,31 +2774,81 @@ namespace { auto _Cur_idx_max = _Traits::_Zero(); // vector of vertical maximum indices auto _Cur_idx = _Traits::_Zero(); // current vector of indices +#ifdef _M_ARM64 + const auto _Cmp_gt_wrap = [](const auto _First, const auto _Second) noexcept { + if constexpr (_Sign || !_Traits::_Has_unsigned_cmp) { + return _Traits::_Cmp_gt(_First, _Second); + } else { + return _Traits::_Cmp_gt_u(_First, _Second); + } + }; + const auto _Min_wrap = [](const auto _First, const auto _Second, const auto _Mask) noexcept { + if constexpr (_Sign || !_Traits::_Has_unsigned_cmp) { + return _Traits::_Min(_First, _Second, _Mask); + } else { + return _Traits::_Min_u(_First, _Second, _Mask); + } + }; + const auto _Max_wrap = [](const auto _First, const auto _Second, const auto _Mask) noexcept { + if constexpr (_Sign || !_Traits::_Has_unsigned_cmp) { + return _Traits::_Max(_First, _Second, _Mask); + } else { + return _Traits::_Max_u(_First, _Second, _Mask); + } + }; + const auto _H_min_wrap = [](const auto _Vals) noexcept { + if constexpr (_Sign || !_Traits::_Has_unsigned_cmp) { + return _Traits::_H_min(_Vals); + } else { + return _Traits::_H_min_u(_Vals); + } + }; + const auto _H_max_wrap = [](const auto _Vals) noexcept { + if constexpr (_Sign || !_Traits::_Has_unsigned_cmp) { + return _Traits::_H_max(_Vals); + } else { + return _Traits::_H_max_u(_Vals); + } + }; +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv + const auto _Cmp_gt_wrap = [](const auto _First, const auto _Second) noexcept { + return _Traits::_Cmp_gt(_First, _Second); + }; + const auto _Min_wrap = [](const auto _First, const auto _Second, const auto _Mask) noexcept { + return _Traits::_Min(_First, _Second, _Mask); + }; + const auto _Max_wrap = [](const auto _First, const auto _Second, const auto _Mask) noexcept { + return _Traits::_Max(_First, _Second, _Mask); + }; + const auto _H_min_wrap = [](const auto _Vals) noexcept { return _Traits::_H_min(_Vals); }; + const auto _H_max_wrap = [](const auto _Vals) noexcept { return _Traits::_H_max(_Vals); }; +#endif // ^^^ !defined(_M_ARM64) ^^^ + const auto _Update_min_max = [&](const auto _Cur_vals, [[maybe_unused]] const auto _Blend_idx_0, const auto _Blend_idx_1) noexcept { if constexpr ((_Mode & _Mode_min) != 0) { // Looking for the first occurrence of minimum, don't overwrite with newly found occurrences - const auto _Is_less = _Traits::_Cmp_gt(_Cur_vals_min, _Cur_vals); // _Cur_vals < _Cur_vals_min + const auto _Is_less = _Cmp_gt_wrap(_Cur_vals_min, _Cur_vals); // _Cur_vals < _Cur_vals_min // Remember their vertical indices _Cur_idx_min = _Blend_idx_1(_Cur_idx_min, _Cur_idx, _Traits::_Mask_cast(_Is_less)); - _Cur_vals_min = _Traits::_Min(_Cur_vals_min, _Cur_vals, _Is_less); // Update the current minimum + _Cur_vals_min = _Min_wrap(_Cur_vals_min, _Cur_vals, _Is_less); // Update the current minimum } if constexpr (_Mode == _Mode_max) { // Looking for the first occurrence of maximum, don't overwrite with newly found occurrences // _Cur_vals > _Cur_vals_max - const auto _Is_greater = _Traits::_Cmp_gt(_Cur_vals, _Cur_vals_max); + const auto _Is_greater = _Cmp_gt_wrap(_Cur_vals, _Cur_vals_max); // Remember their vertical indices _Cur_idx_max = _Blend_idx_1(_Cur_idx_max, _Cur_idx, _Traits::_Mask_cast(_Is_greater)); // Update the current maximum - _Cur_vals_max = _Traits::_Max(_Cur_vals_max, _Cur_vals, _Is_greater); + _Cur_vals_max = _Max_wrap(_Cur_vals_max, _Cur_vals, _Is_greater); } else if constexpr (_Mode == _Mode_both) { // Looking for the last occurrence of maximum, do overwrite with newly found occurrences // !(_Cur_vals >= _Cur_vals_max) - const auto _Is_less = _Traits::_Cmp_gt(_Cur_vals_max, _Cur_vals); + const auto _Is_less = _Cmp_gt_wrap(_Cur_vals_max, _Cur_vals); // Remember their vertical indices _Cur_idx_max = _Blend_idx_0(_Cur_idx_max, _Cur_idx, _Traits::_Mask_cast(_Is_less)); - _Cur_vals_max = _Traits::_Max(_Cur_vals, _Cur_vals_max, _Is_less); // Update the current maximum + _Cur_vals_max = _Max_wrap(_Cur_vals, _Cur_vals_max, _Is_less); // Update the current maximum } }; @@ -2382,29 +2912,26 @@ namespace { if constexpr ((_Mode & _Mode_min) != 0) { // Vector populated by the smallest element - const auto _H_min = _Traits::_H_min(_Cur_vals_min); + const auto _H_min = _H_min_wrap(_Cur_vals_min); const auto _H_min_val = _Traits::_Get_any(_H_min); // Get any element of it if (_H_min_val < _Cur_min_val) { // Current horizontal min is less than the old _Cur_min_val = _H_min_val; // update min // Mask of all elems eq to min const auto _Eq_mask = _Traits::_Cmp_eq(_H_min, _Cur_vals_min); - unsigned long _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); + auto _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); // Indices of minimum elements or the greatest index if none const auto _Idx_min_val = _Traits::_Blend(_Traits::_All_ones(), _Cur_idx_min, _Traits::_Mask_cast(_Eq_mask)); auto _Idx_min = _Traits::_H_min_u(_Idx_min_val); // The smallest indices // Select the smallest vertical indices from the smallest element mask _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_min, _Idx_min_val)); - unsigned long _H_pos; // Find the smallest horizontal index - - // CodeQL [SM02313] _H_pos is always initialized: element exists, so _Mask != 0. - _BitScanForward(&_H_pos, _Mask); + const unsigned long _H_pos = _Traits::_Get_first_h_pos(_Mask); // Extract its vertical index - const auto _V_pos = _Traits::_Get_v_pos(_Cur_idx_min, _H_pos); + const auto _V_pos = _Traits::_Get_v_pos(_Idx_min); // Finally, compute the pointer _Res._Min = _Base + static_cast(_V_pos) * _Traits::_Vec_size + _H_pos; } @@ -2412,7 +2939,7 @@ namespace { if constexpr ((_Mode & _Mode_max) != 0) { // Vector populated by the largest element - const auto _H_max = _Traits::_H_max(_Cur_vals_max); + const auto _H_max = _H_max_wrap(_Cur_vals_max); const auto _H_max_val = _Traits::_Get_any(_H_max); // Get any element of it if (_Mode == _Mode_both && _Cur_max_val <= _H_max_val @@ -2423,9 +2950,10 @@ namespace { // Mask of all elems eq to max const auto _Eq_mask = _Traits::_Cmp_eq(_H_max, _Cur_vals_max); - unsigned long _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); + auto _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Eq_mask)); unsigned long _H_pos; + size_t _V_pos; if constexpr (_Mode == _Mode_both) { // Looking for the last occurrence of maximum // Indices of maximum elements or zero if none @@ -2436,30 +2964,30 @@ namespace { _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); // Find the largest horizontal index - - // CodeQL [SM02313] _H_pos is always initialized: element exists, so _Mask != 0. - _BitScanReverse(&_H_pos, _Mask); - + _H_pos = _Traits::_Get_last_h_pos(_Mask); _H_pos -= sizeof(_Cur_max_val) - 1; // Correct from highest val bit to lowest + + // Extract its vertical index + _V_pos = static_cast(_Traits::_Get_v_pos(_Idx_max)); } else { // Looking for the first occurrence of maximum // Indices of maximum elements or the greatest index if none const auto _Idx_max_val = _Traits::_Blend( _Traits::_All_ones(), _Cur_idx_max, _Traits::_Mask_cast(_Eq_mask)); const auto _Idx_max = _Traits::_H_min_u(_Idx_max_val); // The smallest indices + // Select the smallest vertical indices from the largest element mask _Mask &= _Traits::_Mask(_Traits::_Cmp_eq_idx(_Idx_max, _Idx_max_val)); // Find the smallest horizontal index + _H_pos = _Traits::_Get_first_h_pos(_Mask); - // CodeQL [SM02313] _H_pos is always initialized: element exists, so _Mask != 0. - _BitScanForward(&_H_pos, _Mask); + // Extract its vertical index + _V_pos = static_cast(_Traits::_Get_v_pos(_Idx_max)); } - // Extract its vertical index - const auto _V_pos = _Traits::_Get_v_pos(_Cur_idx_max, _H_pos); // Finally, compute the pointer - _Res._Max = _Base + static_cast(_V_pos) * _Traits::_Vec_size + _H_pos; + _Res._Max = _Base + _V_pos * _Traits::_Vec_size + _H_pos; } } // Horizontal part done, results are saved, now need to see if there is another portion. @@ -2513,7 +3041,7 @@ namespace { using _STy = _Traits::_Signed_t; using _UTy = _Traits::_Unsigned_t; - constexpr _UTy _Correction = _UTy{1} << (sizeof(_UTy) * 8 - 1); + constexpr _UTy _Correction = _Traits::_Has_unsigned_cmp ? 0 : _UTy{1} << (sizeof(_UTy) * 8 - 1); if constexpr (_Mode == _Mode_min) { if (_Sign) { @@ -2542,6 +3070,21 @@ namespace { template <_Min_max_mode _Mode, class _Traits> auto __stdcall _Minmax_element_disp( const void* const _First, const void* const _Last, const bool _Sign) noexcept { +#ifdef _M_ARM64 + if (_Byte_length(_First, _Last) >= 16) { + if (_Sign) { + return _Minmax_element_impl<_Mode, typename _Traits::_Neon, true>(_First, _Last); + } else { + return _Minmax_element_impl<_Mode, typename _Traits::_Neon, false>(_First, _Last); + } + } + + if (_Sign) { + return _Minmax_element_impl<_Mode, typename _Traits::_Scalar, true>(_First, _Last); + } else { + return _Minmax_element_impl<_Mode, typename _Traits::_Scalar, false>(_First, _Last); + } +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv #ifndef _M_ARM64EC if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { return _Minmax_element_impl<_Mode, typename _Traits::_Avx>(_First, _Last, _Sign); @@ -2552,8 +3095,10 @@ namespace { } #endif // ^^^ !defined(_M_ARM64EC) ^^^ return _Minmax_element_impl<_Mode, typename _Traits::_Scalar>(_First, _Last, _Sign); +#endif // ^^^ !defined(_M_ARM64) ^^^ } +#ifndef _M_ARM64 template <_Min_max_mode _Mode, class _Traits, bool _Sign> auto _Minmax_impl(const void* _First, const void* const _Last) noexcept { using _Ty = std::conditional_t<_Sign, typename _Traits::_Signed_t, typename _Traits::_Unsigned_t>; @@ -2770,13 +3315,10 @@ namespace { } const auto _Is_less = _Traits::_Cmp_gt(_Right, _Left); - unsigned long _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Is_less)); + const auto _Mask = _Traits::_Mask(_Traits::_Mask_cast(_Is_less)); if (_Mask != 0) { - unsigned long _H_pos; - - // CodeQL [SM02313] _H_pos is always initialized: we just tested `if (_Mask != 0)`. - _BitScanForward(&_H_pos, _Mask); + const unsigned long _H_pos = _Traits::_Get_first_h_pos(_Mask); _Advance_bytes(_First, _H_pos); return _First; } @@ -2798,14 +3340,10 @@ namespace { } const auto _Is_less = _Traits::_Cmp_gt(_Right, _Left); - unsigned long _Mask = - _Traits::_Mask(_mm256_and_si256(_Traits::_Mask_cast(_Is_less), _Tail_mask)); + const auto _Mask = _Traits::_Mask(_mm256_and_si256(_Traits::_Mask_cast(_Is_less), _Tail_mask)); if (_Mask != 0) { - unsigned long _H_pos; - - // CodeQL [SM02313] _H_pos is always initialized: we just tested `if (_Mask != 0)`. - _BitScanForward(&_H_pos, _Mask); + const unsigned long _H_pos = _Traits::_Get_first_h_pos(_Mask); _Advance_bytes(_First, _H_pos); return _First; } @@ -2847,6 +3385,7 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ return _Is_sorted_until_impl(_First, _Last, _Greater); } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // namespace _Sorting } // unnamed namespace @@ -2867,10 +3406,12 @@ const void* __stdcall __std_min_element_4( return _Sorting::_Minmax_element_disp<_Sorting::_Mode_min, _Sorting::_Traits_4>(_First, _Last, _Signed); } +#ifndef _M_ARM64 const void* __stdcall __std_min_element_8( const void* const _First, const void* const _Last, const bool _Signed) noexcept { return _Sorting::_Minmax_element_disp<_Sorting::_Mode_min, _Sorting::_Traits_8>(_First, _Last, _Signed); } +#endif // ^^^ !defined(_M_ARM64) ^^^ // TRANSITION, ABI: remove unused `bool` const void* __stdcall __std_min_element_f(const void* const _First, const void* const _Last, bool) noexcept { @@ -2897,10 +3438,12 @@ const void* __stdcall __std_max_element_4( return _Sorting::_Minmax_element_disp<_Sorting::_Mode_max, _Sorting::_Traits_4>(_First, _Last, _Signed); } +#ifndef _M_ARM64 const void* __stdcall __std_max_element_8( const void* const _First, const void* const _Last, const bool _Signed) noexcept { return _Sorting::_Minmax_element_disp<_Sorting::_Mode_max, _Sorting::_Traits_8>(_First, _Last, _Signed); } +#endif // ^^^ !defined(_M_ARM64) ^^^ // TRANSITION, ABI: remove unused `bool` const void* __stdcall __std_max_element_f(const void* const _First, const void* const _Last, bool) noexcept { @@ -2927,10 +3470,12 @@ _Min_max_element_t __stdcall __std_minmax_element_4( return _Sorting::_Minmax_element_disp<_Sorting::_Mode_both, _Sorting::_Traits_4>(_First, _Last, _Signed); } +#ifndef _M_ARM64 _Min_max_element_t __stdcall __std_minmax_element_8( const void* const _First, const void* const _Last, const bool _Signed) noexcept { return _Sorting::_Minmax_element_disp<_Sorting::_Mode_both, _Sorting::_Traits_8>(_First, _Last, _Signed); } +#endif // ^^^ !defined(_M_ARM64) ^^^ // TRANSITION, ABI: remove unused `bool` _Min_max_element_t __stdcall __std_minmax_element_f(const void* const _First, const void* const _Last, bool) noexcept { @@ -2942,6 +3487,7 @@ _Min_max_element_t __stdcall __std_minmax_element_d(const void* const _First, co return _Sorting::_Minmax_element_disp<_Sorting::_Mode_both, _Sorting::_Traits_d>(_First, _Last, false); } +#ifndef _M_ARM64 __declspec(noalias) int8_t __stdcall __std_min_1i(const void* const _First, const void* const _Last) noexcept { return _Sorting::_Minmax_disp<_Sorting::_Mode_min, _Sorting::_Traits_1, true>(_First, _Last); } @@ -3111,9 +3657,11 @@ const void* __stdcall __std_is_sorted_until_d( const void* const _First, const void* const _Last, const bool _Greater) noexcept { return _Sorting::_Is_sorted_until_disp<_Sorting::_Traits_d, double>(_First, _Last, _Greater); } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // extern "C" +#ifndef _M_ARM64 namespace { namespace _Finding { #ifdef _M_ARM64EC From eec1e28c104a8b3fdf33bf13a4f8e138e9f9cffc Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 8 Jan 2026 09:14:27 -0800 Subject: [PATCH 05/19] The Uncanny xmath Overhaul (#5959) --- stl/CMakeLists.txt | 7 - stl/inc/complex | 4 +- stl/inc/ymath.h | 4 - .../stl_base/stl.files.settings.targets | 7 - stl/src/xcosh.cpp | 53 ++++- stl/src/xdtest.cpp | 26 +-- stl/src/xexp.cpp | 210 ++++++++++++++++-- stl/src/xfcosh.cpp | 39 ---- stl/src/xfdnorm.cpp | 35 --- stl/src/xfdscale.cpp | 67 ------ stl/src/xfdtest.cpp | 22 -- stl/src/xferaise.cpp | 16 -- stl/src/xfexp.cpp | 63 ------ stl/src/xfsinh.cpp | 60 ----- stl/src/xmath.hpp | 82 ------- stl/src/xsinh.cpp | 77 ++++++- stl/src/xvalues.cpp | 5 +- 17 files changed, 318 insertions(+), 459 deletions(-) delete mode 100644 stl/src/xfcosh.cpp delete mode 100644 stl/src/xfdnorm.cpp delete mode 100644 stl/src/xfdscale.cpp delete mode 100644 stl/src/xfdtest.cpp delete mode 100644 stl/src/xferaise.cpp delete mode 100644 stl/src/xfexp.cpp delete mode 100644 stl/src/xfsinh.cpp delete mode 100644 stl/src/xmath.hpp diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index 1f08f33bb36..f15d7ecbbfb 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -284,13 +284,6 @@ set(SOURCES ${CMAKE_CURRENT_LIST_DIR}/src/xdateord.cpp ${CMAKE_CURRENT_LIST_DIR}/src/xdtest.cpp ${CMAKE_CURRENT_LIST_DIR}/src/xexp.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xfcosh.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xfdnorm.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xfdscale.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xfdtest.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xferaise.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xfexp.cpp - ${CMAKE_CURRENT_LIST_DIR}/src/xfsinh.cpp ${CMAKE_CURRENT_LIST_DIR}/src/xgetwctype.cpp ${CMAKE_CURRENT_LIST_DIR}/src/xlgamma.cpp ${CMAKE_CURRENT_LIST_DIR}/src/xlocale.cpp diff --git a/stl/inc/complex b/stl/inc/complex index acd727f40c6..50ca9ae0aa9 100644 --- a/stl/inc/complex +++ b/stl/inc/complex @@ -547,7 +547,7 @@ public: static bool _Isinf(_Ty _Left) noexcept { // test for infinity #if defined(__LDBL_DIG__) && __LDBL_DIG__ == 18 - return _CSTD _LDtest(&_Left) == _INFCODE; + return _CSTD _LDtest(&_Left) == FP_INFINITE; #else // ^^^ 80-bit long double (not supported by MSVC in general, see GH-1316) / 64-bit long double vvv const auto _Uint = _Bit_cast(_Left); return (_Uint & 0x7fffffffffffffffU) == 0x7ff0000000000000U; @@ -556,7 +556,7 @@ public: static _CONSTEXPR20 bool _Isnan(_Ty _Left) noexcept { #if defined(__LDBL_DIG__) && __LDBL_DIG__ == 18 - return _CSTD _LDtest(&_Left) == _NANCODE; + return _CSTD _LDtest(&_Left) == FP_NAN; #else // ^^^ 80-bit long double (not supported by MSVC in general, see GH-1316) / 64-bit long double vvv const auto _Uint = _Bit_cast(_Left); return (_Uint & 0x7fffffffffffffffU) > 0x7ff0000000000000U; diff --git a/stl/inc/ymath.h b/stl/inc/ymath.h index 6c1adbe4421..222cf9248a8 100644 --- a/stl/inc/ymath.h +++ b/stl/inc/ymath.h @@ -16,10 +16,6 @@ _STL_DISABLE_CLANG_WARNINGS _EXTERN_C_UNLESS_PURE -// macros for _Dtest return (0 => ZERO) -#define _INFCODE 1 -#define _NANCODE 2 - _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Cosh(double, double) noexcept; _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Sinh(double, double) noexcept; _CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _Exp(double*, double, short) noexcept; diff --git a/stl/msbuild/stl_base/stl.files.settings.targets b/stl/msbuild/stl_base/stl.files.settings.targets index 95fd85f8970..f4a9a133986 100644 --- a/stl/msbuild/stl_base/stl.files.settings.targets +++ b/stl/msbuild/stl_base/stl.files.settings.targets @@ -78,13 +78,6 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - - - - - - - diff --git a/stl/src/xcosh.cpp b/stl/src/xcosh.cpp index 16447fca0fd..2eb46bf197f 100644 --- a/stl/src/xcosh.cpp +++ b/stl/src/xcosh.cpp @@ -1,16 +1,18 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "xmath.hpp" +#include +#include +#include _EXTERN_C_UNLESS_PURE _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Cosh(double x, double y) noexcept { // compute y * cosh(x), |y| <= 1 - switch (_Dtest(&x)) { // test for special codes - case _NANCODE: - case _INFCODE: + switch (_STD fpclassify(x)) { // test for special codes + case FP_NAN: + case FP_INFINITE: return x; - case 0: + case FP_ZERO: return y; default: // finite if (y == 0.0) { @@ -21,16 +23,16 @@ _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Cosh(double x, double y) noexcept x = -x; } + constexpr double _Xbig = ((48 + _DOFF) + 2) * 0.347; if (x < _Xbig) { // worth adding in exp(-x) _Exp(&x, 1.0, -1); return y * (x + 0.25 / x); } - switch (_Exp(&x, y, -1)) { // report over/underflow - case 0: - _Feraise(_FE_UNDERFLOW); + switch (_Exp(&x, y, -1)) { + case FP_ZERO: // report underflow + case FP_INFINITE: // report overflow + errno = ERANGE; break; - case _INFCODE: - _Feraise(_FE_OVERFLOW); } return x; } @@ -40,4 +42,35 @@ _CRTIMP2_PURE long double __CLRCALL_PURE_OR_CDECL _LCosh(long double x, long dou return _Cosh(static_cast(x), static_cast(y)); } +_CRTIMP2_PURE float __CLRCALL_PURE_OR_CDECL _FCosh(float x, float y) noexcept { // compute y * cosh(x), |y| <= 1 + switch (_STD fpclassify(x)) { // test for special codes + case FP_NAN: + case FP_INFINITE: + return x; + case FP_ZERO: + return y; + default: // finite + if (y == 0.0) { + return y; + } + + if (x < 0.0) { + x = -x; + } + + constexpr float _FXbig = ((16 + _FOFF) + 2) * 0.347f; + if (x < _FXbig) { // worth adding in exp(-x) + _FExp(&x, 1.0F, -1); + return y * (x + 0.25F / x); + } + switch (_FExp(&x, y, -1)) { + case FP_ZERO: // report underflow + case FP_INFINITE: // report overflow + errno = ERANGE; + break; + } + return x; + } +} + _END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xdtest.cpp b/stl/src/xdtest.cpp index caaa62f0851..7f4c1906293 100644 --- a/stl/src/xdtest.cpp +++ b/stl/src/xdtest.cpp @@ -1,27 +1,23 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// _Dtest function -- IEEE 754 version - -#include "xmath.hpp" +#include _EXTERN_C_UNLESS_PURE -_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _Dtest(double* px) noexcept { // categorize *px - const auto ps = reinterpret_cast<_Dval*>(px); - - if ((ps->_Sh[_D0] & _DMASK) == _DMAX << _DOFF) { - return (ps->_Sh[_D0] & _DFRAC) != 0 || ps->_Sh[_D1] != 0 || ps->_Sh[_D2] != 0 || ps->_Sh[_D3] != 0 ? _NANCODE - : _INFCODE; - } else if ((ps->_Sh[_D0] & ~_DSIGN) != 0 || ps->_Sh[_D1] != 0 || ps->_Sh[_D2] != 0 || ps->_Sh[_D3] != 0) { - return (ps->_Sh[_D0] & _DMASK) == 0 ? _DENORM : _FINITE; - } else { - return 0; - } +// TRANSITION, ABI: preserved for binary compatibility +_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _Dtest(double* px) noexcept { + return static_cast(_STD fpclassify(*px)); } -_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _LDtest(long double* px) noexcept { // categorize *px -- 64-bit +// TRANSITION, ABI: preserved for binary compatibility +_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _LDtest(long double* px) noexcept { return _Dtest(reinterpret_cast(px)); } +// TRANSITION, ABI: preserved for binary compatibility +_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _FDtest(float* px) noexcept { + return static_cast(_STD fpclassify(*px)); +} + _END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xexp.cpp b/stl/src/xexp.cpp index 6b5a45ca5cb..6a4bca0cf63 100644 --- a/stl/src/xexp.cpp +++ b/stl/src/xexp.cpp @@ -1,9 +1,21 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "xmath.hpp" +#include +#include namespace { + constexpr int _D0 = 3; // little-endian + constexpr int _D1 = 2; + constexpr int _D2 = 1; + constexpr int _D3 = 0; + + union _Dval { // pun floating type as integer array + unsigned short _Sh[4]; + double _Val; + }; + static_assert(sizeof(_Dval::_Sh) == sizeof(_Dval::_Val), "_Dval members should be the same size"); + short _Dnorm(_Dval* ps) noexcept { // normalize double fraction short xchar = 1; unsigned short sign = static_cast(ps->_Sh[_D0] & _DSIGN); @@ -39,20 +51,20 @@ namespace { if (xchar == _DMAX) { return (ps->_Sh[_D0] & _DFRAC) != 0 || ps->_Sh[_D1] != 0 || ps->_Sh[_D2] != 0 || ps->_Sh[_D3] != 0 - ? _NANCODE - : _INFCODE; + ? FP_NAN + : FP_INFINITE; } else if (xchar == 0 && 0 < (xchar = _Dnorm(ps))) { - return 0; + return FP_ZERO; } if (0 < lexp && _DMAX - xchar <= lexp) { // overflow, return +/-INF constexpr double inf = _STD numeric_limits::infinity(); *px = ps->_Sh[_D0] & _DSIGN ? -inf : inf; - return _INFCODE; + return FP_INFINITE; } else if (-xchar < lexp) { // finite result, repack ps->_Sh[_D0] = static_cast(ps->_Sh[_D0] & ~_DMASK | (lexp + xchar) << _DOFF); - return _FINITE; + return FP_NORMAL; } else { // denormalized, scale unsigned short sign = static_cast(ps->_Sh[_D0] & _DSIGN); @@ -63,7 +75,7 @@ namespace { ps->_Sh[_D1] = 0; ps->_Sh[_D2] = 0; ps->_Sh[_D3] = 0; - return 0; + return FP_ZERO; } else { // nonzero, align fraction short xexp = static_cast(lexp); unsigned short psx = 0; @@ -89,13 +101,128 @@ namespace { && (++ps->_Sh[_D2] & 0xffff) == 0 && (++ps->_Sh[_D1] & 0xffff) == 0) { ++ps->_Sh[_D0]; // round up } else if (ps->_Sh[_D0] == sign && ps->_Sh[_D1] == 0 && ps->_Sh[_D2] == 0 && ps->_Sh[_D3] == 0) { - return 0; + return FP_ZERO; } - return _FINITE; + return FP_NORMAL; + } + } + } + + constexpr int _F0 = 1; // little-endian + constexpr int _F1 = 0; + + union _Fval { // pun floating type as integer array + unsigned short _Sh[2]; + float _Val; + }; + static_assert(sizeof(_Fval::_Sh) == sizeof(_Fval::_Val), "_Fval members should be the same size"); + + short _FDnorm(_Fval* ps) noexcept { // normalize float fraction + short xchar = 1; + unsigned short sign = static_cast(ps->_Sh[_F0] & _FSIGN); + + if ((ps->_Sh[_F0] &= _FFRAC) != 0 || ps->_Sh[_F1]) { // nonzero, scale + if (ps->_Sh[_F0] == 0) { + ps->_Sh[_F0] = ps->_Sh[_F1]; + ps->_Sh[_F1] = 0; + xchar -= 16; } + + for (; ps->_Sh[_F0] < 1 << _FOFF; --xchar) { // shift left by 1 + ps->_Sh[_F0] = static_cast(ps->_Sh[_F0] << 1 | ps->_Sh[_F1] >> 15); + ps->_Sh[_F1] <<= 1; + } + for (; 1 << (_FOFF + 1) <= ps->_Sh[_F0]; ++xchar) { // shift right by 1 + ps->_Sh[_F1] = static_cast(ps->_Sh[_F1] >> 1 | ps->_Sh[_F0] << 15); + ps->_Sh[_F0] >>= 1; + } + ps->_Sh[_F0] &= _FFRAC; + } + ps->_Sh[_F0] |= sign; + return xchar; + } + + short _FDscale(float* px, long lexp) noexcept { // scale *px by 2^xexp with checking + const auto ps = reinterpret_cast<_Fval*>(px); + short xchar = static_cast((ps->_Sh[_F0] & _FMASK) >> _FOFF); + + if (xchar == _FMAX) { + return (ps->_Sh[_F0] & _FFRAC) != 0 || ps->_Sh[_F1] != 0 ? FP_NAN : FP_INFINITE; + } else if (xchar == 0 && 0 < (xchar = _FDnorm(ps))) { + return FP_ZERO; } + + if (0 < lexp && _FMAX - xchar <= lexp) { // overflow, return +/-INF + constexpr float inf = _STD numeric_limits::infinity(); + + *px = ps->_Sh[_F0] & _FSIGN ? -inf : inf; + return FP_INFINITE; + } else if (-xchar < lexp) { // finite result, repack + ps->_Sh[_F0] = static_cast(ps->_Sh[_F0] & ~_FMASK | (lexp + xchar) << _FOFF); + return FP_NORMAL; + } else { // denormalized, scale + unsigned short sign = static_cast(ps->_Sh[_F0] & _FSIGN); + + ps->_Sh[_F0] = static_cast(1 << _FOFF | ps->_Sh[_F0] & _FFRAC); + lexp += xchar - 1; + if (lexp < -(16 + 1 + _FOFF) || 0 <= lexp) { // underflow, return +/-0 + ps->_Sh[_F0] = sign; + ps->_Sh[_F1] = 0; + return FP_ZERO; + } else { // nonzero, align fraction + short xexp = static_cast(lexp); + unsigned short psx = 0; + + if (xexp <= -16) { // scale by words + psx = ps->_Sh[_F1]; + ps->_Sh[_F1] = ps->_Sh[_F0]; + ps->_Sh[_F0] = 0; + xexp += 16; + } + + if (xexp != 0) { // scale by bits + xexp = -xexp; + psx = (ps->_Sh[_F1] << (16 - xexp)) | (psx != 0 ? 1 : 0); + ps->_Sh[_F1] = static_cast(ps->_Sh[_F1] >> xexp | ps->_Sh[_F0] << (16 - xexp)); + ps->_Sh[_F0] >>= xexp; + } + + ps->_Sh[_F0] |= sign; + if ((0x8000 < psx || 0x8000 == psx && (ps->_Sh[_F1] & 0x0001) != 0) && (++ps->_Sh[_F1] & 0xffff) == 0) { + ++ps->_Sh[_F0]; // round up + } else if (ps->_Sh[_F0] == sign && ps->_Sh[_F1] == 0) { + return FP_ZERO; + } + + return FP_NORMAL; + } + } + } + +// raise IEEE 754 exceptions +#ifndef _M_CEE_PURE +#pragma float_control(except, on, push) +#endif + + template + [[nodiscard]] T _Xfe_overflow(const T sign) noexcept { + static_assert(_STD is_floating_point_v, "Expected is_floating_point_v."); + constexpr T huge = _STD numeric_limits::max(); + return _STD copysign(huge, sign) * huge; } + + template + [[nodiscard]] T _Xfe_underflow(const T sign) noexcept { + static_assert(_STD is_floating_point_v, "Expected is_floating_point_v."); + constexpr T tiny = _STD numeric_limits::min(); + return _STD copysign(tiny, sign) * tiny; + } + +#ifndef _M_CEE_PURE +#pragma float_control(pop) +#endif + } // unnamed namespace _EXTERN_C_UNLESS_PURE @@ -113,13 +240,13 @@ _CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _Exp(double* px, double y, short eof if (y == 0.0) { // zero *px = y; - return 0; + return FP_ZERO; } else if (*px < -hugexp) { // certain underflow *px = _Xfe_underflow(y); - return 0; + return FP_ZERO; } else if (hugexp < *px) { // certain overflow *px = _Xfe_overflow(y); - return _INFCODE; + return FP_INFINITE; } else { // xexp won't overflow double g = *px * invln2; short xexp = static_cast(g + (g < 0.0 ? -0.5 : +0.5)); @@ -142,10 +269,10 @@ _CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _Exp(double* px, double y, short eof const short result_code = _Dscale(px, static_cast(xexp) + eoff); switch (result_code) { - case 0: + case FP_ZERO: *px = _Xfe_underflow(y); break; - case _INFCODE: + case FP_INFINITE: *px = _Xfe_overflow(y); break; default: @@ -160,4 +287,59 @@ _CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _LExp(long double* px, long double y return _Exp(reinterpret_cast(px), static_cast(y), eoff); } +_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _FExp(float* px, float y, short eoff) noexcept { + // compute y * e^(*px), (*px) finite, |y| not huge + + static constexpr float p[] = {1.0F, 60.09114349F}; + static constexpr float q[] = {12.01517514F, 120.18228722F}; + constexpr float c1 = (22713.0F / 32768.0F); + constexpr float c2 = 1.4286068203094172321214581765680755e-6F; + constexpr float hugexp = static_cast(_FMAX * 900L / 1000); + constexpr float invln2 = 1.4426950408889634073599246810018921F; + + if (y == 0.0F) { // zero + *px = y; + return FP_ZERO; + } else if (*px < -hugexp) { // certain underflow + *px = _Xfe_underflow(y); + return FP_ZERO; + } else if (hugexp < *px) { // certain overflow + *px = _Xfe_overflow(y); + return FP_INFINITE; + } else { // xexp won't overflow + float g = *px * invln2; + short xexp = static_cast(g + (g < 0.0F ? -0.5F : +0.5F)); + + g = xexp; + g = static_cast((*px - g * c1) - g * c2); + + constexpr float eps = 0x1p-25f; + if (-eps < g && g < eps) { + *px = y; + } else { // g * g worth computing + const float z = g * g; + const float w = q[0] * z + q[1]; + + g *= z + p[1]; + *px = (w + g) / (w - g) * 2.0F * y; + --xexp; + } + + const short result_code = _FDscale(px, static_cast(xexp) + eoff); + + switch (result_code) { + case FP_ZERO: + *px = _Xfe_underflow(y); + break; + case FP_INFINITE: + *px = _Xfe_overflow(y); + break; + default: + break; + } + + return result_code; + } +} + _END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xfcosh.cpp b/stl/src/xfcosh.cpp deleted file mode 100644 index 5fb3ce51814..00000000000 --- a/stl/src/xfcosh.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -_CRTIMP2_PURE float __CLRCALL_PURE_OR_CDECL _FCosh(float x, float y) noexcept { // compute y * cosh(x), |y| <= 1 - switch (_FDtest(&x)) { // test for special codes - case _NANCODE: - case _INFCODE: - return x; - case 0: - return y; - default: // finite - if (y == 0.0) { - return y; - } - - if (x < 0.0) { - x = -x; - } - - if (x < _FXbig) { // worth adding in exp(-x) - _FExp(&x, 1.0F, -1); - return y * (x + 0.25F / x); - } - switch (_FExp(&x, y, -1)) { // report over/underflow - case 0: - _Feraise(_FE_UNDERFLOW); - break; - case _INFCODE: - _Feraise(_FE_OVERFLOW); - } - return x; - } -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xfdnorm.cpp b/stl/src/xfdnorm.cpp deleted file mode 100644 index 4ac5e671a0f..00000000000 --- a/stl/src/xfdnorm.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -// _FDnorm function -- IEEE 754 version - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -short _FDnorm(_Fval* ps) noexcept { // normalize float fraction - short xchar = 1; - unsigned short sign = static_cast(ps->_Sh[_F0] & _FSIGN); - - if ((ps->_Sh[_F0] &= _FFRAC) != 0 || ps->_Sh[_F1]) { // nonzero, scale - if (ps->_Sh[_F0] == 0) { - ps->_Sh[_F0] = ps->_Sh[_F1]; - ps->_Sh[_F1] = 0; - xchar -= 16; - } - - for (; ps->_Sh[_F0] < 1 << _FOFF; --xchar) { // shift left by 1 - ps->_Sh[_F0] = static_cast(ps->_Sh[_F0] << 1 | ps->_Sh[_F1] >> 15); - ps->_Sh[_F1] <<= 1; - } - for (; 1 << (_FOFF + 1) <= ps->_Sh[_F0]; ++xchar) { // shift right by 1 - ps->_Sh[_F1] = static_cast(ps->_Sh[_F1] >> 1 | ps->_Sh[_F0] << 15); - ps->_Sh[_F0] >>= 1; - } - ps->_Sh[_F0] &= _FFRAC; - } - ps->_Sh[_F0] |= sign; - return xchar; -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xfdscale.cpp b/stl/src/xfdscale.cpp deleted file mode 100644 index cb5e27c2ef5..00000000000 --- a/stl/src/xfdscale.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -// _FDscale function -- IEEE 754 version - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -short _FDscale(float* px, long lexp) noexcept { // scale *px by 2^xexp with checking - const auto ps = reinterpret_cast<_Fval*>(px); - short xchar = static_cast((ps->_Sh[_F0] & _FMASK) >> _FOFF); - - if (xchar == _FMAX) { - return (ps->_Sh[_F0] & _FFRAC) != 0 || ps->_Sh[_F1] != 0 ? _NANCODE : _INFCODE; - } else if (xchar == 0 && 0 < (xchar = _FDnorm(ps))) { - return 0; - } - - if (0 < lexp && _FMAX - xchar <= lexp) { // overflow, return +/-INF - constexpr float inf = _STD numeric_limits::infinity(); - - *px = ps->_Sh[_F0] & _FSIGN ? -inf : inf; - return _INFCODE; - } else if (-xchar < lexp) { // finite result, repack - ps->_Sh[_F0] = static_cast(ps->_Sh[_F0] & ~_FMASK | (lexp + xchar) << _FOFF); - return _FINITE; - } else { // denormalized, scale - unsigned short sign = static_cast(ps->_Sh[_F0] & _FSIGN); - - ps->_Sh[_F0] = static_cast(1 << _FOFF | ps->_Sh[_F0] & _FFRAC); - lexp += xchar - 1; - if (lexp < -(16 + 1 + _FOFF) || 0 <= lexp) { // underflow, return +/-0 - ps->_Sh[_F0] = sign; - ps->_Sh[_F1] = 0; - return 0; - } else { // nonzero, align fraction - short xexp = static_cast(lexp); - unsigned short psx = 0; - - if (xexp <= -16) { // scale by words - psx = ps->_Sh[_F1]; - ps->_Sh[_F1] = ps->_Sh[_F0]; - ps->_Sh[_F0] = 0; - xexp += 16; - } - - if (xexp != 0) { // scale by bits - xexp = -xexp; - psx = (ps->_Sh[_F1] << (16 - xexp)) | (psx != 0 ? 1 : 0); - ps->_Sh[_F1] = static_cast(ps->_Sh[_F1] >> xexp | ps->_Sh[_F0] << (16 - xexp)); - ps->_Sh[_F0] >>= xexp; - } - - ps->_Sh[_F0] |= sign; - if ((0x8000 < psx || 0x8000 == psx && (ps->_Sh[_F1] & 0x0001) != 0) && (++ps->_Sh[_F1] & 0xffff) == 0) { - ++ps->_Sh[_F0]; // round up - } else if (ps->_Sh[_F0] == sign && ps->_Sh[_F1] == 0) { - return 0; - } - - return _FINITE; - } - } -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xfdtest.cpp b/stl/src/xfdtest.cpp deleted file mode 100644 index 04556bca328..00000000000 --- a/stl/src/xfdtest.cpp +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -// _FDtest function -- IEEE 754 version - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _FDtest(float* px) noexcept { // categorize *px - const auto ps = reinterpret_cast<_Fval*>(px); - - if ((ps->_Sh[_F0] & _FMASK) == _FMAX << _FOFF) { - return (ps->_Sh[_F0] & _FFRAC) != 0 || ps->_Sh[_F1] != 0 ? _NANCODE : _INFCODE; - } else if ((ps->_Sh[_F0] & ~_FSIGN) != 0 || ps->_Sh[_F1] != 0) { - return (ps->_Sh[_F0] & _FMASK) == 0 ? _DENORM : _FINITE; - } else { - return 0; - } -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xferaise.cpp b/stl/src/xferaise.cpp deleted file mode 100644 index b973fc23824..00000000000 --- a/stl/src/xferaise.cpp +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -void __CLRCALL_PURE_OR_CDECL _Feraise(int except) noexcept { // report floating-point exception - if ((except & (_FE_DIVBYZERO | _FE_INVALID)) != 0) { - errno = EDOM; - } else if ((except & (_FE_UNDERFLOW | _FE_OVERFLOW)) != 0) { - errno = ERANGE; - } -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xfexp.cpp b/stl/src/xfexp.cpp deleted file mode 100644 index cfe5b7776d5..00000000000 --- a/stl/src/xfexp.cpp +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _FExp(float* px, float y, short eoff) noexcept { - // compute y * e^(*px), (*px) finite, |y| not huge - - static constexpr float p[] = {1.0F, 60.09114349F}; - static constexpr float q[] = {12.01517514F, 120.18228722F}; - constexpr float c1 = (22713.0F / 32768.0F); - constexpr float c2 = 1.4286068203094172321214581765680755e-6F; - constexpr float hugexp = static_cast(_FMAX * 900L / 1000); - constexpr float invln2 = 1.4426950408889634073599246810018921F; - - if (y == 0.0F) { // zero - *px = y; - return 0; - } else if (*px < -hugexp) { // certain underflow - *px = _Xfe_underflow(y); - return 0; - } else if (hugexp < *px) { // certain overflow - *px = _Xfe_overflow(y); - return _INFCODE; - } else { // xexp won't overflow - float g = *px * invln2; - short xexp = static_cast(g + (g < 0.0F ? -0.5F : +0.5F)); - - g = xexp; - g = static_cast((*px - g * c1) - g * c2); - - constexpr float eps = 0x1p-25f; - if (-eps < g && g < eps) { - *px = y; - } else { // g * g worth computing - const float z = g * g; - const float w = q[0] * z + q[1]; - - g *= z + p[1]; - *px = (w + g) / (w - g) * 2.0F * y; - --xexp; - } - - const short result_code = _FDscale(px, static_cast(xexp) + eoff); - - switch (result_code) { - case 0: - *px = _Xfe_underflow(y); - break; - case _INFCODE: - *px = _Xfe_overflow(y); - break; - default: - break; - } - - return result_code; - } -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xfsinh.cpp b/stl/src/xfsinh.cpp deleted file mode 100644 index 4e551083f6e..00000000000 --- a/stl/src/xfsinh.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#include "xmath.hpp" - -_EXTERN_C_UNLESS_PURE - -_CRTIMP2_PURE float __CLRCALL_PURE_OR_CDECL _FSinh(float x, float y) noexcept { - // compute y * sinh(x), |y| <= 1 - - // coefficients - static constexpr float p[] = {0.00020400F, 0.00832983F, 0.16666737F, 0.99999998F}; - - short neg; - - switch (_FDtest(&x)) { // test for special codes - case _NANCODE: - return x; - case _INFCODE: - return y != 0.0F ? x : FSIGN(x) ? -y : y; - case 0: - return x * y; - default: // finite - if (y == 0.0F) { - return x < 0.0F ? -y : y; - } - - if (x < 0.0F) { - x = -x; - neg = 1; - } else { - neg = 0; - } - - constexpr float rteps = 0x1p-12f; - if (x < rteps) { - x *= y; // x tiny - } else if (x < 1.0F) { - float w = x * x; - - x += ((p[0] * w + p[1]) * w + p[2]) * w * x; - x *= y; - } else if (x < _FXbig) { // worth adding in exp(-x) - _FExp(&x, 1.0F, -1); - x = y * (x - 0.25F / x); - } else { - switch (_FExp(&x, y, -1)) { // report over/underflow - case 0: - _Feraise(_FE_UNDERFLOW); - break; - case _INFCODE: - _Feraise(_FE_OVERFLOW); - } - } - - return neg ? -x : x; - } -} - -_END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xmath.hpp b/stl/src/xmath.hpp deleted file mode 100644 index a884f145f89..00000000000 --- a/stl/src/xmath.hpp +++ /dev/null @@ -1,82 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -#pragma once - -#include -#include -#include -#include - -// macros for _Feraise argument -#define _FE_DIVBYZERO 0x04 -#define _FE_INVALID 0x01 -#define _FE_OVERFLOW 0x08 -#define _FE_UNDERFLOW 0x10 - -// float properties -#define _D0 3 // little-endian, small long doubles -#define _D1 2 -#define _D2 1 -#define _D3 0 - -#define _F0 1 // little-endian -#define _F1 0 - -#define DSIGN(x) (reinterpret_cast<_Dval*>(&(x))->_Sh[_D0] & _DSIGN) -#define FSIGN(x) (reinterpret_cast<_Fval*>(&(x))->_Sh[_F0] & _FSIGN) - -// macros for _Dtest return (0 => ZERO) -#define _DENORM (-2) // C9X only -#define _FINITE (-1) - -_EXTERN_C_UNLESS_PURE - -void __CLRCALL_PURE_OR_CDECL _Feraise(int) noexcept; - -_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _Dtest(double*) noexcept; -_CRTIMP2_PURE short __CLRCALL_PURE_OR_CDECL _FDtest(float*) noexcept; - -// double declarations -union _Dval { // pun floating type as integer array - unsigned short _Sh[8]; - double _Val; -}; - -extern const double _Xbig; - -// float declarations -union _Fval { // pun floating type as integer array - unsigned short _Sh[8]; - float _Val; -}; - -short _FDnorm(_Fval*) noexcept; -short _FDscale(float*, long) noexcept; - -extern const float _FXbig; - -_END_EXTERN_C_UNLESS_PURE - -// raise IEEE 754 exceptions -#ifndef _M_CEE_PURE -#pragma float_control(except, on, push) -#endif - -template -[[nodiscard]] T _Xfe_overflow(const T sign) noexcept { - static_assert(_STD is_floating_point_v, "Expected is_floating_point_v."); - constexpr T huge = _STD numeric_limits::max(); - return _STD copysign(huge, sign) * huge; -} - -template -[[nodiscard]] T _Xfe_underflow(const T sign) noexcept { - static_assert(_STD is_floating_point_v, "Expected is_floating_point_v."); - constexpr T tiny = _STD numeric_limits::min(); - return _STD copysign(tiny, sign) * tiny; -} - -#ifndef _M_CEE_PURE -#pragma float_control(pop) -#endif diff --git a/stl/src/xsinh.cpp b/stl/src/xsinh.cpp index 1def63635b0..44914ad9af8 100644 --- a/stl/src/xsinh.cpp +++ b/stl/src/xsinh.cpp @@ -1,9 +1,10 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#include +#include #include - -#include "xmath.hpp" +#include namespace { double _Poly(double x, const double* tab, int n) noexcept { // compute polynomial @@ -29,12 +30,12 @@ _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Sinh(double x, double y) noexcept short neg; - switch (_Dtest(&x)) { // test for special codes - case _NANCODE: + switch (_STD fpclassify(x)) { // test for special codes + case FP_NAN: return x; - case _INFCODE: - return y != 0.0 ? x : DSIGN(x) ? -y : y; - case 0: + case FP_INFINITE: + return y != 0.0 ? x : _STD signbit(x) ? -y : y; + case FP_ZERO: return x * y; default: // finite if (y == 0.0) { @@ -49,6 +50,7 @@ _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Sinh(double x, double y) noexcept } constexpr double rteps = 0x1p-27; + constexpr double _Xbig = ((48 + _DOFF) + 2) * 0.347; if (x < rteps) { x *= y; // x tiny } else if (x < 1.0) { @@ -60,12 +62,11 @@ _CRTIMP2_PURE double __CLRCALL_PURE_OR_CDECL _Sinh(double x, double y) noexcept _Exp(&x, 1.0, -1); x = y * (x - 0.25 / x); } else { - switch (_Exp(&x, y, -1)) { // report over/underflow - case 0: - _Feraise(_FE_UNDERFLOW); + switch (_Exp(&x, y, -1)) { + case FP_ZERO: // report underflow + case FP_INFINITE: // report overflow + errno = ERANGE; break; - case _INFCODE: - _Feraise(_FE_OVERFLOW); } } @@ -77,4 +78,56 @@ _CRTIMP2_PURE long double __CLRCALL_PURE_OR_CDECL _LSinh(long double x, long dou return _Sinh(static_cast(x), static_cast(y)); } +_CRTIMP2_PURE float __CLRCALL_PURE_OR_CDECL _FSinh(float x, float y) noexcept { + // compute y * sinh(x), |y| <= 1 + + // coefficients + static constexpr float p[] = {0.00020400F, 0.00832983F, 0.16666737F, 0.99999998F}; + + short neg; + + switch (_STD fpclassify(x)) { // test for special codes + case FP_NAN: + return x; + case FP_INFINITE: + return y != 0.0F ? x : _STD signbit(x) ? -y : y; + case FP_ZERO: + return x * y; + default: // finite + if (y == 0.0F) { + return x < 0.0F ? -y : y; + } + + if (x < 0.0F) { + x = -x; + neg = 1; + } else { + neg = 0; + } + + constexpr float rteps = 0x1p-12f; + constexpr float _FXbig = ((16 + _FOFF) + 2) * 0.347f; + if (x < rteps) { + x *= y; // x tiny + } else if (x < 1.0F) { + float w = x * x; + + x += ((p[0] * w + p[1]) * w + p[2]) * w * x; + x *= y; + } else if (x < _FXbig) { // worth adding in exp(-x) + _FExp(&x, 1.0F, -1); + x = y * (x - 0.25F / x); + } else { + switch (_FExp(&x, y, -1)) { + case FP_ZERO: // report underflow + case FP_INFINITE: // report overflow + errno = ERANGE; + break; + } + } + + return neg ? -x : x; + } +} + _END_EXTERN_C_UNLESS_PURE diff --git a/stl/src/xvalues.cpp b/stl/src/xvalues.cpp index 0fb45636130..020c3c823a0 100644 --- a/stl/src/xvalues.cpp +++ b/stl/src/xvalues.cpp @@ -9,13 +9,10 @@ #endif #endif -#include "xmath.hpp" +#include _EXTERN_C_UNLESS_PURE -extern const double _Xbig = ((48 + _DOFF) + 2) * 0.347; -extern const float _FXbig = ((16 + _FOFF) + 2) * 0.347f; - // TRANSITION, ABI: preserved for binary compatibility union _Dconst { // pun float types as integer array unsigned short _Word[8]; From 649a76a4825a8e7b42daecb665ce2193578db2dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Thu, 8 Jan 2026 18:20:58 +0100 Subject: [PATCH 06/19] ``: Simplify unwinding (#5961) --- stl/inc/regex | 58 +++++++++++++++++---------------------------------- 1 file changed, 19 insertions(+), 39 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 37621a24271..8343f4cf69b 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1671,8 +1671,7 @@ public: enum class _Rx_unwind_ops { _After_assert = 1, _After_neg_assert, - _Disjunction_eval_alt_on_failure, - _Disjunction_eval_alt_always, + _Disjunction_eval_alternative, _Do_nothing, _Loop_simple_nongreedy, _Loop_simple_greedy_firstrep, @@ -4053,9 +4052,7 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N { auto _Node = static_cast<_Node_if*>(_Nx); if (_Node->_Child) { - _Push_frame(_Longest ? _Rx_unwind_ops::_Disjunction_eval_alt_always - : _Rx_unwind_ops::_Disjunction_eval_alt_on_failure, - _Node->_Child); + _Push_frame(_Rx_unwind_ops::_Disjunction_eval_alternative, _Node->_Child); _Increase_complexity_count(); } break; @@ -4296,34 +4293,24 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N } while (_Nx); while (_Frames_count > 0 && !_Nx) { - auto& _Frame = _Frames[--_Frames_count]; + _STL_INTERNAL_CHECK(_Failed || _Longest); + auto& _Frame = _Frames[--_Frames_count]; switch (_Frame._Code) { - case _Rx_unwind_ops::_After_assert: - { // matching pattern of positive assert failed - _STL_INTERNAL_CHECK(_Failed); - break; - } + case _Rx_unwind_ops::_After_assert: // matching pattern of positive assert failed + case _Rx_unwind_ops::_Do_nothing: + break; case _Rx_unwind_ops::_After_neg_assert: { // matching pattern of negative assert failed - _STL_INTERNAL_CHECK(_Failed); _Tgt_state._Cur = _Frame._Pos; _Nx = _Frame._Node->_Next; _Failed = false; break; } - case _Rx_unwind_ops::_Disjunction_eval_alt_on_failure: - // evaluate next alternative if matching prior alternatives failed - if (!_Failed) { - break; - } - _FALLTHROUGH; - - case _Rx_unwind_ops::_Disjunction_eval_alt_always: - // evaluate next alternative no matter if matching prior alternatives succeeded - { + case _Rx_unwind_ops::_Disjunction_eval_alternative: + { // try next alternative in a disjunction auto _Node = static_cast<_Node_if*>(_Frame._Node); _Nx = _Node->_Next; _Tgt_state._Cur = _Frame._Pos; @@ -4336,12 +4323,8 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N break; } - case _Rx_unwind_ops::_Do_nothing: - break; - case _Rx_unwind_ops::_Loop_simple_nongreedy: - // try one more rep after matching tail if necessary - if (_Longest || _Failed) { + { // try one more rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); auto& _Sav = _Loop_vals[_Node->_Loop_number]; @@ -4356,8 +4339,7 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N break; case _Rx_unwind_ops::_Loop_simple_greedy_firstrep: - // try tail after backtracking from first rep - if (_Failed) { + { // try tail after backtracking from first rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); _Increase_complexity_count(); @@ -4368,9 +4350,9 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N break; case _Rx_unwind_ops::_Loop_simple_greedy_intermediaterep: - // shift capturing groups, set up unwinding prior rep and try tail - // when backtracking between the second and the last attempted rep - if (_Failed) { + { + // shift capturing groups, set up unwinding prior rep and try tail + // when backtracking between the second and the last attempted rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); // adjust capturing group begin and end iterators by rep length @@ -4390,9 +4372,9 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _FALLTHROUGH; case _Rx_unwind_ops::_Loop_simple_greedy_lastrep: - // set up unwinding prior rep and try tail - // when backtracking from last attempted rep - if (_Failed) { + { + // set up unwinding prior rep and try tail + // when backtracking from last attempted rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); _Increase_complexity_count(); @@ -4412,8 +4394,7 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N break; case _Rx_unwind_ops::_Loop_greedy: - // try tail if matching one more rep failed - if (_Failed) { + { // try tail auto _Node = static_cast<_Node_rep*>(_Frame._Node); _Increase_complexity_count(); @@ -4434,8 +4415,7 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N break; case _Rx_unwind_ops::_Loop_nongreedy: - // try another rep if matching tail failed or longest mode - if (_Failed || _Longest) { + { // try another rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); auto& _Sav = _Loop_vals[_Node->_Loop_number]; From 12b58599925cbe45c7c0befe0ec32a19c947815d Mon Sep 17 00:00:00 2001 From: TPPPP <90049492+TPPPP72@users.noreply.github.com> Date: Fri, 9 Jan 2026 01:22:56 +0800 Subject: [PATCH 07/19] Update working draft revision to N5032 (#5967) --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 46e043a6b2e..3524ea9266f 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ issue. The [bug tag][] and [enhancement tag][] are being populated. # Goals -We're implementing the latest C++ Working Draft, currently [N5014][], which will eventually become the next C++ +We're implementing the latest C++ Working Draft, currently [N5032][], which will eventually become the next C++ International Standard. The terms Working Draft (WD) and Working Paper (WP) are interchangeable; we often informally refer to these drafts as "the Standard" while being aware of the difference. (There are other relevant Standards; for example, supporting `/std:c++14` and `/std:c++17` involves understanding how the C++14 and C++17 @@ -584,7 +584,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception [LWG issues]: https://cplusplus.github.io/LWG/lwg-toc.html [LWG tag]: https://github.com/microsoft/STL/issues?q=is%3Aopen+is%3Aissue+label%3ALWG [Microsoft Open Source Code of Conduct]: https://opensource.microsoft.com/codeofconduct/ -[N5014]: https://wg21.link/N5014 +[N5032]: https://wg21.link/N5032 [NOTICE.txt]: NOTICE.txt [STL-CI-badge]: https://dev.azure.com/vclibs/STL/_apis/build/status%2FSTL-CI?branchName=main "STL-CI" [STL-CI-link]: https://dev.azure.com/vclibs/STL/_build/latest?definitionId=4&branchName=main From e7d5bf75c6183588d6f8881b218d914d2781ddb4 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 8 Jan 2026 19:24:28 +0200 Subject: [PATCH 08/19] ``: drop `_Atomic_reinterpret_as` (#5973) --- stl/inc/atomic | 88 +++++++++++++++++++------------------------------- 1 file changed, 34 insertions(+), 54 deletions(-) diff --git a/stl/inc/atomic b/stl/inc/atomic index 831e2654d94..839787c5366 100644 --- a/stl/inc/atomic +++ b/stl/inc/atomic @@ -341,21 +341,6 @@ _NODISCARD inline memory_order _Combine_cas_memory_orders( return _Combined_memory_orders[static_cast(_Success)][static_cast(_Failure)]; } -template -_NODISCARD _Integral _Atomic_reinterpret_as(const _Ty& _Source) noexcept { - // interprets _Source as the supplied integral type - static_assert(is_integral_v<_Integral>, "Tried to reinterpret memory as non-integral"); - if constexpr (is_integral_v<_Ty> && sizeof(_Integral) == sizeof(_Ty)) { - return static_cast<_Integral>(_Source); - } else if constexpr (is_pointer_v<_Ty> && sizeof(_Integral) == sizeof(_Ty)) { - return reinterpret_cast<_Integral>(_Source); - } else { - _Integral _Result{}; // zero padding bits - _CSTD memcpy(&_Result, _STD addressof(_Source), sizeof(_Source)); - return _Result; - } -} - #if 1 // TRANSITION, ABI template struct _Atomic_padded { @@ -428,13 +413,13 @@ void _Atomic_wait_direct( const auto _Storage_ptr = const_cast(static_cast(_STD addressof(_This->_Storage))); for (;;) { - const _Value_type _Observed_bytes = _STD _Atomic_reinterpret_as<_Value_type>(_This->load(_Order)); + const _Value_type _Observed_bytes = _STD _Bit_cast<_Value_type>(_This->load(_Order)); if (_Expected_bytes != _Observed_bytes) { #if _CMPXCHG_MASK_OUT_PADDING_BITS using _TVal = _Remove_cvref_t<_Ty>; if constexpr (_Might_have_non_value_bits<_TVal>) { _Storage_for<_TVal> _Mask{_Form_mask}; - const _Value_type _Mask_val = _STD _Atomic_reinterpret_as<_Value_type>(_Mask._Ref()); + const _Value_type _Mask_val = _STD _Bit_cast<_Value_type>(_Mask._Ref()); if (((_Expected_bytes ^ _Observed_bytes) & _Mask_val) == 0) { _Expected_bytes = _Observed_bytes; @@ -681,13 +666,13 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _STD _Atomic_address_as(_Storage); - const char _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const char _As_bytes = _STD _Bit_cast(_Value); _ATOMIC_STORE_SEQ_CST(8, _Mem, _As_bytes) } void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const auto _Mem = _STD _Atomic_address_as(_Storage); - const char _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const char _As_bytes = _STD _Bit_cast(_Value); _Check_store_memory_order(_Order); @@ -717,24 +702,23 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics // exchange with given memory order char _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _As_bytes, _InterlockedExchange8, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Value)); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Value)); return reinterpret_cast<_TVal&>(_As_bytes); } bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - char _Expected_bytes = _STD _Atomic_reinterpret_as(_Expected); // read before atomic operation + char _Expected_bytes = _STD _Bit_cast(_Expected); // read before atomic operation char _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS if constexpr (_Might_have_non_value_bits<_TVal>) { _Storage_for<_TVal> _Mask{_Form_mask}; - const char _Mask_val = _STD _Atomic_reinterpret_as(_Mask._Ref()); + const char _Mask_val = _STD _Bit_cast(_Mask._Ref()); for (;;) { _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange8, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), - _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -748,7 +732,7 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics } #endif // _CMPXCHG_MASK_OUT_PADDING_BITS _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange8, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -759,7 +743,7 @@ struct _Atomic_storage<_Ty, 1> { // lock-free using 1-byte intrinsics #if _HAS_CXX20 void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { - _STD _Atomic_wait_direct(this, _STD _Atomic_reinterpret_as(_Expected), _Order); + _STD _Atomic_wait_direct(this, _STD _Bit_cast(_Expected), _Order); } void notify_one() noexcept { @@ -788,13 +772,13 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _STD _Atomic_address_as(_Storage); - const short _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const short _As_bytes = _STD _Bit_cast(_Value); _ATOMIC_STORE_SEQ_CST(16, _Mem, _As_bytes) } void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const auto _Mem = _STD _Atomic_address_as(_Storage); - const short _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const short _As_bytes = _STD _Bit_cast(_Value); _Check_store_memory_order(_Order); @@ -824,23 +808,22 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics // exchange with given memory order short _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _As_bytes, _InterlockedExchange16, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Value)); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Value)); return reinterpret_cast<_TVal&>(_As_bytes); } bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - short _Expected_bytes = _STD _Atomic_reinterpret_as(_Expected); // read before atomic operation + short _Expected_bytes = _STD _Bit_cast(_Expected); // read before atomic operation short _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS if constexpr (_Might_have_non_value_bits<_Ty>) { _Storage_for<_TVal> _Mask{_Form_mask}; - const short _Mask_val = _STD _Atomic_reinterpret_as(_Mask._Ref()); + const short _Mask_val = _STD _Bit_cast(_Mask._Ref()); for (;;) { _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange16, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), - _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -854,7 +837,7 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics } #endif // _CMPXCHG_MASK_OUT_PADDING_BITS _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange16, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -865,7 +848,7 @@ struct _Atomic_storage<_Ty, 2> { // lock-free using 2-byte intrinsics #if _HAS_CXX20 void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { - _STD _Atomic_wait_direct(this, _STD _Atomic_reinterpret_as(_Expected), _Order); + _STD _Atomic_wait_direct(this, _STD _Bit_cast(_Expected), _Order); } void notify_one() noexcept { @@ -894,13 +877,13 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _STD _Atomic_address_as(_Storage); - const int _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const int _As_bytes = _STD _Bit_cast(_Value); _ATOMIC_STORE_32_SEQ_CST(_Mem, _As_bytes) } void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const auto _Mem = _STD _Atomic_address_as(_Storage); - const int _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const int _As_bytes = _STD _Bit_cast(_Value); _Check_store_memory_order(_Order); @@ -930,23 +913,22 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics // exchange with given memory order long _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _As_bytes, _InterlockedExchange, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Value)); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Value)); return reinterpret_cast<_TVal&>(_As_bytes); } bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - long _Expected_bytes = _STD _Atomic_reinterpret_as(_Expected); // read before atomic operation + long _Expected_bytes = _STD _Bit_cast(_Expected); // read before atomic operation long _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS if constexpr (_Might_have_non_value_bits<_TVal>) { _Storage_for<_TVal> _Mask{_Form_mask}; - const long _Mask_val = _STD _Atomic_reinterpret_as(_Mask); + const long _Mask_val = _STD _Bit_cast(_Mask._Ref()); for (;;) { _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), - _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -960,7 +942,7 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics } #endif // _CMPXCHG_MASK_OUT_PADDING_BITS _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -971,7 +953,7 @@ struct _Atomic_storage<_Ty, 4> { // lock-free using 4-byte intrinsics #if _HAS_CXX20 void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { - _STD _Atomic_wait_direct(this, _STD _Atomic_reinterpret_as(_Expected), _Order); + _STD _Atomic_wait_direct(this, _STD _Bit_cast(_Expected), _Order); } void notify_one() noexcept { @@ -1000,7 +982,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics void store(const _TVal _Value) noexcept { // store with sequential consistency const auto _Mem = _STD _Atomic_address_as(_Storage); - const long long _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const long long _As_bytes = _STD _Bit_cast(_Value); #if defined(__clang__) && defined(_M_IX86) // TRANSITION, LLVM-126516 static_assert(_M_IX86_FP != 0, "8 byte atomic store is not supported on clang-cl with /arch:IA32"); __atomic_store_n(_Mem, _As_bytes, __ATOMIC_SEQ_CST); @@ -1011,7 +993,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics void store(const _TVal _Value, const memory_order _Order) noexcept { // store with given memory order const auto _Mem = _STD _Atomic_address_as(_Storage); - const long long _As_bytes = _STD _Atomic_reinterpret_as(_Value); + const long long _As_bytes = _STD _Bit_cast(_Value); _Check_store_memory_order(_Order); @@ -1064,25 +1046,24 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics // exchange with given memory order long long _As_bytes; _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _As_bytes, _InterlockedExchange64, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Value)); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Value)); return reinterpret_cast<_TVal&>(_As_bytes); } #endif // ^^^ !defined(_M_IX86) ^^^ bool compare_exchange_strong(_TVal& _Expected, const _TVal _Desired, const memory_order _Order = memory_order_seq_cst) noexcept { // CAS with given memory order - long long _Expected_bytes = _STD _Atomic_reinterpret_as(_Expected); // read before atomic operation + long long _Expected_bytes = _STD _Bit_cast(_Expected); // read before atomic operation long long _Prev_bytes; #if _CMPXCHG_MASK_OUT_PADDING_BITS if constexpr (_Might_have_non_value_bits<_TVal>) { _Storage_for<_TVal> _Mask{_Form_mask}; - const long long _Mask_val = _STD _Atomic_reinterpret_as(_Mask); + const long long _Mask_val = _STD _Bit_cast(_Mask._Ref()); for (;;) { _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange64, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), - _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -1096,8 +1077,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics } #endif // _CMPXCHG_MASK_OUT_PADDING_BITS _ATOMIC_CHOOSE_INTRINSIC(static_cast(_Order), _Prev_bytes, _InterlockedCompareExchange64, - _STD _Atomic_address_as(_Storage), _STD _Atomic_reinterpret_as(_Desired), - _Expected_bytes); + _STD _Atomic_address_as(_Storage), _STD _Bit_cast(_Desired), _Expected_bytes); if (_Prev_bytes == _Expected_bytes) { return true; } @@ -1108,7 +1088,7 @@ struct _Atomic_storage<_Ty, 8> { // lock-free using 8-byte intrinsics #if _HAS_CXX20 void wait(const _TVal _Expected, const memory_order _Order = memory_order_seq_cst) const noexcept { - _STD _Atomic_wait_direct(this, _STD _Atomic_reinterpret_as(_Expected), _Order); + _STD _Atomic_wait_direct(this, _STD _Bit_cast(_Expected), _Order); } void notify_one() noexcept { From b3f2c6757541ca6d7e2b7e99903a8a1e64145d54 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Thu, 8 Jan 2026 19:30:28 +0200 Subject: [PATCH 09/19] Vectorize `replace_copy` for x64 / x86 manually (#5980) Co-authored-by: Stephan T. Lavavej --- benchmarks/src/replace.cpp | 18 --- stl/inc/algorithm | 144 +++++++++++++----- stl/inc/xutility | 1 + stl/src/vector_algorithms.cpp | 94 ++++++++++++ .../test.cpp | 10 +- .../VSO_0000000_vector_algorithms/test.cpp | 99 +++++++++--- 6 files changed, 292 insertions(+), 74 deletions(-) diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index ebb1aa038bf..28ab1c12ddb 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -34,19 +34,6 @@ void rc(benchmark::State& state) { } } -template -void rc_if(benchmark::State& state) { - std::vector> a(lorem_ipsum.begin(), lorem_ipsum.end()); - std::vector> b(lorem_ipsum.size()); - - for (auto _ : state) { - benchmark::DoNotOptimize(a); - (void) std::replace_copy_if( - std::begin(a), std::end(a), std::begin(b), [](auto x) { return x <= T{'Z'}; }, T{'X'}); - benchmark::DoNotOptimize(b); - } -} - // replace() is vectorized for 4 and 8 bytes only. BENCHMARK(r); BENCHMARK(r); @@ -56,9 +43,4 @@ BENCHMARK(rc); BENCHMARK(rc); BENCHMARK(rc); -BENCHMARK(rc_if); -BENCHMARK(rc_if); -BENCHMARK(rc_if); -BENCHMARK(rc_if); - BENCHMARK_MAIN(); diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 85838bca9cc..4920137874e 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -124,6 +124,17 @@ __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; #endif // ^^^ _VECTORIZED_REPLACE ^^^ +#if _VECTORIZED_REPLACE_COPY +__declspec(noalias) void __stdcall __std_replace_copy_1( + const void* _First, const void* _Last, void* _Dest, uint8_t _Old_val, uint8_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_2( + const void* _First, const void* _Last, void* _Dest, uint16_t _Old_val, uint16_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_4( + const void* _First, const void* _Last, void* _Dest, uint32_t _Old_val, uint32_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_8( + const void* _First, const void* _Last, void* _Dest, uint64_t _Old_val, uint64_t _New_val) noexcept; +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_SEARCH_N const void* __stdcall __std_search_n_1(const void* _First, const void* _Last, size_t _Count, uint8_t _Value) noexcept; const void* __stdcall __std_search_n_2(const void* _First, const void* _Last, size_t _Count, uint16_t _Value) noexcept; @@ -364,6 +375,28 @@ __declspec(noalias) void _Replace_vectorized( } #endif // ^^^ _VECTORIZED_REPLACE ^^^ +#if _VECTORIZED_REPLACE_COPY +template +__declspec(noalias) void _Replace_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _Ty* const _Dest, + const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { + if constexpr (sizeof(_Ty) == 1) { + ::__std_replace_copy_1( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 2) { + ::__std_replace_copy_2( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 4) { + ::__std_replace_copy_4( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 8) { + ::__std_replace_copy_8( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else { + static_assert(false, "unexpected size"); + } +} +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_SEARCH_N template _Ty* _Search_n_vectorized(_Ty* const _First, _Ty* const _Last, const size_t _Count, const _TVal _Val) noexcept { @@ -482,6 +515,17 @@ constexpr bool _Output_iterator_for_vector_alg_is_safe() { } #endif // ^^^ _VECTORIZED_REMOVE_COPY || _VECTORIZED_UNIQUE_COPY ^^^ +#if _VECTORIZED_REPLACE_COPY +template +constexpr bool _Output_iterator_for_known_size_vector_alg_is_safe() { + if constexpr (_Iterator_is_contiguous<_Out>) { + return is_same_v<_Iter_value_t<_Out>, remove_const_t<_Iter_value_t<_In>>>; + } else { + return false; + } +} +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_INCLUDES // Can we activate the vector algorithms for includes? template > @@ -4568,15 +4612,6 @@ namespace ranges { } // namespace ranges #endif // _HAS_CXX20 -// TRANSITION, DevCom-10606350: help the compiler auto-vectorize for simple types -template > -constexpr bool _Can_vectorize_replace_copy = conjunction_v, is_same<_InTy, _NewTy>, - disjunction< -#ifdef __cpp_lib_byte - conjunction, is_same<_OutTy, byte>>, -#endif // defined(__cpp_lib_byte) - conjunction, is_integral<_OutTy>>, conjunction, is_pointer<_OutTy>>>>; - _EXPORT_STD template _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const _Ty& _Oldval, const _Ty& _Newval) { // copy replacing each matching _Oldval with _Newval @@ -4585,15 +4620,36 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); - for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { - if constexpr (_Can_vectorize_replace_copy, _Ty>) { - *_UDest = *_UFirst == _Oldval ? _Newval : *_UFirst; - } else { - if (*_UFirst == _Oldval) { - *_UDest = _Newval; + +#if _VECTORIZED_REPLACE_COPY + if constexpr (_Vector_alg_in_find_is_safe + && _Output_iterator_for_known_size_vector_alg_is_safe()) { + if (!_STD _Is_constant_evaluated()) { + const auto _Count = static_cast<_Iter_diff_t>(_ULast - _UFirst); + _STD _Contiguous_iter_verify(_UDest, _Count); + + const auto _First_ptr = _STD _To_address(_UFirst); + const auto _Last_ptr = _STD _To_address(_ULast); + const auto _Dest_ptr = _STD _To_address(_UDest); + + if (_STD _Could_compare_equal_to_value_type(_Oldval)) { + _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Dest_ptr, _Oldval, _Newval); } else { - *_UDest = *_UFirst; + _CSTD memcpy(_Dest_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Dest_ptr)); } + + _UDest += _Count; + _STD _Seek_wrapped(_Dest, _UDest); + return _Dest; + } + } +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + + for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { + if (*_UFirst == _Oldval) { + *_UDest = _Newval; + } else { + *_UDest = *_UFirst; } } @@ -4666,15 +4722,37 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); - for (; _First != _Last; ++_First, (void) ++_Output) { - if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty2>) { - *_Output = _STD invoke(_Proj, *_First) == _Oldval ? _Newval : *_First; - } else { - if (_STD invoke(_Proj, *_First) == _Oldval) { - *_Output = _Newval; +#if _VECTORIZED_REPLACE_COPY + if constexpr (is_same_v<_Pj, identity> && sized_sentinel_for<_Se, _It> + && _Vector_alg_in_find_is_safe<_It, _Ty1> && _Vector_alg_in_find_is_safe<_It, _Ty2> + && _Output_iterator_for_known_size_vector_alg_is_safe<_Out, _It>()) { + if (!_STD is_constant_evaluated()) { + const auto _Count = _Last - _First; + _STD _Contiguous_iter_verify(_First, _Count); + _STD _Contiguous_iter_verify(_Output, static_cast>(_Count)); + + const auto _First_ptr = _STD to_address(_First); + const auto _Last_ptr = _First_ptr + static_cast(_Count); + const auto _Out_ptr = _STD to_address(_Output); + + if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { + _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Out_ptr, _Oldval, _Newval); } else { - *_Output = *_First; + _CSTD memcpy(_Out_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Out_ptr)); } + + _First += _Count; + _Output += static_cast>(_Count); + return {_STD move(_First), _STD move(_Output)}; + } + } +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + + for (; _First != _Last; ++_First, (void) ++_Output) { + if (_STD invoke(_Proj, *_First) == _Oldval) { + *_Output = _Newval; + } else { + *_Output = *_First; } } @@ -4695,14 +4773,10 @@ _CONSTEXPR20 _OutIt replace_copy_if(_InIt _First, _InIt _Last, _OutIt _Dest, _Pr const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { - if constexpr (_Can_vectorize_replace_copy, _Ty>) { - *_UDest = _Pred(*_UFirst) ? _Val : *_UFirst; + if (_Pred(*_UFirst)) { + *_UDest = _Val; } else { - if (_Pred(*_UFirst)) { - *_UDest = _Val; - } else { - *_UDest = *_UFirst; - } + *_UDest = *_UFirst; } } @@ -4777,14 +4851,10 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); for (; _First != _Last; ++_First, (void) ++_Output) { - if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty>) { - *_Output = _STD invoke(_Pred, _STD invoke(_Proj, *_First)) ? _Newval : *_First; + if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) { + *_Output = _Newval; } else { - if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) { - *_Output = _Newval; - } else { - *_Output = *_First; - } + *_Output = *_First; } } diff --git a/stl/inc/xutility b/stl/inc/xutility index ca847ac5015..06c1dfa8dcc 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -95,6 +95,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_REMOVE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REMOVE_COPY _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REPLACE _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_REPLACE_COPY _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REVERSE _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_REVERSE_COPY _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_ROTATE _VECTORIZED_FOR_X64_X86_ARM64 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index ae96d1d5e4b..f6e64148407 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -7034,6 +7034,80 @@ __declspec(noalias) size_t __stdcall __std_mismatch_8( return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } +} // extern "C" + +namespace { + namespace _Replacing { + template + __declspec(noalias) void __stdcall _Replace_copy_impl( + const void* _First, const void* const _Last, void* _Dest, const _Ty _Old_val, const _Ty _New_val) noexcept { +#ifndef _M_ARM64EC + const size_t _Size_bytes = _Byte_length(_First, _Last); + + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { + const __m256i _Comparand = _Traits::_Set_avx(_Old_val); + const __m256i _Replacement = _Traits::_Set_avx(_New_val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Avx_size); + + do { + const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + const __m256i _Val = _mm256_blendv_epi8(_Data, _Replacement, _Mask); + + _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Val); + + _Advance_bytes(_First, 32); + _Advance_bytes(_Dest, 32); + } while (_First != _Stop_at); + + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + const __m256i _Val = _mm256_blendv_epi8(_Data, _Replacement, _Mask); + + _mm256_maskstore_epi32(static_cast(_Dest), _Tail_mask, _Val); + + _Advance_bytes(_First, _Avx_tail_size); + _Advance_bytes(_Dest, _Avx_tail_size); + } + + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + + if constexpr (sizeof(_Ty) >= 4) { + return; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { + const __m128i _Comparand = _Traits::_Set_sse(_Old_val); + const __m128i _Replacement = _Traits::_Set_sse(_New_val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Sse_size); + + do { + const __m128i _Data = _mm_loadu_si128(static_cast(_First)); + const __m128i _Mask = _Traits::_Cmp_sse(_Data, _Comparand); + const __m128i _Val = _mm_blendv_epi8(_Data, _Replacement, _Mask); + + _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Val); + + _Advance_bytes(_First, 16); + _Advance_bytes(_Dest, 16); + } while (_First != _Stop_at); + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + auto _Ptr_dest = static_cast<_Ty*>(_Dest); + for (auto _Ptr_src = static_cast(_First); _Ptr_src != _Last; ++_Ptr_src) { + const _Ty _Val = *_Ptr_src; + *_Ptr_dest = _Val == _Old_val ? _New_val : _Val; + ++_Ptr_dest; + } + } + } // namespace _Replacing +} // unnamed namespace + +extern "C" { + __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { #ifndef _M_ARM64EC @@ -7115,6 +7189,26 @@ __declspec(noalias) void __stdcall __std_replace_8( } } +__declspec(noalias) void __stdcall __std_replace_copy_1(const void* const _First, const void* const _Last, + void* const _Dest, const uint8_t _Old_val, const uint8_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_1>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_2(const void* const _First, const void* const _Last, + void* const _Dest, const uint16_t _Old_val, const uint16_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_2>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_4(const void* const _First, const void* const _Last, + void* const _Dest, const uint32_t _Old_val, const uint32_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_4>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_8(const void* const _First, const void* const _Last, + void* const _Dest, const uint64_t _Old_val, const uint64_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_8>(_First, _Last, _Dest, _Old_val, _New_val); +} + } // extern "C" namespace { diff --git a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp index c2b53e59f58..0a82d9bad27 100644 --- a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp +++ b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp @@ -221,11 +221,17 @@ int main() { assert(r_rot_it == temp_end - rotate_pos); } { - // Out of replace family, only replace for 32-bit and 64-bit elements is manually vectorized, - // replace_copy is auto vectorized (along with replace_copy_if) const int replace_expected[] = { 200, 210, 220, 333, 240, 333, 333, 270, 280, 290, 300, 310, 320, 333, 340, 333, 333, 370, 380, 390}; + auto repl_copy_it = replace_copy(arr_begin, arr_end, temp_begin, 250, 333); + assert(equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); + assert(repl_copy_it == temp_end); + + auto r_repl_copy_it = ranges::replace_copy(arr_begin, arr_end, temp_begin, 250, 333).out; + assert(ranges::equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); + assert(r_repl_copy_it == temp_end); + copy(arr_begin, arr_end, temp_begin); replace(temp_begin, temp_end, 250, 333); assert(equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index eeff29a4bfa..3a77c979cf9 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -723,40 +723,101 @@ void last_known_good_replace(FwdIt first, FwdIt last, const T old_val, const T n } } +template +void last_known_good_replace_copy(FwdIt first, FwdIt last, OutIt dest, const T old_val, const T new_val) { + for (; first != last; ++first, ++dest) { + if (*first == old_val) { + *dest = new_val; + } else { + *dest = *first; + } + } +} + template -void test_case_replace(const vector& input, T old_val, T new_val) { - vector replaced_actual(input); - vector replaced_expected(input); - replace(replaced_actual.begin(), replaced_actual.end(), old_val, new_val); - last_known_good_replace(replaced_expected.begin(), replaced_expected.end(), old_val, new_val); - assert(replaced_expected == replaced_actual); +void test_case_replace(vector& in_out_expected, vector& in_out_actual, vector& in_out_actual_r, + const T old_val, const T new_val) { + replace(in_out_actual.begin(), in_out_actual.end(), old_val, new_val); + last_known_good_replace(in_out_expected.begin(), in_out_expected.end(), old_val, new_val); + assert(in_out_expected == in_out_actual); #if _HAS_CXX20 - vector replaced_actual_r(input); - ranges::replace(replaced_actual_r, old_val, new_val); - assert(replaced_expected == replaced_actual_r); -#endif // _HAS_CXX20 + ranges::replace(in_out_actual_r, old_val, new_val); + assert(in_out_expected == in_out_actual_r); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) in_out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ +} + +template +void test_case_replace_copy(const vector& input, vector& out_expected, vector& out_actual, + vector& out_actual_r, const T old_val, const T new_val) { + + replace_copy(input.begin(), input.end(), out_actual.begin(), old_val, new_val); + last_known_good_replace_copy(input.begin(), input.end(), out_expected.begin(), old_val, new_val); + assert(out_expected == out_actual); + +#if _HAS_CXX20 + ranges::replace_copy(input, out_actual_r.begin(), old_val, new_val); + assert(out_expected == out_actual_r); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ } template void test_replace(mt19937_64& gen) { + // replace() is vectorized for 4 and 8 bytes only. + constexpr bool replace_is_vectorized = sizeof(T) >= 4; + using TD = conditional_t; uniform_int_distribution dis(0, 9); - vector input; - input.reserve(dataCount); + vector source; + vector out_expected; + vector out_actual; + vector out_actual_r; + vector in_out_expected; + vector in_out_actual; + vector in_out_actual_r; + + for (const auto& v : {&source, &out_expected, &out_actual, &out_actual_r}) { + v->reserve(dataCount); + } + + if constexpr (replace_is_vectorized) { + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + v->reserve(dataCount); + } + } { const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - test_case_replace(input, old_val, new_val); + + if constexpr (replace_is_vectorized) { + test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); + } + test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); } - for (size_t i = 0; i != dataCount; ++i) { - input.push_back(static_cast(dis(gen))); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + source.push_back(static_cast(dis(gen))); const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - test_case_replace(input, old_val, new_val); + + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + *v = source; + } + + for (const auto& v : {&out_expected, &out_actual, &out_actual_r}) { + v->assign(source.size(), T{0}); + } + + if constexpr (replace_is_vectorized) { + test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); + } + test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); } } @@ -1287,7 +1348,11 @@ void test_vector_algorithms(mt19937_64& gen) { test_includes(gen); #endif // _HAS_CXX17 - // replace() is vectorized for 4 and 8 bytes only. + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); test_replace(gen); test_replace(gen); test_replace(gen); From ba946a1c142f708e0acf89759a44a993bc743025 Mon Sep 17 00:00:00 2001 From: Que Date: Thu, 8 Jan 2026 23:02:36 +0530 Subject: [PATCH 10/19] updated _MSVC_STL_UPDATE to January 2026 (#5985) --- stl/inc/yvals_core.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index c28df8db1c6..fcb8716af04 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -923,7 +923,7 @@ #define _CPPLIB_VER 650 #define _MSVC_STL_VERSION 145 -#define _MSVC_STL_UPDATE 202512L +#define _MSVC_STL_UPDATE 202601L #ifndef _ALLOW_COMPILER_AND_STL_VERSION_MISMATCH #if defined(__CUDACC__) && defined(__CUDACC_VER_MAJOR__) From 4634ad466dfdb706ba65e197ec8257d6c7bce25a Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 8 Jan 2026 17:34:46 +0000 Subject: [PATCH 11/19] Fix no-match handling when calling `memchr`/`wmemchr` on ARM64EC (#5993) --- stl/src/vector_algorithms.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index f6e64148407..0f84855383d 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -4278,7 +4278,8 @@ const void* __stdcall __std_find_trivial_unsized_8(const void* const _First, con const void* __stdcall __std_find_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { #ifdef _M_ARM64EC - return memchr(_First, _Val, _Byte_length(_First, _Last)); + auto _Result = memchr(_First, _Val, _Byte_length(_First, _Last)); + return _Result ? _Result : _Last; #else return _Finding::_Find_impl<_Finding::_Find_traits_1, _Finding::_Predicate::_Equal>(_First, _Last, _Val); #endif @@ -4287,7 +4288,8 @@ const void* __stdcall __std_find_trivial_1( const void* __stdcall __std_find_trivial_2( const void* const _First, const void* const _Last, const uint16_t _Val) noexcept { #ifdef _M_ARM64EC - return wmemchr(static_cast(_First), _Val, _Byte_length(_First, _Last) / sizeof(wchar_t)); + auto _Result = wmemchr(static_cast(_First), _Val, _Byte_length(_First, _Last) / sizeof(wchar_t)); + return _Result ? _Result : _Last; #else return _Finding::_Find_impl<_Finding::_Find_traits_2, _Finding::_Predicate::_Equal>(_First, _Last, _Val); #endif From 32db7bdc6124f56814f402165dfe00fc5225a70c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 15 Jan 2026 10:26:36 -0800 Subject: [PATCH 12/19] Toolset update: MSVC Compiler 19.50.35722 (#6011) --- CMakeLists.txt | 6 ++-- README.md | 2 +- azure-devops/build-benchmarks.yml | 38 ++++++++++++-------- azure-devops/checkout-self.yml | 22 +++++++++--- azure-devops/cmake-configure-build.yml | 46 +++++++++++++++---------- azure-devops/config.yml | 8 ++--- azure-devops/create-1es-hosted-pool.ps1 | 2 +- azure-devops/format-validation.yml | 42 +++++++++++++++------- azure-devops/provision-image.ps1 | 29 ++++++++++++++-- azure-devops/run-tests.yml | 13 ++++--- benchmarks/CMakeLists.txt | 2 +- tests/CMakeLists.txt | 2 +- tests/libcxx/expected_results.txt | 2 -- tools/CMakeLists.txt | 2 +- tools/format/CMakeLists.txt | 2 +- tools/scripts/check_libcxx_paths.py | 6 ++-- tools/scripts/check_test_lst_paths.py | 14 ++++---- tools/validate/CMakeLists.txt | 2 +- 18 files changed, 158 insertions(+), 82 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c9ba52f1857..3aedb039bb8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,13 +1,13 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 4.1.1) +cmake_minimum_required(VERSION 4.1.2) set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY) project(msvc_standard_libraries LANGUAGES CXX) -if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.50.35720") - message(FATAL_ERROR "The STL must be built with MSVC Compiler 19.50.35720 or later.") +if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "19.50.35722") + message(FATAL_ERROR "The STL must be built with MSVC Compiler 19.50.35722 or later.") endif() include(CheckCXXSourceCompiles) diff --git a/README.md b/README.md index 3524ea9266f..c3f8676a4c1 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ Just try to follow these rules, so we can spend more time fixing bugs and implem - "Windows 11 SDK (10.0.26100)" or later - "C++ Clang tools for Windows (20.1.8 - x64/x86)" - *Optional, see Note 2 below:* "MSVC Build Tools for ARM64/ARM64EC (Latest)" -* Install [Python][] 3.14.0 or later. +* Install [Python][] 3.14.2 or later. + Select "Add python.exe to PATH" if you want to follow the instructions below that invoke `python`. Otherwise, you should be familiar with alternative methods. diff --git a/azure-devops/build-benchmarks.yml b/azure-devops/build-benchmarks.yml index 031e3b2adff..59d9db7d15a 100644 --- a/azure-devops/build-benchmarks.yml +++ b/azure-devops/build-benchmarks.yml @@ -14,18 +14,23 @@ parameters: type: string steps: -- script: | - if exist "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" ( - rmdir /S /Q "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" - ) - call "$(vsDevCmdBat)" -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo - cmake -G Ninja ^ - -DCMAKE_CXX_COMPILER=${{ parameters.compiler }} ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DSTL_BINARY_DIR="$(buildOutputLocation)" ^ - -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ^ - -S $(Build.SourcesDirectory)/benchmarks -B "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" +- task: PowerShell@2 displayName: 'Configure Benchmarks for ${{ parameters.compiler }}' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + if (Test-Path -LiteralPath "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}") { + Remove-Item -LiteralPath "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" -Recurse -Force + } + & "$(launchVsDevShell)" -HostArch ${{ parameters.hostArch }} -Arch ${{ parameters.targetArch }} + cmake -G Ninja ` + -DCMAKE_CXX_COMPILER=${{ parameters.compiler }} ` + -DCMAKE_BUILD_TYPE=Release ` + -DSTL_BINARY_DIR="$(buildOutputLocation)" ` + -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ` + -S $(Build.SourcesDirectory)/benchmarks -B "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" timeoutInMinutes: 2 env: { TMP: $(tmpDir), TEMP: $(tmpDir) } # TRANSITION, we currently only build the benchmarks with Clang for x64 @@ -33,10 +38,15 @@ steps: and(succeeded(), ${{ parameters.buildBenchmarks }}, not(and(eq('${{ parameters.compiler }}', 'clang-cl'), not(eq('${{ parameters.targetPlatform }}', 'x64'))))) -- script: | - call "$(vsDevCmdBat)" -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo - cmake --build "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" +- task: PowerShell@2 displayName: 'Build Benchmarks for ${{ parameters.compiler }}' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + & "$(launchVsDevShell)" -HostArch ${{ parameters.hostArch }} -Arch ${{ parameters.targetArch }} + cmake --build "$(benchmarkBuildOutputLocation)\${{ parameters.compiler }}" timeoutInMinutes: 2 env: { TMP: $(tmpDir), TEMP: $(tmpDir) } # TRANSITION, we currently only build the benchmarks with Clang for x64 diff --git a/azure-devops/checkout-self.yml b/azure-devops/checkout-self.yml index 711c2b00321..9939678389f 100644 --- a/azure-devops/checkout-self.yml +++ b/azure-devops/checkout-self.yml @@ -2,16 +2,28 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception steps: -- script: | - if exist "$(tmpDir)" (rmdir /S /Q $(tmpDir)) - mkdir $(tmpDir) +- task: PowerShell@2 displayName: 'Setup TMP Directory' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + if (Test-Path -LiteralPath "$(tmpDir)") { + Remove-Item -LiteralPath "$(tmpDir)" -Recurse -Force + } + mkdir "$(tmpDir)" -Force | Out-Null - checkout: self clean: true submodules: false fetchDepth: 1 fetchTags: false retryCountOnTaskFailure: 4 -- script: | - git clean --quiet -x -d -f -f +- task: PowerShell@2 displayName: 'Clean After Checkout' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + git clean --quiet -x -d -f -f diff --git a/azure-devops/cmake-configure-build.yml b/azure-devops/cmake-configure-build.yml index 8fa87a301c8..40aab7af428 100644 --- a/azure-devops/cmake-configure-build.yml +++ b/azure-devops/cmake-configure-build.yml @@ -27,28 +27,38 @@ parameters: - '--run-shard=$(System.JobPositionInPhase)' steps: -- script: | - if exist "$(buildOutputLocation)" ( - rmdir /S /Q "$(buildOutputLocation)" - ) - call "$(vsDevCmdBat)" -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo - cmake -G Ninja ^ - -DCMAKE_CXX_COMPILER=cl ^ - -DCMAKE_BUILD_TYPE=Release ^ - -DLIT_FLAGS=${{ join(';', parameters.litFlags) }} ^ - -DSTL_USE_ANALYZE=${{ parameters.analyzeBuild }} ^ - -DSTL_ASAN_BUILD=${{ parameters.asanBuild }} ^ - -DCONFIGURE_TESTING=${{ parameters.configureTesting }} ^ - -DTESTS_BUILD_ONLY=${{ parameters.testsBuildOnly }} ^ - -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ^ - -S $(Build.SourcesDirectory) -B "$(buildOutputLocation)" +- task: PowerShell@2 displayName: 'Configure STL' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + if (Test-Path -LiteralPath "$(buildOutputLocation)") { + Remove-Item -LiteralPath "$(buildOutputLocation)" -Recurse -Force + } + & "$(launchVsDevShell)" -HostArch ${{ parameters.hostArch }} -Arch ${{ parameters.targetArch }} + cmake -G Ninja ` + -DCMAKE_CXX_COMPILER=cl ` + -DCMAKE_BUILD_TYPE=Release ` + -DLIT_FLAGS="${{ join(';', parameters.litFlags) }}" ` + -DSTL_USE_ANALYZE=${{ parameters.analyzeBuild }} ` + -DSTL_ASAN_BUILD=${{ parameters.asanBuild }} ` + -DCONFIGURE_TESTING=${{ parameters.configureTesting }} ` + -DTESTS_BUILD_ONLY=${{ parameters.testsBuildOnly }} ` + -DVCLIBS_TARGET_ARCHITECTURE=${{ parameters.targetPlatform }} ` + -S $(Build.SourcesDirectory) -B "$(buildOutputLocation)" timeoutInMinutes: 2 env: { TMP: $(tmpDir), TEMP: $(tmpDir) } -- script: | - call "$(vsDevCmdBat)" -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo - cmake --build "$(buildOutputLocation)" +- task: PowerShell@2 displayName: 'Build STL' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + & "$(launchVsDevShell)" -HostArch ${{ parameters.hostArch }} -Arch ${{ parameters.targetArch }} + cmake --build "$(buildOutputLocation)" timeoutInMinutes: 5 condition: and(succeeded(), ${{ parameters.buildStl }}) env: { TMP: $(tmpDir), TEMP: $(tmpDir) } diff --git a/azure-devops/config.yml b/azure-devops/config.yml index 91a4aaae7fc..0bf5093a947 100644 --- a/azure-devops/config.yml +++ b/azure-devops/config.yml @@ -5,16 +5,16 @@ variables: - name: poolName - value: 'Stl-2025-12-09T1433-x64-Pool' + value: 'Stl-2026-01-13T1248-x64-Pool' readonly: true - name: arm64PoolName - value: 'Stl-2025-12-09T1433-arm64-Pool' + value: 'Stl-2026-01-13T1248-arm64-Pool' readonly: true - name: poolDemands value: 'EnableSpotVM -equals false' readonly: true -- name: vsDevCmdBat - value: 'C:\Program Files\Microsoft Visual Studio\18\Insiders\Common7\Tools\VsDevCmd.bat' +- name: launchVsDevShell + value: 'C:\Program Files\Microsoft Visual Studio\18\Insiders\Common7\Tools\Launch-VsDevShell.ps1' readonly: true - name: tmpDir value: 'C:\stlTemp' diff --git a/azure-devops/create-1es-hosted-pool.ps1 b/azure-devops/create-1es-hosted-pool.ps1 index 0c475763b61..8bb16ee6a0e 100644 --- a/azure-devops/create-1es-hosted-pool.ps1 +++ b/azure-devops/create-1es-hosted-pool.ps1 @@ -31,7 +31,7 @@ if ($Arch -ieq 'x64') { } else { # CPP_STL_GitHub has quota for 672 cores (21 VMs) in westcentralus, not currently used. $AvailableLocations = @('eastus2', 'northeurope') # Locations where CPP_STL_GitHub has quota for 1024 cores (32 VMs). - $AvailableLocationIdx = 3 # Increment for each new pool, to cycle through the available locations. + $AvailableLocationIdx = 4 # Increment for each new pool, to cycle through the available locations. $Location = $AvailableLocations[$AvailableLocationIdx % $AvailableLocations.Length] $VMSize = 'Standard_D32ps_v6' $PoolSize = 32 diff --git a/azure-devops/format-validation.yml b/azure-devops/format-validation.yml index 77f3635e5b2..e5c695c4551 100644 --- a/azure-devops/format-validation.yml +++ b/azure-devops/format-validation.yml @@ -9,26 +9,41 @@ jobs: displayName: 'Validation' steps: - template: checkout-self.yml - - script: | - if exist "$(validationBuildOutputLocation)" ( - rmdir /S /Q "$(validationBuildOutputLocation)" - ) - call "$(vsDevCmdBat)" -host_arch=x64 -arch=x64 -no_logo - cmake -G Ninja -S $(Build.SourcesDirectory)/tools -B "$(validationBuildOutputLocation)" - cmake --build "$(validationBuildOutputLocation)" + - task: PowerShell@2 displayName: 'Build Validation Tools' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + if (Test-Path -LiteralPath "$(validationBuildOutputLocation)") { + Remove-Item -LiteralPath "$(validationBuildOutputLocation)" -Recurse -Force + } + & "$(launchVsDevShell)" -HostArch x64 -Arch x64 + cmake -G Ninja -S $(Build.SourcesDirectory)/tools -B "$(validationBuildOutputLocation)" + cmake --build "$(validationBuildOutputLocation)" timeoutInMinutes: 5 env: { TMP: $(tmpDir), TEMP: $(tmpDir) } - - script: | - call "$(vsDevCmdBat)" -host_arch=x64 -arch=x64 -no_logo - cmake --build "$(validationBuildOutputLocation)" --target run-format + - task: PowerShell@2 displayName: 'clang-format Files' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + & "$(launchVsDevShell)" -HostArch x64 -Arch x64 + cmake --build "$(validationBuildOutputLocation)" --target run-format timeoutInMinutes: 5 env: { TMP: $(tmpDir), TEMP: $(tmpDir) } - - script: | - call "$(vsDevCmdBat)" -host_arch=x64 -arch=x64 -no_logo - cmake --build "$(validationBuildOutputLocation)" --target run-validate + - task: PowerShell@2 displayName: 'Validate Files' + inputs: + pwsh: true + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + & "$(launchVsDevShell)" -HostArch x64 -Arch x64 + cmake --build "$(validationBuildOutputLocation)" --target run-validate timeoutInMinutes: 2 env: { TMP: $(tmpDir), TEMP: $(tmpDir) } - task: PowerShell@2 @@ -37,6 +52,7 @@ jobs: pwsh: true targetType: inline script: | + $PSNativeCommandUseErrorActionPreference = $true $TempSubDir = Join-Path ([System.IO.Path]::GetTempPath()) ([System.IO.Path]::GetRandomFileName()) mkdir $TempSubDir -Force | Out-Null $DiffFile = Join-Path $TempSubDir 'format.diff' diff --git a/azure-devops/provision-image.ps1 b/azure-devops/provision-image.ps1 index f4cabac438d..f49b26635c4 100644 --- a/azure-devops/provision-image.ps1 +++ b/azure-devops/provision-image.ps1 @@ -67,9 +67,9 @@ $PowerShellArgs = @('/quiet', '/norestart') # https://www.python.org if ($Provisioning_x64) { - $PythonUrl = 'https://www.python.org/ftp/python/3.14.0/python-3.14.0-amd64.exe' + $PythonUrl = 'https://www.python.org/ftp/python/3.14.2/python-3.14.2-amd64.exe' } else { - $PythonUrl = 'https://www.python.org/ftp/python/3.14.0/python-3.14.0-arm64.exe' + $PythonUrl = 'https://www.python.org/ftp/python/3.14.2/python-3.14.2-arm64.exe' } $PythonArgs = @('/quiet', 'InstallAllUsers=1', 'PrependPath=1', 'CompileAll=1', 'Include_doc=0') @@ -137,6 +137,26 @@ Function DownloadAndInstall { } } +<# +.SYNOPSIS +Enables native NVMe support. + +.DESCRIPTION +Native NVMe support is opt-in for Windows Server 2025. +TRANSITION, this will be enabled by default for the next version of Windows Server. +#> +Function EnableNativeNVMe { + $registryKey = 'HKLM:\SYSTEM\CurrentControlSet\Policies\Microsoft\FeatureManagement\Overrides' + $valueName = '1176759950' + $valueData = 1 + + if (!(Test-Path $registryKey)) { + New-Item -Path $registryKey -Force | Out-Null + } + + New-ItemProperty -Path $registryKey -Name $valueName -Value $valueData -PropertyType DWORD -Force | Out-Null +} + Write-Host "Old PowerShell version: $($PSVersionTable.PSVersion)" # Print the Windows version, so we can verify whether Patch Tuesday has been picked up. @@ -161,6 +181,11 @@ Write-Host 'Enabling long paths...' New-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\FileSystem' -Name 'LongPathsEnabled' ` -Value 1 -PropertyType DWORD -Force | Out-Null +if ($Provisioning_x64) { + Write-Host 'Enabling native NVMe...' + EnableNativeNVMe +} + # Tell create-1es-hosted-pool.ps1 that we succeeded. Write-Host 'PROVISION_IMAGE_SUCCEEDED' diff --git a/azure-devops/run-tests.yml b/azure-devops/run-tests.yml index 847adccb36d..55a568ba9e5 100644 --- a/azure-devops/run-tests.yml +++ b/azure-devops/run-tests.yml @@ -13,13 +13,18 @@ parameters: - name: runTesting type: boolean steps: -- script: | - call "$(vsDevCmdBat)" -host_arch=${{ parameters.hostArch }} -arch=${{ parameters.targetArch }} -no_logo - ninja --verbose -k 0 ${{ parameters.testTargets }} +- task: PowerShell@2 displayName: 'Run Tests' + inputs: + pwsh: true + workingDirectory: $(buildOutputLocation) + targetType: inline + script: | + $PSNativeCommandUseErrorActionPreference = $true + & "$(launchVsDevShell)" -HostArch ${{ parameters.hostArch }} -Arch ${{ parameters.targetArch }} + ninja --verbose -k 0 ${{ parameters.testTargets }} timeoutInMinutes: 30 condition: and(succeeded(), ${{ parameters.runTesting }}) - workingDirectory: $(buildOutputLocation) env: { TMP: $(tmpDir), TEMP: $(tmpDir) } - task: PublishTestResults@2 displayName: 'Publish Tests' diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index ce912d68979..ff6c3ad1651 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 4.1.1) +cmake_minimum_required(VERSION 4.1.2) project(msvc_standard_libraries_benchmarks LANGUAGES CXX) if(DEFINED STL_BINARY_DIR) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1b3bd94e558..9d49ceb410d 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -find_package(Python "3.14.0" REQUIRED COMPONENTS Interpreter) +find_package(Python "3.14.2" REQUIRED COMPONENTS Interpreter) set(STL_BUILD_ROOT "${PROJECT_BINARY_DIR}/out") set(STL_SOURCE_DIR "${PROJECT_SOURCE_DIR}") diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 2eb83e0cf0a..2414b59289f 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -1504,8 +1504,6 @@ std/algorithms/alg.modifying.operations/alg.copy/ranges.copy.pass.cpp FAIL # Not analyzed, likely MSVC compiler bugs. std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp:0 FAIL std/utilities/function.objects/refwrap/common_reference.compile.pass.cpp:1 FAIL -std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp:0 FAIL -std/utilities/meta/meta.trans/meta.trans.other/common_reference.compile.pass.cpp:1 FAIL # Not analyzed. MSVC emits warning C4267: 'argument': conversion from 'size_t' to 'unsigned char', possible loss of data std/algorithms/alg.modifying.operations/alg.copy/copy_backward.pass.cpp:0 FAIL diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 6cf8f169ea2..6ca421fb4ea 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 4.1.1) +cmake_minimum_required(VERSION 4.1.2) project(msvc_standard_libraries_tools LANGUAGES CXX) add_subdirectory(format) diff --git a/tools/format/CMakeLists.txt b/tools/format/CMakeLists.txt index 62c5cf0acee..8aedb0fa7c1 100644 --- a/tools/format/CMakeLists.txt +++ b/tools/format/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 4.1.1) +cmake_minimum_required(VERSION 4.1.2) project(msvc_standard_libraries_format NONE) find_program(CLANG_FORMAT diff --git a/tools/scripts/check_libcxx_paths.py b/tools/scripts/check_libcxx_paths.py index 7113bf16d67..82c9368d979 100644 --- a/tools/scripts/check_libcxx_paths.py +++ b/tools/scripts/check_libcxx_paths.py @@ -26,11 +26,11 @@ # Build up a list of nonexistent tests so they can be printed in sorted order. absolute_libcxx_test = absolute_repo_path / "llvm-project/libcxx/test" - nonexistent_tests = [str for str in unique_tests if not (absolute_libcxx_test / str).is_file()] + nonexistent_tests = [s for s in unique_tests if not (absolute_libcxx_test / s).is_file()] if nonexistent_tests: print(f"Failure: {expected_results_txt} contains {len(nonexistent_tests)} nonexistent tests:", file=sys.stderr) - for str in sorted(nonexistent_tests): - print(f"{str}", file=sys.stderr) + for s in sorted(nonexistent_tests): + print(f"{s}", file=sys.stderr) print(f"##vso[task.logissue type=error]{expected_results_txt} failed validation, see log.", file=sys.stderr) sys.exit(1) diff --git a/tools/scripts/check_test_lst_paths.py b/tools/scripts/check_test_lst_paths.py index e308139258b..f10c2e7e1e5 100644 --- a/tools/scripts/check_test_lst_paths.py +++ b/tools/scripts/check_test_lst_paths.py @@ -40,27 +40,27 @@ def is_nonempty_dir(p: Path) -> bool: missing_tests = existing_tests - commented_tests - unique_tests - nonexistent_tests = [str for str in unique_tests if not is_nonempty_dir(absolute_base / str)] + nonexistent_tests = [s for s in unique_tests if not is_nonempty_dir(absolute_base / s)] failed = False if duplicate_tests: failed = True print(f"Failure: {relative_test_lst} contains {len(duplicate_tests)} duplicate tests:", file=sys.stderr) - for str in sorted(duplicate_tests): - print(f"{str}", file=sys.stderr) + for s in sorted(duplicate_tests): + print(f"{s}", file=sys.stderr) if missing_tests: failed = True print(f"Failure: {relative_test_lst} is missing {len(missing_tests)} tests:", file=sys.stderr) - for str in sorted(missing_tests): - print(f"{str}", file=sys.stderr) + for s in sorted(missing_tests): + print(f"{s}", file=sys.stderr) if nonexistent_tests: failed = True print(f"Failure: {relative_test_lst} contains {len(nonexistent_tests)} nonexistent tests:", file=sys.stderr) - for str in sorted(nonexistent_tests): - print(f"{str}", file=sys.stderr) + for s in sorted(nonexistent_tests): + print(f"{s}", file=sys.stderr) for i in range(len(filtered_lines) - 1): # If filtered_lines is empty, range(-1) is empty. if filtered_lines[i] > filtered_lines[i + 1]: # Use greater-than to avoid reporting adjacent duplicates. diff --git a/tools/validate/CMakeLists.txt b/tools/validate/CMakeLists.txt index 13adfb38536..b254752a1f7 100644 --- a/tools/validate/CMakeLists.txt +++ b/tools/validate/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -cmake_minimum_required(VERSION 4.1.1) +cmake_minimum_required(VERSION 4.1.2) project(msvc_standard_libraries_validate LANGUAGES CXX) add_executable(validate-binary validate.cpp) From f10833deb38705deb2c78150657f85fd9dc22d65 Mon Sep 17 00:00:00 2001 From: Gregg Miskelly Date: Thu, 15 Jan 2026 10:28:23 -0800 Subject: [PATCH 13/19] STL.natvis: Add `c_str` intrinsic function (#5997) --- stl/debugger/STL.natvis | 1 + 1 file changed, 1 insertion(+) diff --git a/stl/debugger/STL.natvis b/stl/debugger/STL.natvis index ec8b3cb4180..75260f08a60 100644 --- a/stl/debugger/STL.natvis +++ b/stl/debugger/STL.natvis @@ -1011,6 +1011,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + {_Mypair._Myval2._Bx._Buf,na} {_Mypair._Myval2._Bx._Ptr,na} _Mypair._Myval2._Bx._Buf,na From 44821d98e421e6f48043b1aeccb82f4834023aef Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 15 Jan 2026 18:35:35 +0000 Subject: [PATCH 14/19] Add Neon implementation of minmax (#5963) Co-authored-by: Stephan T. Lavavej --- stl/inc/xutility | 4 +- stl/src/vector_algorithms.cpp | 345 +++++++++++++++++++++++++++------- 2 files changed, 277 insertions(+), 72 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index 06c1dfa8dcc..577161e4571 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -89,7 +89,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_FIND_LAST_OF _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_INCLUDES _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_IS_SORTED_UNTIL _VECTORIZED_FOR_X64_X86 -#define _VECTORIZED_MINMAX _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_MINMAX _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_MINMAX_ELEMENT _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_MISMATCH _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REMOVE _VECTORIZED_FOR_X64_X86 @@ -7302,7 +7302,7 @@ constexpr bool _Is_min_max_value_optimization_safe = // Activate the vector algo #ifndef _M_FP_FAST !is_floating_point_v<_Elem> && #endif // ^^^ !defined(_M_FP_FAST) ^^^ - _Is_min_max_optimization_safe<_Iter, _Pr>; + _Is_min_max_iterators_safe<_Iter> && _Is_predicate_less<_Iter, _Pr>; template constexpr _FwdIt _Max_element_unchecked(_FwdIt _First, _FwdIt _Last, _Pr _Pred) { // find largest element diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0f84855383d..0271995eed8 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -934,6 +934,7 @@ namespace { static constexpr bool _Vectorized = false; static constexpr size_t _Tail_mask = 0; static constexpr bool _Has_unsigned_cmp = false; + using _Vec_t = void; }; #ifdef _M_ARM64 @@ -945,22 +946,6 @@ namespace { static constexpr size_t _Tail_mask = 0; static constexpr bool _Has_unsigned_cmp = true; - static int8x16_t _Zero() noexcept { - return vdupq_n_s8(0); - } - - static int8x16_t _All_ones() noexcept { - return vdupq_n_s8(static_cast(0xFF)); - } - - static int8x16_t _Blend(const int8x16_t _Px1, const int8x16_t _Px2, const int8x16_t _Msk) noexcept { - return vbslq_s8(vreinterpretq_u8_s8(_Msk), _Px2, _Px1); - } - - static int8x16_t _Sign_correction(const int8x16_t _Val, bool) noexcept { - return _Val; - } - static void _Exit_vectorized() noexcept {} }; #elif !defined(_M_ARM64EC) @@ -971,6 +956,7 @@ namespace { static constexpr size_t _Vec_mask = 0xF; static constexpr size_t _Tail_mask = 0; static constexpr bool _Has_unsigned_cmp = false; + using _Vec_t = __m128i; static __m128i _Zero() noexcept { return _mm_setzero_si128(); @@ -1043,6 +1029,7 @@ namespace { struct _Traits_avx_i_base : _Traits_avx_base { static constexpr size_t _Tail_mask = 0x1C; + using _Vec_t = __m256i; static __m256i _Blendval(const __m256i _Px1, const __m256i _Px2, const __m256i _Msk) noexcept { return _mm256_blendv_epi8(_Px1, _Px2, _Msk); @@ -1076,6 +1063,22 @@ namespace { struct _Traits_1_neon : _Traits_1_base, _Traits_neon_base { using _Vec_t = int8x16_t; + static _Vec_t _Sign_correction(const _Vec_t _Val, bool) noexcept { + return _Val; + } + + static _Vec_t _Zero() noexcept { + return vdupq_n_s8(0); + } + + static _Vec_t _All_ones() noexcept { + return vdupq_n_s8(static_cast(0xFF)); + } + + static _Vec_t _Blend(const _Vec_t _Px1, const _Vec_t _Px2, const _Vec_t _Msk) noexcept { + return vbslq_s8(vreinterpretq_u8_s8(_Msk), _Px2, _Px1); + } + // Compresses a 128-bit Mask of 16 8-bit values into a 64-bit Mask of 16 4-bit values. static uint64_t _Mask(const _Vec_t _Val) noexcept { const uint8x8_t _Res = vshrn_n_u16(vreinterpretq_u16_s8(_Val), 4); @@ -1371,6 +1374,22 @@ namespace { struct _Traits_2_neon : _Traits_2_base, _Traits_neon_base { using _Vec_t = int16x8_t; + static _Vec_t _Sign_correction(const _Vec_t _Val, bool) noexcept { + return _Val; + } + + static _Vec_t _Zero() noexcept { + return vdupq_n_s16(0); + } + + static _Vec_t _All_ones() noexcept { + return vreinterpretq_s16_s8(vdupq_n_s8(static_cast(0xFF))); + } + + static _Vec_t _Blend(const _Vec_t _Px1, const _Vec_t _Px2, const _Vec_t _Msk) noexcept { + return vbslq_s16(vreinterpretq_u16_s16(_Msk), _Px2, _Px1); + } + // Compresses a 128-bit Mask of 8 16-bit values into a 64-bit Mask of 8 8-bit values. static uint64_t _Mask(const _Vec_t _Val) noexcept { const uint16x4_t _Res = vshrn_n_u32(vreinterpretq_u32_s16(_Val), 8); @@ -1663,6 +1682,22 @@ namespace { struct _Traits_4_neon : _Traits_4_base, _Traits_neon_base { using _Vec_t = int32x4_t; + static _Vec_t _Sign_correction(const _Vec_t _Val, bool) noexcept { + return _Val; + } + + static _Vec_t _Zero() noexcept { + return vdupq_n_s32(0); + } + + static _Vec_t _All_ones() noexcept { + return vreinterpretq_s32_s8(vdupq_n_s8(static_cast(0xFF))); + } + + static _Vec_t _Blend(const _Vec_t _Px1, const _Vec_t _Px2, const _Vec_t _Msk) noexcept { + return vbslq_s32(vreinterpretq_u32_s32(_Msk), _Px2, _Px1); + } + // Compresses a 128-bit Mask of 4 32-bit values into a 64-bit Mask of 4 16-bit values. static uint64_t _Mask(const int32x4_t _Val) noexcept { const uint32x2_t _Res = vshrn_n_u64(vreinterpretq_u64_s32(_Val), 16); @@ -1940,7 +1975,81 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#if !defined(_M_ARM64) && !defined(_M_ARM64EC) +#ifdef _M_ARM64 + struct _Traits_8_neon : _Traits_8_base, _Traits_neon_base { + using _Vec_t = int64x2_t; + + static _Vec_t _Sign_correction(const _Vec_t _Val, bool) noexcept { + return _Val; + } + + static _Vec_t _Load(const void* const _Src) noexcept { + return vld1q_s64(reinterpret_cast(_Src)); + } + + static _Vec_t _H_min(const _Vec_t _Cur) noexcept { + int64x2_t _Swapped = vextq_s64(_Cur, _Cur, 1); + uint64x2_t _Mask_lt = vcltq_s64(_Swapped, _Cur); + return vbslq_s64(_Mask_lt, _Swapped, _Cur); + } + + static _Vec_t _H_max(const _Vec_t _Cur) noexcept { + int64x2_t _Swapped = vextq_s64(_Cur, _Cur, 1); + uint64x2_t _Mask_gt = vcgtq_s64(_Swapped, _Cur); + return vbslq_s64(_Mask_gt, _Swapped, _Cur); + } + + static _Vec_t _H_min_u(const _Vec_t _Cur) noexcept { + const uint64x2_t _Cur_u = vreinterpretq_u64_s64(_Cur); + uint64x2_t _Swapped = vextq_u64(_Cur_u, _Cur_u, 1); + uint64x2_t _Mask_lt = vcltq_u64(_Swapped, _Cur_u); + return vreinterpretq_s64_u64(vbslq_u64(_Mask_lt, _Swapped, _Cur_u)); + } + + static _Vec_t _H_max_u(const _Vec_t _Cur) noexcept { + const uint64x2_t _Cur_u = vreinterpretq_u64_s64(_Cur); + uint64x2_t _Swapped = vextq_u64(_Cur_u, _Cur_u, 1); + uint64x2_t _Mask_gt = vcgtq_u64(_Swapped, _Cur_u); + return vreinterpretq_s64_u64(vbslq_u64(_Mask_gt, _Swapped, _Cur_u)); + } + + static _Signed_t _Get_any(const _Vec_t _Cur) noexcept { + return static_cast<_Signed_t>(vgetq_lane_s64(_Cur, 0)); + } + + static _Vec_t _Cmp_gt(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s64_u64(vcgtq_s64(_First, _Second)); + } + + static _Vec_t _Cmp_gt_u(const _Vec_t _First, const _Vec_t _Second) noexcept { + return vreinterpretq_s64_u64(vcgtq_u64(vreinterpretq_u64_s64(_First), vreinterpretq_u64_s64(_Second))); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, const _Vec_t _Mask) noexcept { + return vbslq_s64(vreinterpretq_u64_s64(_Mask), _Second, _First); + } + + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Min(_First, _Second, _Cmp_gt(_First, _Second)); + } + + static _Vec_t _Min_u(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Min(_First, _Second, _Cmp_gt_u(_First, _Second)); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, const _Vec_t _Mask) noexcept { + return vbslq_s64(vreinterpretq_u64_s64(_Mask), _Second, _First); + } + + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Max(_First, _Second, _Cmp_gt(_Second, _First)); + } + + static _Vec_t _Max_u(const _Vec_t _First, const _Vec_t _Second) noexcept { + return _Max(_First, _Second, _Cmp_gt_u(_Second, _First)); + } + }; +#elif !defined(_M_ARM64EC) struct _Traits_8_sse : _Traits_8_base, _Traits_sse_base { static __m128i _Load(const void* const _Src) noexcept { return _mm_loadu_si128(reinterpret_cast(_Src)); @@ -2125,7 +2234,7 @@ namespace { return _Mask; } }; -#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ +#endif // ^^^ !defined(_M_ARM64EC) ^^^ struct _Traits_f_base { static constexpr bool _Is_floating = true; @@ -2155,6 +2264,22 @@ namespace { using _Idx_t = int32x4_t; static constexpr bool _Has_unsigned_cmp = false; + static _Vec_t _Sign_correction(const _Vec_t _Val, bool) noexcept { + return _Val; + } + + static _Idx_t _Zero() noexcept { + return vdupq_n_s32(0); + } + + static _Idx_t _All_ones() noexcept { + return vreinterpretq_s32_s8(vdupq_n_s8(static_cast(0xFF))); + } + + static _Idx_t _Blend(const _Idx_t _Px1, const _Idx_t _Px2, const _Idx_t _Msk) noexcept { + return vbslq_s32(vreinterpretq_u32_s32(_Msk), _Px2, _Px1); + } + static uint64_t _Mask(const _Idx_t _Val) noexcept { return _Traits_4_neon::_Mask(_Val); } @@ -2211,20 +2336,22 @@ namespace { return _Traits_4_neon::_Cmp_eq_idx(_First, _Second); } - static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f32(0)) noexcept { + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Idx_t = vdupq_n_s32(0)) noexcept { return vminq_f32(_First, _Second); } - static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f32(0)) noexcept { + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Idx_t = vdupq_n_s32(0)) noexcept { return vmaxq_f32(_First, _Second); } - static _Idx_t _Mask_cast(const _Vec_t _Mask) noexcept { - return vreinterpretq_s32_f32(_Mask); + static _Idx_t _Mask_cast(const _Idx_t _Mask) noexcept { + return _Mask; } }; #elif !defined(_M_ARM64EC) struct _Traits_f_sse : _Traits_f_base, _Traits_sse_base { + using _Vec_t = __m128; + static __m128 _Load(const void* const _Src) noexcept { return _mm_loadu_ps(reinterpret_cast(_Src)); } @@ -2298,6 +2425,7 @@ namespace { struct _Traits_f_avx : _Traits_f_base, _Traits_avx_base { static constexpr size_t _Tail_mask = 0x1C; + using _Vec_t = __m256; static __m256 _Blendval(const __m256 _Px1, const __m256 _Px2, const __m256i _Msk) noexcept { return _mm256_blendv_ps(_Px1, _Px2, _mm256_castsi256_ps(_Msk)); @@ -2403,6 +2531,22 @@ namespace { using _Idx_t = int64x2_t; static constexpr bool _Has_unsigned_cmp = false; + static _Vec_t _Sign_correction(const _Vec_t _Val, bool) noexcept { + return _Val; + } + + static _Idx_t _Zero() noexcept { + return vdupq_n_s64(0); + } + + static _Idx_t _All_ones() noexcept { + return vreinterpretq_s64_s8(vdupq_n_s8(static_cast(0xFF))); + } + + static _Idx_t _Blend(const _Idx_t _Px1, const _Idx_t _Px2, const _Idx_t _Msk) noexcept { + return vbslq_s64(vreinterpretq_u64_s64(_Msk), _Px2, _Px1); + } + // Compresses a 128-bit Mask of 2 64-bit values into a 64-bit Mask of 2 32-bit values. static uint64_t _Mask(const int64x2_t _Val) noexcept { const uint32x2_t _Res = vreinterpret_u32_s32(vmovn_s64(_Val)); @@ -2434,17 +2578,11 @@ namespace { } static _Idx_t _H_min_u(const _Idx_t _Cur) noexcept { - const uint64x2_t _Cur_u = vreinterpretq_u64_s64(_Cur); - const uint64x2_t _Swapped = vextq_u64(_Cur_u, _Cur_u, 1); - const uint64x2_t _Mask_lt = vcltq_u64(_Swapped, _Cur_u); - return vreinterpretq_s64_u64(vbslq_u64(_Mask_lt, _Swapped, _Cur_u)); + return _Traits_8_neon::_H_min_u(_Cur); } static _Idx_t _H_max_u(const _Idx_t _Cur) noexcept { - const uint64x2_t _Cur_u = vreinterpretq_u64_s64(_Cur); - const uint64x2_t _Swapped = vextq_u64(_Cur_u, _Cur_u, 1); - const uint64x2_t _Mask_gt = vcgtq_u64(_Swapped, _Cur_u); - return vreinterpretq_s64_u64(vbslq_u64(_Mask_gt, _Swapped, _Cur_u)); + return _Traits_8_neon::_H_max_u(_Cur); } static double _Get_any(const _Vec_t _Cur) noexcept { @@ -2467,20 +2605,22 @@ namespace { return vreinterpretq_s64_u64(vceqq_s64(_First, _Second)); } - static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f64(0)) noexcept { + static _Vec_t _Min(const _Vec_t _First, const _Vec_t _Second, _Idx_t = vdupq_n_s64(0)) noexcept { return vminq_f64(_First, _Second); } - static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Vec_t = vdupq_n_f64(0)) noexcept { + static _Vec_t _Max(const _Vec_t _First, const _Vec_t _Second, _Idx_t = vdupq_n_s64(0)) noexcept { return vmaxq_f64(_First, _Second); } - static _Idx_t _Mask_cast(const _Vec_t _Mask) noexcept { - return vreinterpretq_s64_f64(_Mask); + static _Idx_t _Mask_cast(const _Idx_t _Mask) noexcept { + return _Mask; } }; #elif !defined(_M_ARM64EC) struct _Traits_d_sse : _Traits_d_base, _Traits_sse_base { + using _Vec_t = __m128d; + static __m128d _Load(const void* const _Src) noexcept { return _mm_loadu_pd(reinterpret_cast(_Src)); } @@ -2552,6 +2692,7 @@ namespace { struct _Traits_d_avx : _Traits_d_base, _Traits_avx_base { static constexpr size_t _Tail_mask = 0x18; + using _Vec_t = __m256d; static __m256d _Blendval(const __m256d _Px1, const __m256d _Px2, const __m256i _Msk) noexcept { return _mm256_blendv_pd(_Px1, _Px2, _mm256_castsi256_pd(_Msk)); @@ -2663,15 +2804,15 @@ namespace { #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#ifndef _M_ARM64 struct _Traits_8 { using _Scalar = _Traits_scalar<_Traits_8_base>; -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + using _Neon = _Traits_8_neon; +#elif !defined(_M_ARM64EC) using _Sse = _Traits_8_sse; using _Avx = _Traits_8_avx; #endif // ^^^ !defined(_M_ARM64EC) ^^^ }; -#endif // ^^^ !defined(_M_ARM64) ^^^ struct _Traits_f { using _Scalar = _Traits_scalar<_Traits_f_base>; @@ -3098,10 +3239,10 @@ namespace { #endif // ^^^ !defined(_M_ARM64) ^^^ } -#ifndef _M_ARM64 - template <_Min_max_mode _Mode, class _Traits, bool _Sign> + template <_Min_max_mode _Mode, class _Traits, bool _Sign, bool _Unrolled = false> auto _Minmax_impl(const void* _First, const void* const _Last) noexcept { - using _Ty = std::conditional_t<_Sign, typename _Traits::_Signed_t, typename _Traits::_Unsigned_t>; + using _Ty = std::conditional_t<_Sign, typename _Traits::_Signed_t, typename _Traits::_Unsigned_t>; + using _VecTy = _Traits::_Vec_t; _Ty _Cur_min_val; // initialized in both of the branches below _Ty _Cur_max_val; // initialized in both of the branches below @@ -3110,56 +3251,85 @@ namespace { #ifdef _M_ARM64EC static_assert(false, "No vectorization for _M_ARM64EC yet"); #else // ^^^ defined(_M_ARM64EC) / !defined(_M_ARM64EC) vvv + constexpr size_t _Lanes = _Unrolled ? 2 : 1; + constexpr size_t _Bytes_per_iter = _Lanes * _Traits::_Vec_size; + const size_t _Total_size_bytes = _Byte_length(_First, _Last); - const size_t _Vec_byte_size = _Total_size_bytes & ~_Traits::_Vec_mask; + const size_t _Vec_byte_size = _Total_size_bytes & ~(_Bytes_per_iter - 1); const void* _Stop_at = _First; _Advance_bytes(_Stop_at, _Vec_byte_size); - auto _Cur_vals = _Traits::_Load(_First); - // We don't have unsigned 64-bit stuff, so we'll use sign correction just for that case - constexpr bool _Sign_correction = sizeof(_Ty) == 8 && !_Sign; - - if constexpr (_Sign_correction) { - _Cur_vals = _Traits::_Sign_correction(_Cur_vals, false); + constexpr bool _Sign_correction = sizeof(_Ty) == 8 && !_Sign && !_Traits::_Has_unsigned_cmp; + + _VecTy _Cur_vals[_Lanes]; + _VecTy _Cur_vals_min[_Lanes]; // vector of vertical minimum values + _VecTy _Cur_vals_max[_Lanes]; // vector of vertical maximum values + for (size_t _Lane = 0; _Lane < _Lanes; ++_Lane) { + _Cur_vals[_Lane] = _Traits::_Load(static_cast(_First) + _Lane * _Traits::_Vec_size); + if constexpr (_Sign_correction) { + _Cur_vals[_Lane] = _Traits::_Sign_correction(_Cur_vals[_Lane], false); + } + _Cur_vals_min[_Lane] = _Cur_vals[_Lane]; + _Cur_vals_max[_Lane] = _Cur_vals[_Lane]; } - auto _Cur_vals_min = _Cur_vals; // vector of vertical minimum values - auto _Cur_vals_max = _Cur_vals; // vector of vertical maximum values - - const auto _Update_min_max = [&](const auto _Cur_vals) noexcept { + const auto _Update_min_max = [&](const auto _Cur_vals, size_t _Lane = 0) noexcept { if constexpr ((_Mode & _Mode_min) != 0) { if constexpr (_Sign || _Sign_correction) { - _Cur_vals_min = _Traits::_Min(_Cur_vals_min, _Cur_vals); // Update the current minimum + _Cur_vals_min[_Lane] = + _Traits::_Min(_Cur_vals_min[_Lane], _Cur_vals); // Update the current minimum } else { - _Cur_vals_min = _Traits::_Min_u(_Cur_vals_min, _Cur_vals); // Update the current minimum + _Cur_vals_min[_Lane] = + _Traits::_Min_u(_Cur_vals_min[_Lane], _Cur_vals); // Update the current minimum } } if constexpr ((_Mode & _Mode_max) != 0) { if constexpr (_Sign || _Sign_correction) { - _Cur_vals_max = _Traits::_Max(_Cur_vals_max, _Cur_vals); // Update the current maximum + _Cur_vals_max[_Lane] = + _Traits::_Max(_Cur_vals_max[_Lane], _Cur_vals); // Update the current maximum } else { - _Cur_vals_max = _Traits::_Max_u(_Cur_vals_max, _Cur_vals); // Update the current maximum + _Cur_vals_max[_Lane] = + _Traits::_Max_u(_Cur_vals_max[_Lane], _Cur_vals); // Update the current maximum } } }; for (;;) { - _Advance_bytes(_First, _Traits::_Vec_size); + _Advance_bytes(_First, _Bytes_per_iter); if (_First != _Stop_at) { // This is the main part, finding vertical minimum/maximum - _Cur_vals = _Traits::_Load(_First); + for (size_t _Lane = 0; _Lane < _Lanes; ++_Lane) { + _Cur_vals[_Lane] = + _Traits::_Load(static_cast(_First) + _Lane * _Traits::_Vec_size); - if constexpr (_Sign_correction) { - _Cur_vals = _Traits::_Sign_correction(_Cur_vals, false); - } + if constexpr (_Sign_correction) { + _Cur_vals[_Lane] = _Traits::_Sign_correction(_Cur_vals[_Lane], false); + } - _Update_min_max(_Cur_vals); + _Update_min_max(_Cur_vals[_Lane], _Lane); + } } else { + if constexpr (_Unrolled) { + const bool _Has_vec_tail = (_Byte_length(_First, _Last) & ~_Traits::_Vec_mask) != 0; + + if (_Has_vec_tail) { + _Cur_vals[0] = _Traits::_Load(_First); + + if constexpr (_Sign_correction) { + _Cur_vals[0] = _Traits::_Sign_correction(_Cur_vals[0], false); + } + + _Update_min_max(_Cur_vals[0], 0); + + _Advance_bytes(_First, _Traits::_Vec_size); + } + } + if constexpr (_Traits::_Tail_mask != 0) { const size_t _Tail_byte_size = _Total_size_bytes & _Traits::_Tail_mask; if (_Tail_byte_size != 0) { @@ -3170,7 +3340,7 @@ namespace { _Tail_vals = _Traits::_Sign_correction(_Tail_vals, false); } - _Tail_vals = _Traits::_Blendval(_Cur_vals, _Tail_vals, _Tail_mask); + _Tail_vals = _Traits::_Blendval(_Cur_vals[0], _Tail_vals, _Tail_mask); _Update_min_max(_Tail_vals); @@ -3182,24 +3352,48 @@ namespace { if constexpr ((_Mode & _Mode_min) != 0) { if constexpr (_Sign || _Sign_correction) { + if constexpr (_Unrolled) { + for (size_t _Lane = 1; _Lane < _Lanes; ++_Lane) { + _Cur_vals_min[0] = _Traits::_Min(_Cur_vals_min[0], _Cur_vals_min[_Lane]); + } + } + // Vector populated by the smallest element - const auto _H_min = _Traits::_H_min(_Cur_vals_min); + const auto _H_min = _Traits::_H_min(_Cur_vals_min[0]); _Cur_min_val = _Traits::_Get_any(_H_min); // Get any element of it } else { + if constexpr (_Unrolled) { + for (size_t _Lane = 1; _Lane < _Lanes; ++_Lane) { + _Cur_vals_min[0] = _Traits::_Min_u(_Cur_vals_min[0], _Cur_vals_min[_Lane]); + } + } + // Vector populated by the smallest element - const auto _H_min = _Traits::_H_min_u(_Cur_vals_min); + const auto _H_min = _Traits::_H_min_u(_Cur_vals_min[0]); _Cur_min_val = _Traits::_Get_any(_H_min); // Get any element of it } } if constexpr ((_Mode & _Mode_max) != 0) { if constexpr (_Sign || _Sign_correction) { + if constexpr (_Unrolled) { + for (size_t _Lane = 1; _Lane < _Lanes; ++_Lane) { + _Cur_vals_max[0] = _Traits::_Max(_Cur_vals_max[0], _Cur_vals_max[_Lane]); + } + } + // Vector populated by the largest element - const auto _H_max = _Traits::_H_max(_Cur_vals_max); + const auto _H_max = _Traits::_H_max(_Cur_vals_max[0]); _Cur_max_val = _Traits::_Get_any(_H_max); // Get any element of it } else { + if constexpr (_Unrolled) { + for (size_t _Lane = 1; _Lane < _Lanes; ++_Lane) { + _Cur_vals_max[0] = _Traits::_Max_u(_Cur_vals_max[0], _Cur_vals_max[_Lane]); + } + } + // Vector populated by the largest element - const auto _H_max = _Traits::_H_max_u(_Cur_vals_max); + const auto _H_max = _Traits::_H_max_u(_Cur_vals_max[0]); _Cur_max_val = _Traits::_Get_any(_H_max); // Get any element of it } } @@ -3229,6 +3423,8 @@ namespace { _Advance_bytes(_First, sizeof(_Ty)); } +// Avoid auto-vectorization of the scalar tail, as this is not beneficial for performance. +#pragma loop(no_vector) for (auto _Ptr = static_cast(_First); _Ptr != _Last; ++_Ptr) { if constexpr ((_Mode & _Mode_min) != 0) { if (*_Ptr < _Cur_min_val) { @@ -3258,7 +3454,7 @@ namespace { } } -#ifndef _M_ARM64EC +#if !defined(_M_ARM64) && !defined(_M_ARM64EC) // TRANSITION, DevCom-10767462 template <_Min_max_mode _Mode, class _Traits, bool _Sign> auto _Minmax_impl_wrap(const void* const _First, const void* const _Last) noexcept { @@ -3266,11 +3462,19 @@ namespace { _mm256_zeroupper(); return _Rx; } -#endif // ^^^ !defined(_M_ARM64EC) ^^^ +#endif // ^^^ !defined(_M_ARM64) && !defined(_M_ARM64EC) ^^^ template <_Min_max_mode _Mode, class _Traits, bool _Sign> auto __stdcall _Minmax_disp(const void* const _First, const void* const _Last) noexcept { -#ifndef _M_ARM64EC +#ifdef _M_ARM64 + if (_Byte_length(_First, _Last) >= 32) { + return _Minmax_impl<_Mode, typename _Traits::_Neon, _Sign, true>(_First, _Last); + } + + if (_Byte_length(_First, _Last) >= 16) { + return _Minmax_impl<_Mode, typename _Traits::_Neon, _Sign, false>(_First, _Last); + } +#elif !defined(_M_ARM64EC) if (_Byte_length(_First, _Last) >= 32 && _Use_avx2()) { if constexpr (_Traits::_Avx::_Is_floating) { return _Minmax_impl_wrap<_Mode, typename _Traits::_Avx, _Sign>(_First, _Last); @@ -3286,6 +3490,7 @@ namespace { return _Minmax_impl<_Mode, typename _Traits::_Scalar, _Sign>(_First, _Last); } +#ifndef _M_ARM64 template const void* _Is_sorted_until_impl(const void* _First, const void* const _Last, const bool _Greater) noexcept { const ptrdiff_t _Left_off = 0 - static_cast(_Greater); @@ -3487,7 +3692,6 @@ _Min_max_element_t __stdcall __std_minmax_element_d(const void* const _First, co return _Sorting::_Minmax_element_disp<_Sorting::_Mode_both, _Sorting::_Traits_d>(_First, _Last, false); } -#ifndef _M_ARM64 __declspec(noalias) int8_t __stdcall __std_min_1i(const void* const _First, const void* const _Last) noexcept { return _Sorting::_Minmax_disp<_Sorting::_Mode_min, _Sorting::_Traits_1, true>(_First, _Last); } @@ -3608,6 +3812,7 @@ __declspec(noalias) _Min_max_d __stdcall __std_minmax_d(const void* const _First return _Sorting::_Minmax_disp<_Sorting::_Mode_both, _Sorting::_Traits_d, true>(_First, _Last); } +#ifndef _M_ARM64 const void* __stdcall __std_is_sorted_until_1i( const void* const _First, const void* const _Last, const bool _Greater) noexcept { return _Sorting::_Is_sorted_until_disp<_Sorting::_Traits_1, int8_t>(_First, _Last, _Greater); From 9553d3a21dc1d13dadf7f3773f60cebdcf122421 Mon Sep 17 00:00:00 2001 From: Hari Limaye Date: Thu, 15 Jan 2026 18:40:18 +0000 Subject: [PATCH 15/19] Add Neon implementation of `find` for ARM64 targets (#6003) Co-authored-by: Stephan T. Lavavej --- stl/inc/xutility | 2 +- stl/src/vector_algorithms.cpp | 317 ++++++++++++++++++++++++++++++++-- 2 files changed, 304 insertions(+), 15 deletions(-) diff --git a/stl/inc/xutility b/stl/inc/xutility index 577161e4571..a2e18498b63 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -82,7 +82,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_BITSET_FROM_STRING _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_BITSET_TO_STRING _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_COUNT _VECTORIZED_FOR_X64_X86 -#define _VECTORIZED_FIND _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_FIND _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_FIND_END _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_FIND_FIRST_OF _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_FIND_LAST _VECTORIZED_FOR_X64_X86 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index 0271995eed8..a9d49bd754a 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -3866,10 +3866,187 @@ const void* __stdcall __std_is_sorted_until_d( } // extern "C" -#ifndef _M_ARM64 namespace { namespace _Finding { -#ifdef _M_ARM64EC +#ifdef _M_ARM64 + struct _Find_traits_1 { + static uint8x16_t _Load_q(const void* const _Ptr) noexcept { + return vld1q_u8(static_cast(_Ptr)); + } + + static uint8x8_t _Load(const void* const _Ptr) noexcept { + return vld1_u8(static_cast(_Ptr)); + } + + static uint8x16_t _Set_neon_q(const uint8_t _Val) noexcept { + return vdupq_n_u8(_Val); + } + + static uint8x8_t _Set_neon(const uint8_t _Val) noexcept { + return vdup_n_u8(_Val); + } + + static uint8x16_t _Cmp_neon_q(const uint8x16_t _Lhs, const uint8x16_t _Rhs) noexcept { + return vceqq_u8(_Lhs, _Rhs); + } + + static uint8x8_t _Cmp_neon(const uint8x8_t _Lhs, const uint8x8_t _Rhs) noexcept { + return vceq_u8(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 16 8-bit values into a 64-bit Mask of 16 4-bit values. + static uint64_t _Mask_q(const uint8x16_t _Val) noexcept { + const uint8x8_t _Res = vshrn_n_u16(vreinterpretq_u16_u8(_Val), 4); + return vget_lane_u64(vreinterpret_u64_u8(_Res), 0); + } + + static uint64_t _Mask(const uint8x8_t _Val) noexcept { + return vget_lane_u64(vreinterpret_u64_u8(_Val), 0); + } + + static uint64_t _Match_mask_eq(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { + auto _Cmp = vreinterpretq_u64_u8(vorrq_u8(_Cmp_lo, _Cmp_hi)); + return vgetq_lane_u64(vpaddq_u64(_Cmp, _Cmp), 0); + } + + static uint64_t _Match_mask_ne(const uint8x16_t _Cmp_lo, const uint8x16_t _Cmp_hi) noexcept { + auto _Cmp = vminq_u8(_Cmp_lo, _Cmp_hi); + auto _Comb = vreinterpretq_u64_u8(vpminq_u8(_Cmp, _Cmp)); + return vgetq_lane_u64(_Comb, 0) ^ 0xFFFF'FFFF'FFFF'FFFF; + } + }; + + struct _Find_traits_2 { + static uint16x8_t _Load_q(const void* const _Ptr) noexcept { + return vld1q_u16(static_cast(_Ptr)); + } + + static uint16x4_t _Load(const void* const _Ptr) noexcept { + return vld1_u16(static_cast(_Ptr)); + } + + static uint16x8_t _Set_neon_q(const uint16_t _Val) noexcept { + return vdupq_n_u16(_Val); + } + + static uint16x4_t _Set_neon(const uint16_t _Val) noexcept { + return vdup_n_u16(_Val); + } + + static uint16x8_t _Cmp_neon_q(const uint16x8_t _Lhs, const uint16x8_t _Rhs) noexcept { + return vceqq_u16(_Lhs, _Rhs); + } + + static uint16x4_t _Cmp_neon(const uint16x4_t _Lhs, const uint16x4_t _Rhs) noexcept { + return vceq_u16(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 8 16-bit values into a 64-bit Mask of 8 8-bit values. + static uint64_t _Mask_q(const uint16x8_t _Val) noexcept { + const uint16x4_t _Res = vshrn_n_u32(vreinterpretq_u32_u16(_Val), 8); + return vget_lane_u64(vreinterpret_u64_u16(_Res), 0); + } + + static uint64_t _Mask(const uint16x4_t _Val) noexcept { + return vget_lane_u64(vreinterpret_u64_u16(_Val), 0); + } + + static uint64_t _Match_mask_eq(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { + uint8x8_t _Cmp = vaddhn_u16(_Cmp_lo, _Cmp_hi); + return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); + } + + static uint64_t _Match_mask_ne(const uint16x8_t _Cmp_lo, const uint16x8_t _Cmp_hi) noexcept { + auto _Cmp = vminq_u16(_Cmp_lo, _Cmp_hi); + auto _Comb = vreinterpretq_u64_u16(vpminq_u16(_Cmp, _Cmp)); + return vgetq_lane_u64(_Comb, 0) ^ 0xFFFF'FFFF'FFFF'FFFF; + } + }; + + struct _Find_traits_4 { + static uint32x4_t _Load_q(const void* const _Ptr) noexcept { + return vld1q_u32(static_cast(_Ptr)); + } + + static uint32x2_t _Load(const void* const _Ptr) noexcept { + return vld1_u32(static_cast(_Ptr)); + } + + static uint32x4_t _Set_neon_q(const uint32_t _Val) noexcept { + return vdupq_n_u32(_Val); + } + + static uint32x2_t _Set_neon(const uint32_t _Val) noexcept { + return vdup_n_u32(_Val); + } + + static uint32x4_t _Cmp_neon_q(const uint32x4_t _Lhs, const uint32x4_t _Rhs) noexcept { + return vceqq_u32(_Lhs, _Rhs); + } + + static uint32x2_t _Cmp_neon(const uint32x2_t _Lhs, const uint32x2_t _Rhs) noexcept { + return vceq_u32(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 4 32-bit values into a 64-bit Mask of 4 16-bit values. + static uint64_t _Mask_q(const uint32x4_t _Val) noexcept { + const uint32x2_t _Res = vshrn_n_u64(vreinterpretq_u64_u32(_Val), 16); + return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); + } + + static uint64_t _Mask(const uint32x2_t _Val) noexcept { + return vget_lane_u64(vreinterpret_u64_u32(_Val), 0); + } + + static uint64_t _Match_mask_eq(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { + uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u32(_Cmp_lo), vreinterpretq_u16_u32(_Cmp_hi)); + return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); + } + + static uint64_t _Match_mask_ne(const uint32x4_t _Cmp_lo, const uint32x4_t _Cmp_hi) noexcept { + auto _Cmp = vminq_u32(_Cmp_lo, _Cmp_hi); + auto _Comb = vreinterpretq_u64_u32(vpminq_u32(_Cmp, _Cmp)); + return vgetq_lane_u64(_Comb, 0) ^ 0xFFFF'FFFF'FFFF'FFFF; + } + }; + + struct _Find_traits_8 { + static uint64x2_t _Load_q(const void* const _Ptr) noexcept { + return vld1q_u64(static_cast(_Ptr)); + } + + static uint64x2_t _Set_neon_q(const uint64_t _Val) noexcept { + return vdupq_n_u64(_Val); + } + + static uint64x2_t _Cmp_neon_q(const uint64x2_t _Lhs, const uint64x2_t _Rhs) noexcept { + return vceqq_u64(_Lhs, _Rhs); + } + + // Compresses a 128-bit Mask of 2 64-bit values into a 64-bit Mask of 2 32-bit values. + static uint64_t _Mask_q(const uint64x2_t _Val) noexcept { + const uint32x2_t _Res = vmovn_u64(_Val); + return vget_lane_u64(vreinterpret_u64_u32(_Res), 0); + } + + static uint64_t _Match_mask_eq(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { + uint8x8_t _Cmp = vaddhn_u16(vreinterpretq_u16_u64(_Cmp_lo), vreinterpretq_u16_u64(_Cmp_hi)); + return vget_lane_u64(vreinterpret_u64_u8(_Cmp), 0); + } + + static uint64_t _Match_mask_ne(const uint64x2_t _Cmp_lo, const uint64x2_t _Cmp_hi) noexcept { + return _Mask_q(vandq_u64(_Cmp_lo, _Cmp_hi)) ^ 0xFFFF'FFFF'FFFF'FFFF; + } + }; + + unsigned long _Get_first_h_pos_q(const uint64_t _Mask) noexcept { + return _CountTrailingZeros64(_Mask) >> 2; + } + + unsigned long _Get_first_h_pos_d(const uint64_t _Mask) noexcept { + return _CountTrailingZeros64(_Mask) >> 3; + } +#elif defined(_M_ARM64EC) using _Find_traits_1 = void; using _Find_traits_2 = void; using _Find_traits_4 = void; @@ -3948,6 +4125,7 @@ namespace { }; #endif // ^^^ !defined(_M_ARM64EC) ^^^ +#ifndef _M_ARM64 // TRANSITION, ABI: used only in functions preserved for binary compatibility template const void* __stdcall _Find_unsized_impl(const void* const _First, const _Ty _Val) noexcept { @@ -3957,13 +4135,125 @@ namespace { } return _Ptr; } +#endif // ^^^ !defined(_M_ARM64) ^^^ enum class _Predicate { _Equal, _Not_equal }; + template <_Predicate _Pred, class _Ty> + const void* _Find_scalar_tail(const void* const _First, const void* const _Last, const _Ty _Val) noexcept { + auto _Ptr = static_cast(_First); + if constexpr (_Pred == _Predicate::_Not_equal) { + while (_Ptr != _Last && *_Ptr == _Val) { + ++_Ptr; + } + } else { + while (_Ptr != _Last && *_Ptr != _Val) { + ++_Ptr; + } + } + return _Ptr; + } + // The below functions have exactly the same signature as the extern "C" functions, up to calling convention. // This makes sure the template specialization can be fused with the extern "C" function. // In optimized builds it avoids an extra call, as these functions are too large to inline. +#ifdef _M_ARM64 + template + const void* __stdcall _Find_impl(const void* _First, const void* const _Last, const _Ty _Val) noexcept { + const size_t _Size_bytes = _Byte_length(_First, _Last); + + if (const size_t _Neon_size = _Size_bytes & ~size_t{0x1F}; _Neon_size != 0) { + const auto _Comparand = _Traits::_Set_neon_q(_Val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Neon_size); + + do { + const auto _Data_lo = _Traits::_Load_q(static_cast(_First) + 0); + const auto _Data_hi = _Traits::_Load_q(static_cast(_First) + 16); + + auto _Comparison_lo = _Traits::_Cmp_neon_q(_Data_lo, _Comparand); + auto _Comparison_hi = _Traits::_Cmp_neon_q(_Data_hi, _Comparand); + + // Use a fast check for the termination condition. + uint64_t _Any_match = 0; + if constexpr (_Pred == _Predicate::_Not_equal) { + _Any_match = _Traits::_Match_mask_ne(_Comparison_lo, _Comparison_hi); + } else { + _Any_match = _Traits::_Match_mask_eq(_Comparison_lo, _Comparison_hi); + } + + if (_Any_match != 0) { + auto _Mask_lo = _Traits::_Mask_q(_Comparison_lo); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Mask_lo ^= 0xFFFF'FFFF'FFFF'FFFF; + } + + if (_Mask_lo != 0) { + const auto _Offset = _Get_first_h_pos_q(_Mask_lo); + _Advance_bytes(_First, _Offset); + return _First; + } + + auto _Mask_hi = _Traits::_Mask_q(_Comparison_hi); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Mask_hi ^= 0xFFFF'FFFF'FFFF'FFFF; + } + + const auto _Offset = _Get_first_h_pos_q(_Mask_hi) + 16; + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 32); + } while (_First != _Stop_at); + } + + if ((_Size_bytes & size_t{0x10}) != 0) { + const auto _Comparand = _Traits::_Set_neon_q(_Val); + const auto _Data = _Traits::_Load_q(_First); + + auto _Comparison = _Traits::_Cmp_neon_q(_Data, _Comparand); + + auto _Match = _Traits::_Mask_q(_Comparison); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Match ^= 0xFFFF'FFFF'FFFF'FFFF; + } + + if (_Match != 0) { + const auto _Offset = _Get_first_h_pos_q(_Match); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 16); + } + + if constexpr (sizeof(_Ty) < 8) { + if ((_Size_bytes & size_t{0x08}) != 0) { + const auto _Comparand = _Traits::_Set_neon(_Val); + const auto _Data = _Traits::_Load(_First); + + auto _Comparison = _Traits::_Cmp_neon(_Data, _Comparand); + + auto _Match = _Traits::_Mask(_Comparison); + if constexpr (_Pred == _Predicate::_Not_equal) { + _Match ^= 0xFFFF'FFFF'FFFF'FFFF; + } + + if (_Match != 0) { + const auto _Offset = _Get_first_h_pos_d(_Match); + _Advance_bytes(_First, _Offset); + return _First; + } + + _Advance_bytes(_First, 8); + } + } + + return _Find_scalar_tail<_Pred>(_First, _Last, _Val); + } +#else // ^^^ defined(_M_ARM64) / !defined(_M_ARM64) vvv template const void* __stdcall _Find_impl(const void* _First, const void* const _Last, const _Ty _Val) noexcept { #ifndef _M_ARM64EC @@ -4040,17 +4330,8 @@ namespace { } while (_First != _Stop_at); } #endif // ^^^ !defined(_M_ARM64EC) ^^^ - auto _Ptr = static_cast(_First); - if constexpr (_Pred == _Predicate::_Not_equal) { - while (_Ptr != _Last && *_Ptr == _Val) { - ++_Ptr; - } - } else { - while (_Ptr != _Last && *_Ptr != _Val) { - ++_Ptr; - } - } - return _Ptr; + + return _Find_scalar_tail<_Pred>(_First, _Last, _Val); } template @@ -4449,11 +4730,13 @@ namespace { } } } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // namespace _Finding } // unnamed namespace extern "C" { +#ifndef _M_ARM64 // TRANSITION, ABI: preserved for binary compatibility const void* __stdcall __std_find_trivial_unsized_1(const void* const _First, const uint8_t _Val) noexcept { // C23 7.27.5.2 "The memchr generic function"/2 says "The implementation shall behave as if @@ -4479,10 +4762,11 @@ const void* __stdcall __std_find_trivial_unsized_4(const void* const _First, con const void* __stdcall __std_find_trivial_unsized_8(const void* const _First, const uint64_t _Val) noexcept { return _Finding::_Find_unsized_impl(_First, _Val); } +#endif // ^^^ !defined(_M_ARM64) ^^^ const void* __stdcall __std_find_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { -#ifdef _M_ARM64EC +#if defined(_M_ARM64) || defined(_M_ARM64EC) auto _Result = memchr(_First, _Val, _Byte_length(_First, _Last)); return _Result ? _Result : _Last; #else @@ -4510,6 +4794,7 @@ const void* __stdcall __std_find_trivial_8( return _Finding::_Find_impl<_Finding::_Find_traits_8, _Finding::_Predicate::_Equal>(_First, _Last, _Val); } +#ifndef _M_ARM64 const void* __stdcall __std_find_last_trivial_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { return _Finding::_Find_last_impl<_Finding::_Find_traits_1, _Finding::_Predicate::_Equal>(_First, _Last, _Val); @@ -4529,6 +4814,7 @@ const void* __stdcall __std_find_last_trivial_8( const void* const _First, const void* const _Last, const uint64_t _Val) noexcept { return _Finding::_Find_last_impl<_Finding::_Find_traits_8, _Finding::_Predicate::_Equal>(_First, _Last, _Val); } +#endif // ^^^ !defined(_M_ARM64) ^^^ const void* __stdcall __std_find_not_ch_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { @@ -4550,6 +4836,7 @@ const void* __stdcall __std_find_not_ch_8( return _Finding::_Find_impl<_Finding::_Find_traits_8, _Finding::_Predicate::_Not_equal>(_First, _Last, _Val); } +#ifndef _M_ARM64 __declspec(noalias) size_t __stdcall __std_find_last_not_ch_pos_1( const void* const _First, const void* const _Last, const uint8_t _Val) noexcept { return _Finding::_Find_last_pos_impl<_Finding::_Find_traits_1, _Finding::_Predicate::_Not_equal>( @@ -4609,9 +4896,11 @@ const void* __stdcall __std_search_n_8( const void* const _First, const void* const _Last, const size_t _Count, const uint64_t _Value) noexcept { return _Finding::_Search_n_impl<_Finding::_Find_traits_8>(_First, _Last, _Count, _Value); } +#endif // ^^^ !defined(_M_ARM64) ^^^ } // extern "C" +#ifndef _M_ARM64 namespace { namespace _Counting { #ifdef _M_ARM64EC From e98005d833f1be72a48a2fc956ad9a9620f5ab4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Thu, 15 Jan 2026 19:47:06 +0100 Subject: [PATCH 16/19] ``: Revise complexity limit (#6005) Co-authored-by: Stephan T. Lavavej --- stl/inc/regex | 117 ++++++++---------- .../std/tests/VSO_0000000_regex_use/test.cpp | 17 +++ 2 files changed, 70 insertions(+), 64 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 8343f4cf69b..932cddeefe4 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -43,10 +43,6 @@ _STL_DISABLE_CLANG_WARNINGS #define _REGEX_LEGACY_MULTILINE_MODE 0 #endif -#ifndef _REGEX_MAX_COMPLEXITY_COUNT -#define _REGEX_MAX_COMPLEXITY_COUNT 10000000L // set to 0 to disable -#endif // !defined(_REGEX_MAX_COMPLEXITY_COUNT) - #ifndef _ENHANCED_REGEX_VISUALIZER #ifdef _DEBUG #define _ENHANCED_REGEX_VISUALIZER 1 @@ -1720,8 +1716,9 @@ public: _Char_class_d = _Lookup_char_class(static_cast<_Elem>('D')); } - _Input_length = _STD distance(_Pfirst, _Plast); - _Frames_limit = _Calculate_frames_limit(); + _Iter_diff_t<_It> _Input_length = _STD distance(_Pfirst, _Plast); + _Frames_limit = _Calculate_frames_limit(_Input_length); + _Complexity_limit = _Calculate_complexity_limit(_Input_length); // sanitize multiline mode setting #if _REGEX_LEGACY_MULTILINE_MODE @@ -1761,9 +1758,8 @@ public: _Tgt_state._Grp_valid.resize(_Ncap); _Tgt_state._Grps.resize(_Ncap); } - _Full = _Full_match; - _Max_complexity_count = _REGEX_MAX_COMPLEXITY_COUNT; - _Frames_count = 0; + _Full = _Full_match; + _Frames_count = 0; _Matched = false; @@ -1817,13 +1813,13 @@ private: vector<_Rx_state_frame_t<_It>> _Frames; size_t _Frames_count; size_t _Frames_limit; - _Iter_diff_t<_It> _Input_length; + long long _Complexity_limit; size_t _Push_frame(_Rx_unwind_ops _Code, _Node_base* _Node); - void _Pop_frame(size_t); - size_t _Calculate_frames_limit(); - void _Increase_complexity_count(); + size_t _Calculate_frames_limit(_Iter_diff_t<_It> _Input_length); + long long _Calculate_complexity_limit(_Iter_diff_t<_It> _Input_length); + void _Increase_complexity_count(_Iter_diff_t<_It> _Count); void _Prepare_rep(_Node_rep*); bool _Find_first_inner_capture_group(_Node_base*, _Loop_vals_v3_t<_Iter_diff_t<_It>>*); @@ -1844,7 +1840,6 @@ private: bool _Longest; const _RxTraits& _Traits; bool _Full; - long _Max_complexity_count; typename _RxTraits::char_class_type _Char_class_w{}; typename _RxTraits::char_class_type _Char_class_s{}; typename _RxTraits::char_class_type _Char_class_d{}; @@ -3401,13 +3396,7 @@ size_t _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Push_frame(_Rx_unwind_ } template -void _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Pop_frame(size_t _Idx) { - _STL_INTERNAL_CHECK(_Idx + 1 == _Frames_count); - _Frames_count = _Idx; -} - -template -size_t _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Calculate_frames_limit() { +size_t _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Calculate_frames_limit(_Iter_diff_t<_It> _Input_length) { constexpr size_t _Fixed_part = 10000000U / sizeof(_Rx_state_frame_t<_It>); constexpr size_t _Divisor = sizeof(_Rx_state_frame_t<_It>) / 10U; const auto _Variable_part = _Input_length / static_cast<_Iter_diff_t<_It>>(_Divisor); @@ -3421,9 +3410,22 @@ size_t _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Calculate_frames_limit } template -void _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Increase_complexity_count() { - if (0 < _Max_complexity_count && --_Max_complexity_count <= 0) { +long long _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Calculate_complexity_limit( + _Iter_diff_t<_It> _Input_length) { + constexpr long long _Intercept = 300000LL; + constexpr long long _Gradient = 256LL; + if ((LLONG_MAX - _Intercept) / _Gradient < _Input_length) { + return LLONG_MAX; + } + return _Gradient * static_cast(_Input_length) + _Intercept; +} + +template +void _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Increase_complexity_count(_Iter_diff_t<_It> _Count) { + if (_Complexity_limit < _Count) { _Xregex_error(regex_constants::error_complexity); + } else { + _Complexity_limit -= static_cast(_Count); } } @@ -3538,11 +3540,12 @@ void _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Reset_capture_groups(uns } } -template -_BidIt1 _Cmp_chrange(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, _Pr _Pred) { +template +_BidIt1 _Cmp_chrange(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, _Counter& _Count, _Pr _Pred) { // compare character ranges _BidIt1 _Res = _Begin1; while (_Begin1 != _End1 && _Begin2 != _End2) { + ++_Count; if (!_Pred(*_Begin1++, *_Begin2++)) { return _Res; } @@ -3550,33 +3553,34 @@ _BidIt1 _Cmp_chrange(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _E return _Begin2 == _End2 ? _Begin1 : _Res; } -template +template _BidIt1 _Compare_translate_both(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, - const _RxTraits& _Traits, regex_constants::syntax_option_type _Sflags) { + const _RxTraits& _Traits, regex_constants::syntax_option_type _Sflags, _Counter& _Count) { // compare character ranges, translating characters in both ranges according to syntax options if (_Sflags & regex_constants::icase) { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_icase<_RxTraits>{_Traits}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, _Cmp_icase<_RxTraits>{_Traits}); } else if constexpr (_Is_any_of_v<_RxTraits, regex_traits, regex_traits>) { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, equal_to{}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, equal_to{}); } else if (_Sflags & regex_constants::collate) { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_collate<_RxTraits>{_Traits}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, _Cmp_collate<_RxTraits>{_Traits}); } else { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, equal_to{}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, equal_to{}); } } -template +template _BidIt1 _Compare_translate_left(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, - const _RxTraits& _Traits, regex_constants::syntax_option_type _Sflags) { + const _RxTraits& _Traits, regex_constants::syntax_option_type _Sflags, _Counter& _Count) { // compare character ranges, translating characters in the left range according to syntax options if (_Sflags & regex_constants::icase) { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_icase_translateleft<_RxTraits>{_Traits}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, _Cmp_icase_translateleft<_RxTraits>{_Traits}); } else if constexpr (_Is_any_of_v<_RxTraits, regex_traits, regex_traits>) { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, equal_to{}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, equal_to{}); } else if (_Sflags & regex_constants::collate) { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_collate_translateleft<_RxTraits>{_Traits}); + return _STD _Cmp_chrange( + _Begin1, _End1, _Begin2, _End2, _Count, _Cmp_collate_translateleft<_RxTraits>{_Traits}); } else { - return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, equal_to{}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Count, equal_to{}); } } @@ -3831,12 +3835,11 @@ bool _Is_ecmascript_line_terminator(_Elem _Ch) { template bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _Nx) { // check for match - _Increase_complexity_count(); - bool _Failed = false; while (_Nx) { do { // match current node + _Increase_complexity_count(_Iter_diff_t<_It>{1}); // one state transition _Node_base* _Next = _Nx->_Next; switch (_Nx->_Kind) { // handle current node's type case _N_nop: @@ -3890,14 +3893,18 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N { // check for string match _Node_str<_Elem>* _Node = static_cast<_Node_str<_Elem>*>(_Nx); _It _Res0; + _Iter_diff_t<_It> _Count{}; if ((_Res0 = _STD _Compare_translate_left(_Tgt_state._Cur, _End, _Node->_Data._Str(), - _Node->_Data._Str() + _Node->_Data._Size(), _Traits, _Sflags)) + _Node->_Data._Str() + _Node->_Data._Size(), _Traits, _Sflags, _Count)) != _Tgt_state._Cur) { _Tgt_state._Cur = _Res0; } else { _Failed = true; } + // divide by 64 because character comparisons are cheaper than state transitions + _Increase_complexity_count(static_cast<_Iter_diff_t<_It>>(_Count / 64)); + break; } @@ -3923,8 +3930,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N auto _Node = static_cast<_Node_assert*>(_Nx); _Push_frame(_Rx_unwind_ops::_After_assert, _Node); _Next = _Node->_Child; - - _Increase_complexity_count(); break; } @@ -3933,8 +3938,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N auto _Node = static_cast<_Node_assert*>(_Nx); _Push_frame(_Rx_unwind_ops::_After_neg_assert, _Node); _Next = _Node->_Child; - - _Increase_complexity_count(); break; } @@ -4035,13 +4038,18 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _It _Res0 = _Tgt_state._Cur; _It _Bx = _Tgt_state._Grps[_Node->_Idx]._Begin; _It _Ex = _Tgt_state._Grps[_Node->_Idx]._End; + _Iter_diff_t<_It> _Count{}; if (_Bx != _Ex // _Bx == _Ex for zero-length match - && (_Res0 = _STD _Compare_translate_both(_Tgt_state._Cur, _End, _Bx, _Ex, _Traits, _Sflags)) + && (_Res0 = _STD _Compare_translate_both( + _Tgt_state._Cur, _End, _Bx, _Ex, _Traits, _Sflags, _Count)) == _Tgt_state._Cur) { _Failed = true; } else { _Tgt_state._Cur = _Res0; } + + // divide by 64 because character comparisons are cheaper than state transitions + _Increase_complexity_count(static_cast<_Iter_diff_t<_It>>(_Count / 64)); } else if (_Sflags & (regex_constants::basic | regex_constants::grep)) { _Failed = true; } @@ -4053,7 +4061,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N auto _Node = static_cast<_Node_if*>(_Nx); if (_Node->_Child) { _Push_frame(_Rx_unwind_ops::_Disjunction_eval_alternative, _Node->_Child); - _Increase_complexity_count(); } break; } @@ -4070,7 +4077,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N if (_Node->_Simple_loop == 1) { _Sav._Loop_frame_idx = _Push_frame(_Rx_unwind_ops::_Do_nothing, _Node); - _Increase_complexity_count(); if (_Node->_Min > 0 || (_Greedy && !_Longest && _Node->_Max != 0)) { // try a rep first _Sav._Loop_idx = 1; // _Next is already assigned correctly for matching a rep @@ -4100,11 +4106,9 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _Frame._Loop_frame_idx_sav = _Sav._Loop_frame_idx; _Sav._Loop_idx = 1; _Sav._Loop_frame_idx = _Frame_idx; - _Increase_complexity_count(); // _Next is already assigned correctly for matching a rep } else { // try tail first _Next = _Node->_End_rep->_Next; - _Increase_complexity_count(); // set up stack unwinding for non-greedy matching if at least one rep is allowed if (_Node->_Max != 0) { auto _Frame_idx = _Push_frame(_Rx_unwind_ops::_Loop_nongreedy, _Node); @@ -4136,8 +4140,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N // _Next is already assigned correctly for matching tail if (!(_Sflags & regex_constants::_Any_posix) && _Nr->_Min == 0) { _Failed = true; - } else { - _Increase_complexity_count(); } break; } @@ -4197,8 +4199,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N } // _Next is already assigned correctly for matching tail } - - _Increase_complexity_count(); } else { const bool _Progress = _Frames[_Sav._Loop_frame_idx]._Pos != _Tgt_state._Cur; if (_Sav._Loop_idx < _Nr->_Min) { // try another required match @@ -4218,7 +4218,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _Reset_capture_groups(_Sav._Group_first); _Next = _Nr->_Next; - _Increase_complexity_count(); } else if (!_Progress) { // latest rep match empty // An empty match is allowed if it is needed to reach the minimum number of reps. // Moreover, POSIX allows an empty repetition if the subexpression is matched only once. @@ -4226,8 +4225,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N if (_Sav._Loop_idx != _Nr->_Min && !((_Sflags & regex_constants::_Any_posix) && _Sav._Loop_idx == 1)) { _Failed = true; - } else { - _Increase_complexity_count(); } // _Next is already assigned correctly for matching tail } else if (_Greedy && !_Longest && _Sav._Loop_idx != _Nr->_Max) { // one more rep to try next @@ -4243,10 +4240,8 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _Reset_capture_groups(_Sav._Group_first); _Next = _Nr->_Next; - _Increase_complexity_count(); } else { // non-greedy matching or greedy matching with maximum reached // set up stack unwinding for non-greedy matching if one more rep is allowed - _Increase_complexity_count(); if (_Sav._Loop_idx != _Nr->_Max) { auto _Frame_idx = _Push_frame(_Rx_unwind_ops::_Loop_nongreedy, _Nr); auto& _Frame = _Frames[_Frame_idx]; @@ -4315,7 +4310,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _Nx = _Node->_Next; _Tgt_state._Cur = _Frame._Pos; _Failed = false; - _Increase_complexity_count(); if (_Node->_Child) { _Frame._Node = _Node->_Child; ++_Frames_count; @@ -4328,7 +4322,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N auto _Node = static_cast<_Node_rep*>(_Frame._Node); auto& _Sav = _Loop_vals[_Node->_Loop_number]; - _Increase_complexity_count(); _Nx = _Node->_Next; _Tgt_state._Cur = _Frame._Pos; _Failed = false; @@ -4342,7 +4335,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N { // try tail after backtracking from first rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); - _Increase_complexity_count(); _Nx = _Node->_End_rep->_Next; _Tgt_state._Cur = _Frame._Pos; _Failed = false; @@ -4377,7 +4369,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N // when backtracking from last attempted rep auto _Node = static_cast<_Node_rep*>(_Frame._Node); - _Increase_complexity_count(); _Nx = _Node->_End_rep->_Next; _Tgt_state._Cur = _Frame._Pos; _Failed = false; @@ -4397,7 +4388,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N { // try tail auto _Node = static_cast<_Node_rep*>(_Frame._Node); - _Increase_complexity_count(); _Nx = _Node->_End_rep->_Next; _Tgt_state._Cur = _Frame._Pos; _Failed = false; @@ -4419,7 +4409,6 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N auto _Node = static_cast<_Node_rep*>(_Frame._Node); auto& _Sav = _Loop_vals[_Node->_Loop_number]; - _Increase_complexity_count(); _Nx = _Node->_Next; _Tgt_state._Cur = _Frame._Pos; _Failed = false; diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index 7b9ee69b694..cce150a1f88 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -2392,6 +2392,22 @@ void test_gh_5939() { g_regexTester.should_not_match("abcbbacdab", R"((?:([abc])([abc])){2,}abbacd\1\2)"); } +void test_gh_5944() { + // GH-5944: : Revising the stack and complexity limits + + // long strings should be matched successfully if the regex is simple + g_regexTester.should_match(string(20000000, 'a'), "a+"); + + // too much backtracking in complex regex expressions must result in a complexity exception + try { + regex re("a*[^b]*a*[^b]*a*[^b]*a*[^b]*a*[^b]*a*[^b]*"); + (void) regex_match("aaaaaaaaaaaaaaaaaaaaaaaaaaaaab", re); + assert(false); + } catch (const regex_error& ex) { + assert(ex.code() == error_complexity); + } +} + int main() { test_dev10_449367_case_insensitivity_should_work(); test_dev11_462743_regex_collate_should_not_disable_regex_icase(); @@ -2452,6 +2468,7 @@ int main() { test_gh_5865(); test_gh_5918(); test_gh_5939(); + test_gh_5944(); return g_regexTester.result(); } From 46d99b6237df46ff81c6368b36454150bf60ac2a Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 15 Jan 2026 10:48:52 -0800 Subject: [PATCH 17/19] Remove workarounds for ancient Clang versions (#6012) --- stl/inc/type_traits | 8 ++------ stl/inc/xutility | 14 +++----------- stl/inc/yvals.h | 2 -- stl/inc/yvals_core.h | 4 +--- 4 files changed, 6 insertions(+), 22 deletions(-) diff --git a/stl/inc/type_traits b/stl/inc/type_traits index e4c4d1e392f..f18d731c04d 100644 --- a/stl/inc/type_traits +++ b/stl/inc/type_traits @@ -346,8 +346,7 @@ _EXPORT_STD template _NO_SPECIALIZATIONS_OF_TYPE_TRAITS constexpr bool is_enum_v = __is_enum(_Ty); #if _HAS_CXX23 -#if defined(__clang__) && !defined(__EDG__) \ - && __clang_major__ >= 16 // TRANSITION, DevCom-10870354 (MSVC, EDG), VSO-2397560 (RWC relying on ancient Clang) +#ifdef __clang__ // TRANSITION, DevCom-10870354 (MSVC, EDG) _EXPORT_STD template _NO_SPECIALIZATIONS_OF_TYPE_TRAITS constexpr bool is_scoped_enum_v = __is_scoped_enum(_Ty); @@ -2014,15 +2013,13 @@ _NO_SPECIALIZATIONS_OF_TYPE_TRAITS constexpr bool is_nothrow_invocable_r_v = #endif // _HAS_CXX17 #if _HAS_CXX20 -#if !defined(__clang__) || defined(__EDG__) \ - || __clang_major__ >= 19 // TRANSITION, VSO-2397560 (RWC relying on ancient Clang) _EXPORT_STD template struct _NO_SPECIALIZATIONS_OF_TYPE_TRAITS is_layout_compatible : bool_constant<__is_layout_compatible(_Ty1, _Ty2)> {}; _EXPORT_STD template _NO_SPECIALIZATIONS_OF_TYPE_TRAITS constexpr bool is_layout_compatible_v = __is_layout_compatible(_Ty1, _Ty2); -#if !defined(__clang__) || defined(__EDG__) // TRANSITION, LLVM-135273 +#ifndef __clang__ // TRANSITION, LLVM-135273 _EXPORT_STD template struct _NO_SPECIALIZATIONS_OF_TYPE_TRAITS is_pointer_interconvertible_base_of : bool_constant<__is_pointer_interconvertible_base_of(_Base, _Derived)> {}; @@ -2039,7 +2036,6 @@ _EXPORT_STD template _NO_SPECIALIZATIONS_OF_TYPE_TRAITS constexpr bool is_pointer_interconvertible_base_of_v = __is_pointer_interconvertible_base_of(remove_cv_t<_Base>, remove_cv_t<_Derived>); #endif // ^^^ workaround ^^^ -#endif // ^^^ no workaround ^^^ #ifndef __clang__ // TRANSITION, LLVM-48860 _EXPORT_STD template diff --git a/stl/inc/xutility b/stl/inc/xutility index a2e18498b63..77d4a94bcfa 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -117,14 +117,6 @@ _STL_DISABLE_CLANG_WARNINGS #endif // _USE_STD_VECTOR_FLOATING_ALGORITHMS && !_USE_STD_VECTOR_ALGORITHMS #endif // ^^^ defined(_USE_STD_VECTOR_FLOATING_ALGORITHMS) ^^^ -#ifndef _USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE -#if defined(__clang__) && __clang_major__ >= 19 -#define _USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE 1 -#else // ^^^ defined(__clang__) && __clang_major__ >= 19 / !defined(__clang__) || __clang_major__ < 19 vvv -#define _USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE 0 -#endif // ^^^ !defined(__clang__) || __clang_major__ < 19 ^^^ -#endif // ^^^ !defined(_USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE) ^^^ - extern "C" { // The "noalias" attribute tells the compiler optimizer that pointers going into these hand-vectorized algorithms // won't be stored beyond the lifetime of the function, and that the function will only reference arrays denoted by @@ -5673,14 +5665,14 @@ inline constexpr bool _Can_memcmp_elements = true; template constexpr bool _Can_memcmp_elements<_Ty1*, _Ty2*, false> = _Is_pointer_address_comparable<_Ty1, _Ty2>; -#if _USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE +#ifdef __clang__ template constexpr bool _Is_same_and_builtin_trivially_equality_comparable = is_same_v<_Elem1, _Elem2> && __is_trivially_equality_comparable(_Elem1); -#else // ^^^ _USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE / !_USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE vvv +#else // ^^^ defined(__clang__) / !defined(__clang__) vvv template constexpr bool _Is_same_and_builtin_trivially_equality_comparable = false; -#endif // ^^^ !_USE_BUILTIN_IS_TRIVIALLY_EQUALITY_COMPARABLE ^^^ +#endif // ^^^ !defined(__clang__) ^^^ template constexpr bool _Can_memcmp_elements<_Elem1, _Elem2, false> = diff --git a/stl/inc/yvals.h b/stl/inc/yvals.h index e060f5ae8e6..97361f553af 100644 --- a/stl/inc/yvals.h +++ b/stl/inc/yvals.h @@ -261,8 +261,6 @@ _EMIT_STL_ERROR(STL1008, "_STL_CALL_ABORT_INSTEAD_OF_INVALID_PARAMETER has been #ifndef _MSVC_STL_DOOM_FUNCTION #ifdef _MSVC_STL_USE_ABORT_AS_DOOM_FUNCTION // The user wants to use abort(): #define _MSVC_STL_DOOM_FUNCTION(mesg) _CSTD abort() -#elif defined(__clang__) && __clang_major__ < 19 // TRANSITION, VSO-2397560, Real World Code relying on ancient Clang -#define _MSVC_STL_DOOM_FUNCTION(mesg) __builtin_trap() #elif defined(__clang__) // Use the Clang intrinsic: #define _MSVC_STL_DOOM_FUNCTION(mesg) __builtin_verbose_trap("MSVC STL error", mesg) #elif defined(_M_CEE) // TRANSITION, VSO-2457624 (/clr silent bad codegen for __fastfail); /clr:pure lacks __fastfail diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index fcb8716af04..7301d3fda1e 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -1951,9 +1951,7 @@ _EMIT_STL_ERROR(STL1013, "The STL doesn't support /RTCc because it rejects confo #define _STL_INTERNAL_STATIC_ASSERT(...) #endif // ^^^ !defined(_ENABLE_STL_INTERNAL_CHECK) ^^^ -#if defined(__CUDACC__) || (defined(__clang__) && __clang_major__ < 16) -// TRANSITION, CUDA 12.4 doesn't have downlevel support for static call operators. -// TRANSITION, VSO-2397560, temporary workaround for Real World Code relying on ancient Clang versions. +#ifdef __CUDACC__ // TRANSITION, CUDA 12.4 doesn't have downlevel support for static call operators. #define _STATIC_CALL_OPERATOR #define _CONST_CALL_OPERATOR const #else // ^^^ workaround / no workaround vvv From c865e85d77b9c802359212e8371c65a2f9d44722 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 15 Jan 2026 10:50:28 -0800 Subject: [PATCH 18/19] Update llvm-project (#6006) --- llvm-project | 2 +- stl/inc/ranges | 142 +++++++++++++++--------------- tests/libcxx/expected_results.txt | 44 ++++++--- 3 files changed, 107 insertions(+), 81 deletions(-) diff --git a/llvm-project b/llvm-project index 27651133e21..5d501d1a8d2 160000 --- a/llvm-project +++ b/llvm-project @@ -1 +1 @@ -Subproject commit 27651133e213a6a1eb4d0e47837625cee3613111 +Subproject commit 5d501d1a8d287b9c410aa46ffb5534859e518a8a diff --git a/stl/inc/ranges b/stl/inc/ranges index 81e7faf436d..b8bd6e0493f 100644 --- a/stl/inc/ranges +++ b/stl/inc/ranges @@ -5014,17 +5014,19 @@ namespace ranges { return _Get_current(_Left) == _Right._Last; } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for, _Maybe_const_iter<_OtherConst>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> operator-( + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-( const _Iterator<_OtherConst>& _Left, const _Sentinel& _Right) noexcept(noexcept(_Get_current(_Left) - _Right._Last)) /* strengthened */ { return _Get_current(_Left) - _Right._Last; } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for, _Maybe_const_iter<_OtherConst>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> operator-( + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-( const _Sentinel& _Left, const _Iterator<_OtherConst>& _Right) noexcept(noexcept(_Left._Last - _Get_current(_Right))) /* strengthened */ { return _Left._Last - _Get_current(_Right); @@ -5327,7 +5329,7 @@ namespace ranges { template requires sized_sentinel_for<_Base_sentinel, iterator_t<_Maybe_const<_OtherConst, _Vw>>> - _NODISCARD constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> _Distance_from( + _NODISCARD constexpr _Iterator<_OtherConst>::difference_type _Distance_from( const _Iterator<_OtherConst>& _It) const noexcept(noexcept(_End - _It._Current)) { return _End - _It._Current; } @@ -5352,19 +5354,20 @@ namespace ranges { return _Se._Equal(_It); } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Base_sentinel, iterator_t<_Maybe_const<_OtherConst, _Vw>>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> operator-( - const _Iterator<_OtherConst>& _It, const _Sentinel& _Se) // + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-( + const _Iterator<_OtherConst>& _It, const _Sentinel& _Se) noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { return -_Se._Distance_from(_It); } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Base_sentinel, iterator_t<_Maybe_const<_OtherConst, _Vw>>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> operator-( - const _Sentinel& _Se, const _Iterator<_OtherConst>& _It) // - noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-(const _Sentinel& _Se, + const _Iterator<_OtherConst>& _It) noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { return _Se._Distance_from(_It); } }; @@ -7397,6 +7400,9 @@ namespace ranges { private: friend zip_view; + template + using _Difference_type = _Iterator<_IteratorConst>::difference_type; + using _My_tuple = tuple>...>; /* [[no_unique_address]] */ _My_tuple _End; @@ -7435,22 +7441,20 @@ namespace ranges { requires (sized_sentinel_for>, iterator_t<_Maybe_const<_IteratorConst, _ViewTypes>>> && ...) - _NODISCARD friend constexpr common_type_t>...> - operator-(const _Iterator<_IteratorConst>& _Lhs, const _Sentinel& _Rhs) - noexcept(noexcept(_Zip_get_smallest_distance< - common_type_t>...>>( - _Get_iterator_tuple(_Lhs), _Rhs._End))) /* strengthened */ { - using _Difference_type = common_type_t>...>; - return _Zip_get_smallest_distance<_Difference_type>(_Get_iterator_tuple(_Lhs), _Rhs._End); + _NODISCARD friend constexpr _Difference_type<_IteratorConst> operator-( + const _Iterator<_IteratorConst>& _Lhs, const _Sentinel& _Rhs) + noexcept(noexcept(_Zip_get_smallest_distance<_Difference_type<_IteratorConst>>( + _Get_iterator_tuple(_Lhs), _Rhs._End))) /* strengthened */ { + return _Zip_get_smallest_distance<_Difference_type<_IteratorConst>>( + _Get_iterator_tuple(_Lhs), _Rhs._End); } template requires (sized_sentinel_for>, iterator_t<_Maybe_const<_IteratorConst, _ViewTypes>>> && ...) - _NODISCARD friend constexpr common_type_t>...> - operator-(const _Sentinel& _Lhs, const _Iterator<_IteratorConst>& _Rhs) - noexcept(noexcept(-(_Rhs - _Lhs))) /* strengthened */ { + _NODISCARD friend constexpr _Difference_type<_IteratorConst> operator-(const _Sentinel& _Lhs, + const _Iterator<_IteratorConst>& _Rhs) noexcept(noexcept(-(_Rhs - _Lhs))) /* strengthened */ { return -(_Rhs - _Lhs); } }; @@ -7781,10 +7785,6 @@ namespace ranges { private: friend zip_transform_view; - // TRANSITION, DevCom-10253131 - template - using _Iter_sent_difference_type = range_difference_t<_Maybe_const<_IteratorConst, _Inner_view>>; - /* [[no_unique_address]] */ _Zentinel<_IsConst> _Inner; constexpr explicit _Sentinel(_Zentinel<_IsConst> _Inner_) @@ -7812,19 +7812,21 @@ namespace ranges { return _Get_iterator_inner(_Lhs) == _Rhs._Inner; } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Zentinel<_IsConst>, _Ziperator<_IteratorConst>> - _NODISCARD friend constexpr _Iter_sent_difference_type<_IteratorConst> // TRANSITION, DevCom-10253131 - operator-(const _Iterator<_IteratorConst>& _Lhs, const _Sentinel& _Rhs) - noexcept(noexcept(_Get_iterator_inner(_Lhs) - _Rhs._Inner)) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_IteratorConst>::difference_type operator-( + const _Iterator<_IteratorConst>& _Lhs, const _Sentinel& _Rhs) + noexcept(noexcept(_Get_iterator_inner(_Lhs) - _Rhs._Inner)) /* strengthened */ { return _Get_iterator_inner(_Lhs) - _Rhs._Inner; } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Zentinel<_IsConst>, _Ziperator<_IteratorConst>> - _NODISCARD friend constexpr _Iter_sent_difference_type<_IteratorConst> // TRANSITION, DevCom-10253131 - operator-(const _Sentinel& _Lhs, const _Iterator<_IteratorConst>& _Rhs) - noexcept(noexcept(_Lhs._Inner - _Get_iterator_inner(_Rhs))) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_IteratorConst>::difference_type operator-( + const _Sentinel& _Lhs, const _Iterator<_IteratorConst>& _Rhs) + noexcept(noexcept(_Lhs._Inner - _Get_iterator_inner(_Rhs))) /* strengthened */ { return _Lhs._Inner - _Get_iterator_inner(_Rhs); } }; @@ -8238,7 +8240,7 @@ namespace ranges { template requires sized_sentinel_for<_Base_sentinel, iterator_t<_Maybe_const<_OtherConst, _Vw>>> - _NODISCARD constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> _Distance_from( + _NODISCARD constexpr _Iterator<_OtherConst>::difference_type _Distance_from( const _Iterator<_OtherConst>& _It) const noexcept(noexcept(_End - _It._Current.back())) { return _End - _It._Current.back(); } @@ -8258,19 +8260,20 @@ namespace ranges { return _Se._Equal(_It); } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Base_sentinel, iterator_t<_Maybe_const<_OtherConst, _Vw>>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> operator-( - const _Iterator<_OtherConst>& _It, const _Sentinel& _Se) noexcept( // - noexcept(_Se._Distance_from(_It))) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-( + const _Iterator<_OtherConst>& _It, const _Sentinel& _Se) + noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { return -_Se._Distance_from(_It); } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Base_sentinel, iterator_t<_Maybe_const<_OtherConst, _Vw>>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Vw>> operator-( - const _Sentinel& _Se, const _Iterator<_OtherConst>& _It) noexcept( // - noexcept(_Se._Distance_from(_It))) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-(const _Sentinel& _Se, + const _Iterator<_OtherConst>& _It) noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { return _Se._Distance_from(_It); } }; @@ -8602,7 +8605,7 @@ namespace ranges { template requires sized_sentinel_for<_Inner_sentinel<_Const>, _Inner_iterator<_OtherConst>> - _NODISCARD constexpr range_difference_t<_Maybe_const<_OtherConst, _Inner_view>> _Distance_from( + _NODISCARD constexpr _Iterator<_OtherConst>::difference_type _Distance_from( const _Iterator<_OtherConst>& _It) const noexcept(noexcept(_Inner - _It._Inner)) { return _Inner - _It._Inner; } @@ -8621,19 +8624,20 @@ namespace ranges { return _Se._Equal(_It); } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Inner_sentinel<_Const>, _Inner_iterator<_OtherConst>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Inner_view>> operator-( - const _Iterator<_OtherConst>& _It, const _Sentinel& _Se) noexcept( // - noexcept(_Se._Distance_from(_It))) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-( + const _Iterator<_OtherConst>& _It, const _Sentinel& _Se) + noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { return -_Se._Distance_from(_It); } + // typename is a workaround for VSO-2680018 (EDG) template requires sized_sentinel_for<_Inner_sentinel<_Const>, _Inner_iterator<_OtherConst>> - _NODISCARD friend constexpr range_difference_t<_Maybe_const<_OtherConst, _Inner_view>> operator-( - const _Sentinel& _Se, const _Iterator<_OtherConst>& _It) noexcept( // - noexcept(_Se._Distance_from(_It))) /* strengthened */ { + _NODISCARD friend constexpr typename _Iterator<_OtherConst>::difference_type operator-(const _Sentinel& _Se, + const _Iterator<_OtherConst>& _It) noexcept(noexcept(_Se._Distance_from(_It))) /* strengthened */ { return _Se._Distance_from(_It); } }; @@ -8826,6 +8830,21 @@ namespace ranges { template class _Iterator { + public: + using iterator_category = input_iterator_tag; + + using iterator_concept = conditional_t<_Cartesian_product_is_random_access<_Const, _First, _Rest...>, + random_access_iterator_tag, + conditional_t<_Cartesian_product_is_bidirectional<_Const, _First, _Rest...>, bidirectional_iterator_tag, + conditional_t>, forward_iterator_tag, + input_iterator_tag>>>; + + using value_type = + tuple>, range_value_t<_Maybe_const<_Const, _Rest>>...>; + using reference = tuple>, + range_reference_t<_Maybe_const<_Const, _Rest>>...>; + using difference_type = _Difference_type<_Const>; + private: friend cartesian_product_view; using _Parent_t = _Maybe_const<_Const, cartesian_product_view>; @@ -8866,7 +8885,7 @@ namespace ranges { } template - constexpr void _Advance(const _Difference_type<_Const> _Off) { + constexpr void _Advance(const difference_type _Off) { if (_Off == 0) { return; } @@ -8874,14 +8893,13 @@ namespace ranges { auto& _Range = _STD get<_Index>(_Parent->_Bases); auto& _It = _STD get<_Index>(_Current); using _Iter = remove_reference_t; - using _Diff = _Difference_type<_Const>; if constexpr (_Index > 0) { - const auto _Size = static_cast<_Diff>(_RANGES ssize(_Range)); + const auto _Size = static_cast(_RANGES ssize(_Range)); const auto _Begin = _RANGES begin(_Range); - const auto _It_off = static_cast<_Diff>(_It - _Begin); - const auto _It_new_off = static_cast<_Diff>((_It_off + _Off) % _Size); - const auto _Next_off = static_cast<_Diff>((_It_off + _Off) / _Size); + const auto _It_off = static_cast(_It - _Begin); + const auto _It_new_off = static_cast((_It_off + _Off) % _Size); + const auto _Next_off = static_cast((_It_off + _Off) / _Size); if (_It_new_off < 0) { _It = _Begin + static_cast>(_It_new_off + _Size); _Advance<_Index - 1>(_Next_off - 1); @@ -8891,11 +8909,11 @@ namespace ranges { } } else { #if _ITERATOR_DEBUG_LEVEL != 0 - const auto _It_off = static_cast<_Diff>(_It - _RANGES begin(_Range)); + const auto _It_off = static_cast(_It - _RANGES begin(_Range)); _STL_VERIFY(_It_off + _Off >= 0, "Cannot advance cartesian_product_view iterator before begin " "(N4950 [range.cartesian.iterator]/19)."); if constexpr (sized_range) { - const auto _Size = static_cast<_Diff>(_RANGES ssize(_Range)); + const auto _Size = static_cast(_RANGES ssize(_Range)); _STL_VERIFY(_It_off + _Off < _Size || (_It_off + _Off == _Size && _Entire_tail_at_begin(make_index_sequence{})), @@ -8920,13 +8938,11 @@ namespace ranges { } template - _NODISCARD constexpr _Difference_type<_Const> _Distance_from(const _Tuple& _Tpl) const { - const auto _Diff = - static_cast<_Difference_type<_Const>>(_STD get<_Index>(_Current) - _STD get<_Index>(_Tpl)); + _NODISCARD constexpr difference_type _Distance_from(const _Tuple& _Tpl) const { + const auto _Diff = static_cast(_STD get<_Index>(_Current) - _STD get<_Index>(_Tpl)); if constexpr (_Index > 0) { - _Difference_type<_Const> _Result{1}; - const auto _Size = - static_cast<_Difference_type<_Const>>(_RANGES size(_STD get<_Index>(_Parent->_Bases))); + difference_type _Result{1}; + const auto _Size = static_cast(_RANGES size(_STD get<_Index>(_Parent->_Bases))); [[maybe_unused]] const bool _Overflow = _Mul_overflow(_Size, _Distance_from<_Index - 1>(_Tpl), _Result) || _Add_overflow(_Result, _Diff, _Result); @@ -8961,20 +8977,6 @@ namespace ranges { : _Parent(_STD addressof(_Parent_)), _Current(_STD move(_Current_)) {} public: - using iterator_category = input_iterator_tag; - - using iterator_concept = conditional_t<_Cartesian_product_is_random_access<_Const, _First, _Rest...>, - random_access_iterator_tag, - conditional_t<_Cartesian_product_is_bidirectional<_Const, _First, _Rest...>, bidirectional_iterator_tag, - conditional_t>, forward_iterator_tag, - input_iterator_tag>>>; - - using value_type = - tuple>, range_value_t<_Maybe_const<_Const, _Rest>>...>; - using reference = tuple>, - range_reference_t<_Maybe_const<_Const, _Rest>>...>; - using difference_type = _Difference_type<_Const>; - _Iterator() = default; constexpr _Iterator(_Iterator _It) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 2414b59289f..75d9b248ee3 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -61,6 +61,12 @@ std/language.support/support.exception/except.nested/ctor_default.pass.cpp:2 SKI # SKIPPED because this is ARM64EC-specific. std/language.support/support.coroutines/coroutine.handle/coroutine.handle.noop/noop_coroutine.pass.cpp:2 SKIPPED +# LLVM-171438: [Clang] __builtin_common_type is overly permissive +# These tests are improperly working around this Clang bug by detecting Clang and asserting buggy results, +# but only libc++ with Clang is affected; MSVC's STL with Clang is unaffected. +std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp:2 FAIL +std/language.support/cmp/cmp.concept/three_way_comparable_with.compile.pass.cpp:2 FAIL + # Non-Standard regex behavior. # "It seems likely that the test is still non-conforming due to how libc++ handles the 'w' character class." std/re/re.traits/lookup_classname.pass.cpp FAIL @@ -111,16 +117,6 @@ std/ranges/range.access/end.pass.cpp FAIL std/ranges/range.access/rbegin.pass.cpp FAIL std/ranges/range.access/rend.pass.cpp FAIL -# libc++ has not implemented P2404R3: "Move-Only Types For Comparison Concepts" -std/algorithms/alg.sorting/alg.sort/partial.sort.copy/ranges_partial_sort_copy.pass.cpp FAIL -std/language.support/support.limits/support.limits.general/concepts.version.compile.pass.cpp FAIL -std/utilities/function.objects/range.cmp/equal_to.pass.cpp FAIL -std/utilities/function.objects/range.cmp/greater.pass.cpp FAIL -std/utilities/function.objects/range.cmp/greater_equal.pass.cpp FAIL -std/utilities/function.objects/range.cmp/less.pass.cpp FAIL -std/utilities/function.objects/range.cmp/less_equal.pass.cpp FAIL -std/utilities/function.objects/range.cmp/not_equal_to.pass.cpp FAIL - # libc++ has not implemented P2407R5: "Freestanding Library: Partial Classes" std/language.support/support.limits/support.limits.general/array.version.compile.pass.cpp FAIL std/language.support/support.limits/support.limits.general/optional.version.compile.pass.cpp FAIL @@ -309,12 +305,14 @@ std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.p std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_rv.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_sorted_unique.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_rv.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_initializer_list.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_sorted_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_transparent.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/replace.pass.cpp FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_exception.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_free.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_member.pass.cpp FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/try_emplace_transparent.pass.cpp FAIL @@ -335,6 +333,7 @@ std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound_trans std/containers/container.adaptors/flat.map/flat.map.operations/upper_bound.pass.cpp FAIL std/containers/container.adaptors/flat.map/incomplete_type.pass.cpp FAIL std/containers/container.adaptors/flat.map/op_compare.pass.cpp FAIL +std/containers/container.adaptors/flat.map/robust_against_nonbool.compile.pass.cpp FAIL std/containers/container.adaptors/flat.map/types.compile.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/empty.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.capacity/max_size.pass.cpp FAIL @@ -387,12 +386,14 @@ std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_i std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_cv.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_rv.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range_sorted_equivalent.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_rv.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_initializer_list.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_sorted_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_transparent.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/replace.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_exception.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_free.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_member.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.observers/comp.pass.cpp FAIL @@ -411,6 +412,7 @@ std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_b std/containers/container.adaptors/flat.multimap/flat.multimap.operations/upper_bound.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/incomplete_type.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/op_compare.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/robust_against_nonbool.compile.pass.cpp FAIL std/containers/container.adaptors/flat.multimap/types.compile.pass.cpp FAIL std/language.support/support.limits/support.limits.general/flat_map.version.compile.pass.cpp FAIL std/utilities/format/format.formattable/concept.formattable.compile.pass.cpp FAIL @@ -480,11 +482,13 @@ std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_i std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_cv.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_rv.pass.cpp FAIL +std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_range_sorted_equivalent.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_range.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_rv.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_sorted_initializer_list.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_sorted_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/replace.pass.cpp FAIL +std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/swap_exception.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/swap_free.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/swap_member.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.observers/comp.pass.cpp FAIL @@ -502,6 +506,7 @@ std/containers/container.adaptors/flat.multiset/flat.multiset.operations/upper_b std/containers/container.adaptors/flat.multiset/flat.multiset.operations/upper_bound.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/incomplete_type.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/op_compare.pass.cpp FAIL +std/containers/container.adaptors/flat.multiset/robust_against_nonbool.compile.pass.cpp FAIL std/containers/container.adaptors/flat.multiset/types.compile.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.capacity/empty.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.capacity/max_size.pass.cpp FAIL @@ -548,12 +553,14 @@ std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_initializer std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_cv.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_rv.pass.cpp FAIL +std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range_sorted_unique.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_rv.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_initializer_list.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_sorted_iter_iter.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/replace.pass.cpp FAIL +std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_exception.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp FAIL std/containers/container.adaptors/flat.set/flat.set.observers/comp.pass.cpp FAIL @@ -571,9 +578,15 @@ std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound_trans std/containers/container.adaptors/flat.set/flat.set.operations/upper_bound.pass.cpp FAIL std/containers/container.adaptors/flat.set/incomplete_type.pass.cpp FAIL std/containers/container.adaptors/flat.set/op_compare.pass.cpp FAIL +std/containers/container.adaptors/flat.set/robust_against_nonbool.compile.pass.cpp FAIL std/containers/container.adaptors/flat.set/types.compile.pass.cpp FAIL std/language.support/support.limits/support.limits.general/flat_set.version.compile.pass.cpp FAIL +# P1789R3 Library Support For Expansion Statements +# Add 'std-at-least-c++26' to tests/utils/stl/test/tests.py when beginning work on C++26. +std/utilities/intseq/intseq.binding/structured_binding.pass.cpp FAIL +std/utilities/intseq/intseq.binding/tuple_interface.compile.pass.cpp FAIL + # P2255R2 "Type Traits To Detect References Binding To Temporaries" # (type_traits.version.compile.pass.cpp still fails for Clang due to LLVM-48860) std/language.support/support.limits/support.limits.general/type_traits.version.compile.pass.cpp FAIL @@ -582,6 +595,10 @@ std/utilities/meta/meta.unary/meta.unary.prop/reference_constructs_from_temporar std/utilities/meta/meta.unary/meta.unary.prop/reference_converts_from_temporary.pass.cpp:0 FAIL std/utilities/meta/meta.unary/meta.unary.prop/reference_converts_from_temporary.pass.cpp:1 FAIL +# P2592R3 Hashing Support For chrono Value Classes +# Add 'std-at-least-c++26' to tests/utils/stl/test/tests.py when beginning work on C++26. +std/time/time.hash/time.hash_enabled.pass.cpp FAIL + # P2835R7 atomic_ref::address() # Add 'std-at-least-c++26' to tests/utils/stl/test/tests.py when beginning work on C++26. std/atomics/atomics.ref/address.pass.cpp FAIL @@ -725,6 +742,10 @@ std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer_delet std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.ctor/pointer_deleter.pass.cpp:1 FAIL # VSO-2574473 constexpr error in ranges::join_with_view: failure was caused by unevaluable pointer value +std/ranges/range.adaptors/range.adjacent.transform/general.pass.cpp:0 FAIL +std/ranges/range.adaptors/range.adjacent.transform/general.pass.cpp:1 FAIL +std/ranges/range.adaptors/range.adjacent/general.pass.cpp:0 FAIL +std/ranges/range.adaptors/range.adjacent/general.pass.cpp:1 FAIL std/ranges/range.adaptors/range.join.with/range.join.with.iterator/ctor.not_const.pass.cpp:0 FAIL std/ranges/range.adaptors/range.join.with/range.join.with.iterator/ctor.not_const.pass.cpp:1 FAIL @@ -979,6 +1000,8 @@ std/re/re.regex/re.regex.swap/swap.pass.cpp FAIL # Test emits a Clang -Wself-move warning, so it should avoid the self-move or suppress the warning. std/containers/sequences/vector/addressof.compile.pass.cpp:2 FAIL +std/thread/thread.mutex/thread.lock/thread.lock.shared/thread.lock.shared.cons/move_assign.pass.cpp:2 FAIL +std/thread/thread.mutex/thread.lock/thread.lock.unique/thread.lock.unique.cons/move_assign.pass.cpp:2 FAIL # throwing_allocator appears to be totally non-Standard. std/containers/sequences/vector.bool/ctor_exceptions.pass.cpp FAIL @@ -1219,6 +1242,7 @@ std/algorithms/alg.modifying.operations/alg.swap/ranges.swap_ranges.pass.cpp FAI std/algorithms/alg.nonmodifying/alg.contains/ranges.contains_subrange.pass.cpp:2 FAIL std/algorithms/alg.nonmodifying/alg.count/count.pass.cpp FAIL std/algorithms/alg.nonmodifying/alg.count/ranges.count.pass.cpp FAIL +std/algorithms/alg.sorting/alg.sort/partial.sort.copy/ranges_partial_sort_copy.pass.cpp:2 FAIL std/containers/sequences/vector.bool/append_range.pass.cpp FAIL std/containers/sequences/vector.bool/assign_range.pass.cpp FAIL std/containers/sequences/vector.bool/flip.pass.cpp FAIL From fd076f6c8cf48eb4c8049c779072134eacfcbaf6 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 15 Jan 2026 11:51:19 -0800 Subject: [PATCH 19/19] Update libcxx failures. --- tests/libcxx/expected_results.txt | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 5b5e8aa0b7e..52705a56c91 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -67,6 +67,9 @@ std/language.support/support.coroutines/coroutine.handle/coroutine.handle.noop/n std/concepts/concepts.compare/concept.equalitycomparable/equality_comparable_with.compile.pass.cpp:2 FAIL std/language.support/cmp/cmp.concept/three_way_comparable_with.compile.pass.cpp:2 FAIL +# LLVM-176232: The implementation of P3567R2 "flat_meow Fixes" forgot to update `__cpp_lib_flat_set` +std/language.support/support.limits/support.limits.general/flat_set.version.compile.pass.cpp FAIL + # Non-Standard regex behavior. # "It seems likely that the test is still non-conforming due to how libc++ handles the 'w' character class." std/re/re.traits/lookup_classname.pass.cpp FAIL @@ -142,18 +145,6 @@ std/atomics/atomics.ref/member_types.compile.pass.cpp FAIL std/thread/futures/futures.promise/uses_allocator.pass.cpp FAIL std/thread/futures/futures.task/futures.task.members/ctor2.compile.pass.cpp FAIL -# libc++ has not implemented P3567R2: "flat_meow Fixes" -std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_free.pass.cpp FAIL -std/containers/container.adaptors/flat.map/flat.map.modifiers/swap_member.pass.cpp FAIL -std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_free.pass.cpp FAIL -std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/swap_member.pass.cpp FAIL -std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/swap_free.pass.cpp FAIL -std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/swap_member.pass.cpp FAIL -std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_free.pass.cpp FAIL -std/containers/container.adaptors/flat.set/flat.set.modifiers/swap_member.pass.cpp FAIL -std/language.support/support.limits/support.limits.general/flat_map.version.compile.pass.cpp FAIL -std/language.support/support.limits/support.limits.general/flat_set.version.compile.pass.cpp FAIL - # Various bogosity (LLVM-D141004) std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.ctor/ctor_does_not_allocate.pass.cpp FAIL std/utilities/utility/mem.res/mem.res.pool/mem.res.pool.ctor/sync_with_default_resource.pass.cpp FAIL @@ -344,6 +335,8 @@ std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.p std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_iter_iter.pass.cpp:1 FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp:0 FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_or_assign_transparent.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_sorted_unique.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range_sorted_unique.pass.cpp:1 FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp:0 FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_range.pass.cpp:1 FAIL std/containers/container.adaptors/flat.map/flat.map.modifiers/insert_rv.pass.cpp:0 FAIL @@ -378,6 +371,8 @@ std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/extract. std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/extract.pass.cpp:1 FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp:0 FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range_sorted_equivalent.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range_sorted_equivalent.pass.cpp:1 FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp:0 FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_range.pass.cpp:1 FAIL std/containers/container.adaptors/flat.multimap/flat.multimap.modifiers/insert_rv.pass.cpp:0 FAIL @@ -406,6 +401,8 @@ std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/extract. std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/extract.pass.cpp:1 FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp:0 FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_range_sorted_equivalent.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_range_sorted_equivalent.pass.cpp:1 FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_range.pass.cpp:0 FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.modifiers/insert_range.pass.cpp:1 FAIL std/containers/container.adaptors/flat.multiset/flat.multiset.observers/comp.pass.cpp:0 FAIL @@ -432,6 +429,8 @@ std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp:0 std/containers/container.adaptors/flat.set/flat.set.modifiers/extract.pass.cpp:1 FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp:0 FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_iter_iter.pass.cpp:1 FAIL +std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range_sorted_unique.pass.cpp:0 FAIL +std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range_sorted_unique.pass.cpp:1 FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp:0 FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_range.pass.cpp:1 FAIL std/containers/container.adaptors/flat.set/flat.set.modifiers/insert_transparent.pass.cpp:0 FAIL @@ -1361,6 +1360,10 @@ std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp:1 FAIL std/input.output/file.streams/fstreams/filebuf.virtuals/xsputn.pass.cpp:0 FAIL std/input.output/file.streams/fstreams/filebuf.virtuals/xsputn.pass.cpp:1 FAIL +# FIXME, not analyzed. _Flat_map_base::_Erase_if's usage of ranges::remove_if is failing to compile. +std/containers/container.adaptors/flat.map/robust_against_nonbool.compile.pass.cpp FAIL +std/containers/container.adaptors/flat.multimap/robust_against_nonbool.compile.pass.cpp FAIL + # *** XFAILS WHICH PASS *** # These tests contain `// XFAIL: msvc` comments, which accurately describe runtime failures for x86 and x64.