From 365b13deba101eec410d869c6ce8b182fe2b7bf8 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 27 Dec 2025 23:24:36 +0200 Subject: [PATCH 01/12] coverage --- .../test.cpp | 10 +- .../VSO_0000000_vector_algorithms/test.cpp | 103 ++++++++++++++---- 2 files changed, 91 insertions(+), 22 deletions(-) diff --git a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp index c2b53e59f58..0a82d9bad27 100644 --- a/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp +++ b/tests/std/tests/GH_005421_vector_algorithms_integer_class_type_iterator/test.cpp @@ -221,11 +221,17 @@ int main() { assert(r_rot_it == temp_end - rotate_pos); } { - // Out of replace family, only replace for 32-bit and 64-bit elements is manually vectorized, - // replace_copy is auto vectorized (along with replace_copy_if) const int replace_expected[] = { 200, 210, 220, 333, 240, 333, 333, 270, 280, 290, 300, 310, 320, 333, 340, 333, 333, 370, 380, 390}; + auto repl_copy_it = replace_copy(arr_begin, arr_end, temp_begin, 250, 333); + assert(equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); + assert(repl_copy_it == temp_end); + + auto r_repl_copy_it = ranges::replace_copy(arr_begin, arr_end, temp_begin, 250, 333).out; + assert(ranges::equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); + assert(r_repl_copy_it == temp_end); + copy(arr_begin, arr_end, temp_begin); replace(temp_begin, temp_end, 250, 333); assert(equal(temp_begin, temp_end, begin(replace_expected), end(replace_expected))); diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index eeff29a4bfa..432c2e02b63 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -723,40 +723,98 @@ void last_known_good_replace(FwdIt first, FwdIt last, const T old_val, const T n } } +template +void last_known_good_replace_copy(FwdIt first, FwdIt last, OutIt dest, const T old_val, const T new_val) { + for (; first != last; ++first, ++dest) { + if (*first == old_val) { + *dest = new_val; + } else { + *dest = *first; + } + } +} + template -void test_case_replace(const vector& input, T old_val, T new_val) { - vector replaced_actual(input); - vector replaced_expected(input); - replace(replaced_actual.begin(), replaced_actual.end(), old_val, new_val); - last_known_good_replace(replaced_expected.begin(), replaced_expected.end(), old_val, new_val); - assert(replaced_expected == replaced_actual); +void test_case_replace(vector& in_out_expected, vector& in_out_actual, vector& in_out_actual_r, + const T old_val, const T new_val) { + replace(in_out_actual.begin(), in_out_actual.end(), old_val, new_val); + last_known_good_replace(in_out_expected.begin(), in_out_expected.end(), old_val, new_val); + assert(in_out_expected == in_out_actual); #if _HAS_CXX20 - vector replaced_actual_r(input); - ranges::replace(replaced_actual_r, old_val, new_val); - assert(replaced_expected == replaced_actual_r); -#endif // _HAS_CXX20 + ranges::replace(in_out_actual_r, old_val, new_val); + assert(in_out_expected == in_out_actual_r); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) in_out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^^ } template +void test_case_replace_copy(const vector& input, vector& out_expected, vector& out_actual, + vector& out_actual_r, const T old_val, const T new_val) { + + replace_copy(input.begin(), input.end(), out_actual.begin(), old_val, new_val); + last_known_good_replace_copy(input.begin(), input.end(), out_expected.begin(), old_val, new_val); + assert(out_expected == out_actual); + +#if _HAS_CXX20 + ranges::replace_copy(input, out_actual_r.begin(), old_val, new_val); + assert(out_expected == out_actual_r); +#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv + (void) out_actual_r; +#endif // ^^^ !_HAS_CXX20 ^^ +} + +template void test_replace(mt19937_64& gen) { using TD = conditional_t; uniform_int_distribution dis(0, 9); - vector input; - input.reserve(dataCount); + vector source; + vector out_expected; + vector out_actual; + vector out_actual_r; + vector in_out_expected; + vector in_out_actual; + vector in_out_actual_r; + + for (const auto& v : {&source, &out_expected, &out_actual, &out_actual_r}) { + v->reserve(dataCount); + } + + if constexpr (in_place) { + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + v->reserve(dataCount); + } + } { const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - test_case_replace(input, old_val, new_val); + + if constexpr (in_place) { + test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); + } + test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); } - for (size_t i = 0; i != dataCount; ++i) { - input.push_back(static_cast(dis(gen))); + for (size_t attempts = 0; attempts < dataCount; ++attempts) { + source.push_back(static_cast(dis(gen))); const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - test_case_replace(input, old_val, new_val); + + for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { + *v = source; + } + + for (const auto& v : {&out_expected, &out_actual, &out_actual_r}) { + v->assign(source.size(), T{0}); + } + + if constexpr (in_place) { + test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); + } + test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); } } @@ -1288,10 +1346,15 @@ void test_vector_algorithms(mt19937_64& gen) { #endif // _HAS_CXX17 // replace() is vectorized for 4 and 8 bytes only. - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); test_reverse(gen); test_reverse(gen); From ed50f3d57babef295ce9094f48accf7ef9cefbd8 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 27 Dec 2025 23:25:10 +0200 Subject: [PATCH 02/12] revert auto vectorization --- stl/inc/algorithm | 49 ++++++++++++----------------------------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 3fc1b239309..b3ee8d65e09 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -4562,15 +4562,6 @@ namespace ranges { } // namespace ranges #endif // _HAS_CXX20 -// TRANSITION, DevCom-10606350: help the compiler auto-vectorize for simple types -template > -constexpr bool _Can_vectorize_replace_copy = conjunction_v, is_same<_InTy, _NewTy>, - disjunction< -#ifdef __cpp_lib_byte - conjunction, is_same<_OutTy, byte>>, -#endif // defined(__cpp_lib_byte) - conjunction, is_integral<_OutTy>>, conjunction, is_pointer<_OutTy>>>>; - _EXPORT_STD template _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const _Ty& _Oldval, const _Ty& _Newval) { // copy replacing each matching _Oldval with _Newval @@ -4580,14 +4571,10 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { - if constexpr (_Can_vectorize_replace_copy, _Ty>) { - *_UDest = *_UFirst == _Oldval ? _Newval : *_UFirst; + if (*_UFirst == _Oldval) { + *_UDest = _Newval; } else { - if (*_UFirst == _Oldval) { - *_UDest = _Newval; - } else { - *_UDest = *_UFirst; - } + *_UDest = *_UFirst; } } @@ -4661,14 +4648,10 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); for (; _First != _Last; ++_First, (void) ++_Output) { - if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty2>) { - *_Output = _STD invoke(_Proj, *_First) == _Oldval ? _Newval : *_First; + if (_STD invoke(_Proj, *_First) == _Oldval) { + *_Output = _Newval; } else { - if (_STD invoke(_Proj, *_First) == _Oldval) { - *_Output = _Newval; - } else { - *_Output = *_First; - } + *_Output = *_First; } } @@ -4689,14 +4672,10 @@ _CONSTEXPR20 _OutIt replace_copy_if(_InIt _First, _InIt _Last, _OutIt _Dest, _Pr const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { - if constexpr (_Can_vectorize_replace_copy, _Ty>) { - *_UDest = _Pred(*_UFirst) ? _Val : *_UFirst; + if (_Pred(*_UFirst)) { + *_UDest = _Val; } else { - if (_Pred(*_UFirst)) { - *_UDest = _Val; - } else { - *_UDest = *_UFirst; - } + *_UDest = *_UFirst; } } @@ -4771,14 +4750,10 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); for (; _First != _Last; ++_First, (void) ++_Output) { - if constexpr (_Can_vectorize_replace_copy<_Out, iter_value_t<_It>, _Ty>) { - *_Output = _STD invoke(_Pred, _STD invoke(_Proj, *_First)) ? _Newval : *_First; + if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) { + *_Output = _Newval; } else { - if (_STD invoke(_Pred, _STD invoke(_Proj, *_First))) { - *_Output = _Newval; - } else { - *_Output = *_First; - } + *_Output = *_First; } } From a6f0a271b5ef910ca589b392b001720f377e501b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sat, 27 Dec 2025 23:33:50 +0200 Subject: [PATCH 03/12] manual vectorization --- stl/inc/algorithm | 95 +++++++++++++++++++++++++++++++++++ stl/inc/xutility | 1 + stl/src/vector_algorithms.cpp | 94 ++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index b3ee8d65e09..1a01df10f41 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -122,6 +122,17 @@ __declspec(noalias) void __stdcall __std_replace_8( void* _First, void* _Last, uint64_t _Old_val, uint64_t _New_val) noexcept; #endif // ^^^ _VECTORIZED_REPLACE ^^^ +#if _VECTORIZED_REPLACE_COPY +__declspec(noalias) void __stdcall __std_replace_copy_1( + const void* _First, const void* _Last, void* _Dest, uint8_t _Old_val, uint8_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_2( + const void* _First, const void* _Last, void* _Dest, uint16_t _Old_val, uint16_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_4( + const void* _First, const void* _Last, void* _Dest, uint32_t _Old_val, uint32_t _New_val) noexcept; +__declspec(noalias) void __stdcall __std_replace_copy_8( + const void* _First, const void* _Last, void* _Dest, uint64_t _Old_val, uint64_t _New_val) noexcept; +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_SEARCH_N const void* __stdcall __std_search_n_1(const void* _First, const void* _Last, size_t _Count, uint8_t _Value) noexcept; const void* __stdcall __std_search_n_2(const void* _First, const void* _Last, size_t _Count, uint16_t _Value) noexcept; @@ -358,6 +369,28 @@ __declspec(noalias) void _Replace_vectorized( } #endif // ^^^ _VECTORIZED_REPLACE ^^^ +#if _VECTORIZED_REPLACE_COPY +template +__declspec(noalias) void _Replace_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _Ty* _Dest, + const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { + if constexpr (sizeof(_Ty) == 1) { + ::__std_replace_copy_1( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 2) { + ::__std_replace_copy_2( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 4) { + ::__std_replace_copy_4( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else if constexpr (sizeof(_Ty) == 8) { + ::__std_replace_copy_8( + _First, _Last, _Dest, _STD _Find_arg_cast(_Old_val), _STD _Find_arg_cast(_New_val)); + } else { + static_assert(false, "unexpected size"); + } +} +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + #if _VECTORIZED_SEARCH_N template _Ty* _Search_n_vectorized(_Ty* const _First, _Ty* const _Last, const size_t _Count, const _TVal _Val) noexcept { @@ -476,6 +509,17 @@ constexpr bool _Output_iterator_for_vector_alg_is_safe() { } #endif // ^^^ _VECTORIZED_REMOVE_COPY || _VECTORIZED_UNIQUE_COPY ^^^ +#if _VECTORIZED_REPLACE_COPY +template +constexpr bool _Output_iterator_for_known_size_vector_alg_is_safe() { + if constexpr (_Iterator_is_contiguous<_Out>) { + return is_same_v<_Iter_value_t<_Out>, remove_const_t<_Iter_value_t<_In>>>; + } else { + return false; + } +} +#endif // ^^^ _VECTORIZED_REPLACE_COPY + #if _VECTORIZED_INCLUDES // Can we activate the vector algorithms for includes? template > @@ -4570,6 +4614,31 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const auto _UFirst = _STD _Get_unwrapped(_First); const auto _ULast = _STD _Get_unwrapped(_Last); auto _UDest = _STD _Get_unwrapped_n(_Dest, _STD _Idl_distance<_InIt>(_UFirst, _ULast)); + +#if _VECTORIZED_REPLACE_COPY + if constexpr (_Vector_alg_in_find_is_safe + && _Output_iterator_for_known_size_vector_alg_is_safe()) { + if (!_STD _Is_constant_evaluated()) { + const auto _First_ptr = _STD _To_address(_UFirst); + const auto _Dest_ptr = _STD _To_address(_UDest); + const auto _Last_ptr = _STD _To_address(_ULast); + const auto _Count = _ULast - _UFirst; + + _UDest += static_cast<_Iter_diff_t>(_Count); + (void) _STD _To_address(_UDest); + + if (_STD _Could_compare_equal_to_value_type(_Oldval)) { + _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Dest_ptr, _Oldval, _Newval); + } else { + _CSTD memcpy(_Dest_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Dest_ptr)); + } + + _STD _Seek_wrapped(_Dest, _UDest); + return _Dest; + } + } +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + for (; _UFirst != _ULast; ++_UFirst, (void) ++_UDest) { if (*_UFirst == _Oldval) { *_UDest = _Newval; @@ -4647,6 +4716,32 @@ namespace ranges { _STD _Verify_ranges_do_not_overlap(_First, _Last, _Output); +#if _VECTORIZED_REPLACE_COPY + if constexpr (is_same_v<_Pj, identity> && sized_sentinel_for<_Se, _It> + && _Vector_alg_in_find_is_safe<_It, _Ty1> && _Vector_alg_in_find_is_safe<_It, _Ty2> + && _Output_iterator_for_known_size_vector_alg_is_safe<_Out, _It>()) { + if (!_STD is_constant_evaluated()) { + const auto _Count = _Last - _First; + const auto _First_ptr = _STD to_address(_First); + const auto _Out_ptr = _STD to_address(_Output); + + _First += _Count; + const auto _Last_ptr = _STD to_address(_First); + + _Output += static_cast>(_Count); + (void) _STD to_address(_Output); + + if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { + _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Out_ptr, _Oldval, _Newval); + } else { + _CSTD memcpy(_Out_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Out_ptr)); + } + + return {_STD move(_First), _STD move(_Output)}; + } + } +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ + for (; _First != _Last; ++_First, (void) ++_Output) { if (_STD invoke(_Proj, *_First) == _Oldval) { *_Output = _Newval; diff --git a/stl/inc/xutility b/stl/inc/xutility index 53a64c6c787..f14bd773894 100644 --- a/stl/inc/xutility +++ b/stl/inc/xutility @@ -95,6 +95,7 @@ _STL_DISABLE_CLANG_WARNINGS #define _VECTORIZED_REMOVE _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REMOVE_COPY _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REPLACE _VECTORIZED_FOR_X64_X86 +#define _VECTORIZED_REPLACE_COPY _VECTORIZED_FOR_X64_X86 #define _VECTORIZED_REVERSE _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_REVERSE_COPY _VECTORIZED_FOR_X64_X86_ARM64 #define _VECTORIZED_ROTATE _VECTORIZED_FOR_X64_X86_ARM64 diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index cb8bbbdbca0..d534dbbeb6e 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -6486,6 +6486,80 @@ __declspec(noalias) size_t __stdcall __std_mismatch_8( return _Mismatching::_Mismatch_impl(_First1, _First2, _Count); } +} // extern "C" + +namespace { + namespace _Replacing { + template + void __stdcall _Replace_copy_impl( + const void* _First, const void* const _Last, void* _Dest, const _Ty _Old_val, const _Ty _New_val) noexcept { +#ifndef _M_ARM64EC + const size_t _Size_bytes = _Byte_length(_First, _Last); + + if (const size_t _Avx_size = _Size_bytes & ~size_t{0x1F}; _Avx_size != 0 && _Use_avx2()) { + const __m256i _Comparand = _Traits::_Set_avx(_Old_val); + const __m256i _Replacement = _Traits::_Set_avx(_New_val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Avx_size); + + do { + const __m256i _Data = _mm256_loadu_si256(static_cast(_First)); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + const __m256i _Val = _mm256_blendv_epi8(_Data, _Replacement, _Mask); + + _mm256_storeu_si256(static_cast<__m256i*>(_Dest), _Val); + + _Advance_bytes(_First, 32); + _Advance_bytes(_Dest, 32); + } while (_First != _Stop_at); + + if (const size_t _Avx_tail_size = _Size_bytes & 0x1C; _Avx_tail_size != 0) { + const __m256i _Tail_mask = _Avx2_tail_mask_32(_Avx_tail_size); + const __m256i _Data = _mm256_maskload_epi32(static_cast(_First), _Tail_mask); + const __m256i _Mask = _Traits::_Cmp_avx(_Data, _Comparand); + const __m256i _Val = _mm256_blendv_epi8(_Data, _Replacement, _Mask); + + _mm256_maskstore_epi32(static_cast(_Dest), _Tail_mask, _Val); + + _Advance_bytes(_First, _Avx_tail_size); + _Advance_bytes(_Dest, _Avx_tail_size); + } + + _mm256_zeroupper(); // TRANSITION, DevCom-10331414 + + if constexpr (sizeof(_Ty) >= 4) { + return; + } + } else if (const size_t _Sse_size = _Size_bytes & ~size_t{0xF}; _Sse_size != 0 && _Use_sse42()) { + const __m128i _Comparand = _Traits::_Set_sse(_Old_val); + const __m128i _Replacement = _Traits::_Set_sse(_New_val); + const void* _Stop_at = _First; + _Advance_bytes(_Stop_at, _Sse_size); + + do { + const __m128i _Data = _mm_loadu_si128(static_cast(_First)); + const __m128i _Mask = _Traits::_Cmp_sse(_Data, _Comparand); + const __m128i _Val = _mm_blendv_epi8(_Data, _Replacement, _Mask); + + _mm_storeu_si128(static_cast<__m128i*>(_Dest), _Val); + + _Advance_bytes(_First, 16); + _Advance_bytes(_Dest, 16); + } while (_First != _Stop_at); + } +#endif // ^^^ !defined(_M_ARM64EC) ^^^ + auto _Ptr_dest = static_cast<_Ty*>(_Dest); + for (auto _Ptr_src = static_cast(_First); _Ptr_src != _Last; ++_Ptr_src) { + const _Ty _Val = *_Ptr_src; + *_Ptr_dest = _Val == _Old_val ? _New_val : _Val; + ++_Ptr_dest; + } + } + } // namespace _Replacing +} // unnamed namespace + +extern "C" { + __declspec(noalias) void __stdcall __std_replace_4( void* _First, void* const _Last, const uint32_t _Old_val, const uint32_t _New_val) noexcept { #ifndef _M_ARM64EC @@ -6567,6 +6641,26 @@ __declspec(noalias) void __stdcall __std_replace_8( } } +__declspec(noalias) void __stdcall __std_replace_copy_1(const void* const _First, const void* const _Last, + void* const _Dest, const uint8_t _Old_val, const uint8_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_1>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_2(const void* const _First, const void* const _Last, + void* const _Dest, const uint16_t _Old_val, const uint16_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_2>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_4(const void* const _First, const void* const _Last, + void* const _Dest, const uint32_t _Old_val, const uint32_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_4>(_First, _Last, _Dest, _Old_val, _New_val); +} + +__declspec(noalias) void __stdcall __std_replace_copy_8(const void* const _First, const void* const _Last, + void* const _Dest, const uint64_t _Old_val, const uint64_t _New_val) noexcept { + _Replacing::_Replace_copy_impl<_Finding::_Find_traits_8>(_First, _Last, _Dest, _Old_val, _New_val); +} + } // extern "C" namespace { From 1fa9a37a5d00e64336c41625105ffc290c0d68b1 Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 28 Dec 2025 13:49:11 +0200 Subject: [PATCH 04/12] Don't test unsigned short for replace --- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index 432c2e02b63..b2e9d9d7806 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -1350,7 +1350,7 @@ void test_vector_algorithms(mt19937_64& gen) { test_replace(gen); test_replace(gen); test_replace(gen); - test_replace(gen); + test_replace(gen); test_replace(gen); test_replace(gen); test_replace(gen); From 7245eadca14882e32cfe302b90d160c8edc78a8b Mon Sep 17 00:00:00 2001 From: Alex Guteniev Date: Sun, 28 Dec 2025 13:51:54 +0200 Subject: [PATCH 05/12] no ifs --- benchmarks/src/replace.cpp | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/benchmarks/src/replace.cpp b/benchmarks/src/replace.cpp index ebb1aa038bf..28ab1c12ddb 100644 --- a/benchmarks/src/replace.cpp +++ b/benchmarks/src/replace.cpp @@ -34,19 +34,6 @@ void rc(benchmark::State& state) { } } -template -void rc_if(benchmark::State& state) { - std::vector> a(lorem_ipsum.begin(), lorem_ipsum.end()); - std::vector> b(lorem_ipsum.size()); - - for (auto _ : state) { - benchmark::DoNotOptimize(a); - (void) std::replace_copy_if( - std::begin(a), std::end(a), std::begin(b), [](auto x) { return x <= T{'Z'}; }, T{'X'}); - benchmark::DoNotOptimize(b); - } -} - // replace() is vectorized for 4 and 8 bytes only. BENCHMARK(r); BENCHMARK(r); @@ -56,9 +43,4 @@ BENCHMARK(rc); BENCHMARK(rc); BENCHMARK(rc); -BENCHMARK(rc_if); -BENCHMARK(rc_if); -BENCHMARK(rc_if); -BENCHMARK(rc_if); - BENCHMARK_MAIN(); From 49925b0e0168dad585cc93d61b1c13013ceeb755 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 12:39:59 -0800 Subject: [PATCH 06/12] Add const. --- stl/inc/algorithm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 1a01df10f41..3e0cea49b6d 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -371,7 +371,7 @@ __declspec(noalias) void _Replace_vectorized( #if _VECTORIZED_REPLACE_COPY template -__declspec(noalias) void _Replace_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _Ty* _Dest, +__declspec(noalias) void _Replace_copy_vectorized(const _Ty* const _First, const _Ty* const _Last, _Ty* const _Dest, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept { if constexpr (sizeof(_Ty) == 1) { ::__std_replace_copy_1( From 915238b0c7a2516c542bbb77f9fc21f9ec116319 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 12:42:17 -0800 Subject: [PATCH 07/12] Add arrows. --- stl/inc/algorithm | 2 +- tests/std/tests/VSO_0000000_vector_algorithms/test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 3e0cea49b6d..406a33090a2 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -518,7 +518,7 @@ constexpr bool _Output_iterator_for_known_size_vector_alg_is_safe() { return false; } } -#endif // ^^^ _VECTORIZED_REPLACE_COPY +#endif // ^^^ _VECTORIZED_REPLACE_COPY ^^^ #if _VECTORIZED_INCLUDES // Can we activate the vector algorithms for includes? diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index b2e9d9d7806..f48e5eaec03 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -762,7 +762,7 @@ void test_case_replace_copy(const vector& input, vector& out_expected, vec assert(out_expected == out_actual_r); #else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv (void) out_actual_r; -#endif // ^^^ !_HAS_CXX20 ^^ +#endif // ^^^ !_HAS_CXX20 ^^^ } template From 19783d020bba86731dd2a67d68c047006fcec103 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 12:55:40 -0800 Subject: [PATCH 08/12] Reorder: first, last, dest --- stl/inc/algorithm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 406a33090a2..9d4cb6c0460 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -4620,8 +4620,8 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const && _Output_iterator_for_known_size_vector_alg_is_safe()) { if (!_STD _Is_constant_evaluated()) { const auto _First_ptr = _STD _To_address(_UFirst); - const auto _Dest_ptr = _STD _To_address(_UDest); const auto _Last_ptr = _STD _To_address(_ULast); + const auto _Dest_ptr = _STD _To_address(_UDest); const auto _Count = _ULast - _UFirst; _UDest += static_cast<_Iter_diff_t>(_Count); From ea5ec086dd1fd07baeafe6d82a48b3b591a153c9 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 13:12:26 -0800 Subject: [PATCH 09/12] Use `_Contiguous_iter_verify`, following vectorized `reverse_copy`. --- stl/inc/algorithm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 9d4cb6c0460..51ba29d2d8e 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -4619,13 +4619,12 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const if constexpr (_Vector_alg_in_find_is_safe && _Output_iterator_for_known_size_vector_alg_is_safe()) { if (!_STD _Is_constant_evaluated()) { + const auto _Count = static_cast<_Iter_diff_t>(_ULast - _UFirst); + _STD _Contiguous_iter_verify(_UDest, _Count); + const auto _First_ptr = _STD _To_address(_UFirst); const auto _Last_ptr = _STD _To_address(_ULast); const auto _Dest_ptr = _STD _To_address(_UDest); - const auto _Count = _ULast - _UFirst; - - _UDest += static_cast<_Iter_diff_t>(_Count); - (void) _STD _To_address(_UDest); if (_STD _Could_compare_equal_to_value_type(_Oldval)) { _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Dest_ptr, _Oldval, _Newval); @@ -4633,6 +4632,7 @@ _CONSTEXPR20 _OutIt replace_copy(_InIt _First, _InIt _Last, _OutIt _Dest, const _CSTD memcpy(_Dest_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Dest_ptr)); } + _UDest += _Count; _STD _Seek_wrapped(_Dest, _UDest); return _Dest; } From e8f994668ff58f46172990c075dc5dacc483b2d4 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 13:53:46 -0800 Subject: [PATCH 10/12] And use `_Contiguous_iter_verify` for the range algorithm. --- stl/inc/algorithm | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/stl/inc/algorithm b/stl/inc/algorithm index 51ba29d2d8e..4d7f0a23852 100644 --- a/stl/inc/algorithm +++ b/stl/inc/algorithm @@ -4721,22 +4721,22 @@ namespace ranges { && _Vector_alg_in_find_is_safe<_It, _Ty1> && _Vector_alg_in_find_is_safe<_It, _Ty2> && _Output_iterator_for_known_size_vector_alg_is_safe<_Out, _It>()) { if (!_STD is_constant_evaluated()) { - const auto _Count = _Last - _First; + const auto _Count = _Last - _First; + _STD _Contiguous_iter_verify(_First, _Count); + _STD _Contiguous_iter_verify(_Output, static_cast>(_Count)); + const auto _First_ptr = _STD to_address(_First); + const auto _Last_ptr = _First_ptr + static_cast(_Count); const auto _Out_ptr = _STD to_address(_Output); - _First += _Count; - const auto _Last_ptr = _STD to_address(_First); - - _Output += static_cast>(_Count); - (void) _STD to_address(_Output); - if (_STD _Could_compare_equal_to_value_type<_It>(_Oldval)) { _STD _Replace_copy_vectorized(_First_ptr, _Last_ptr, _Out_ptr, _Oldval, _Newval); } else { _CSTD memcpy(_Out_ptr, _First_ptr, static_cast(_Count) * sizeof(*_Out_ptr)); } + _First += _Count; + _Output += static_cast>(_Count); return {_STD move(_First), _STD move(_Output)}; } } From 6c7e4ff4b88170b0bbcb31a8d269ba3b73ec80a9 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 14:08:03 -0800 Subject: [PATCH 11/12] Mark `_Replace_copy_impl()` as `__declspec(noalias)`. --- stl/src/vector_algorithms.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/vector_algorithms.cpp b/stl/src/vector_algorithms.cpp index d534dbbeb6e..ba40ad57138 100644 --- a/stl/src/vector_algorithms.cpp +++ b/stl/src/vector_algorithms.cpp @@ -6491,7 +6491,7 @@ __declspec(noalias) size_t __stdcall __std_mismatch_8( namespace { namespace _Replacing { template - void __stdcall _Replace_copy_impl( + __declspec(noalias) void __stdcall _Replace_copy_impl( const void* _First, const void* const _Last, void* _Dest, const _Ty _Old_val, const _Ty _New_val) noexcept { #ifndef _M_ARM64EC const size_t _Size_bytes = _Byte_length(_First, _Last); From 0011363661540150a224e5600f4659eae5390263 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 6 Jan 2026 14:33:17 -0800 Subject: [PATCH 12/12] Avoid shadowing: `in_place` => `replace_is_vectorized`, determine from size --- .../VSO_0000000_vector_algorithms/test.cpp | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp index f48e5eaec03..3a77c979cf9 100644 --- a/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp +++ b/tests/std/tests/VSO_0000000_vector_algorithms/test.cpp @@ -765,8 +765,11 @@ void test_case_replace_copy(const vector& input, vector& out_expected, vec #endif // ^^^ !_HAS_CXX20 ^^^ } -template +template void test_replace(mt19937_64& gen) { + // replace() is vectorized for 4 and 8 bytes only. + constexpr bool replace_is_vectorized = sizeof(T) >= 4; + using TD = conditional_t; uniform_int_distribution dis(0, 9); @@ -782,7 +785,7 @@ void test_replace(mt19937_64& gen) { v->reserve(dataCount); } - if constexpr (in_place) { + if constexpr (replace_is_vectorized) { for (const auto& v : {&in_out_expected, &in_out_actual, &in_out_actual_r}) { v->reserve(dataCount); } @@ -792,7 +795,7 @@ void test_replace(mt19937_64& gen) { const T old_val = static_cast(dis(gen)); const T new_val = static_cast(dis(gen)); - if constexpr (in_place) { + if constexpr (replace_is_vectorized) { test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); } test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); @@ -811,7 +814,7 @@ void test_replace(mt19937_64& gen) { v->assign(source.size(), T{0}); } - if constexpr (in_place) { + if constexpr (replace_is_vectorized) { test_case_replace(in_out_expected, in_out_actual, in_out_actual_r, old_val, new_val); } test_case_replace_copy(source, out_expected, out_actual, out_actual_r, old_val, new_val); @@ -1345,16 +1348,15 @@ void test_vector_algorithms(mt19937_64& gen) { test_includes(gen); #endif // _HAS_CXX17 - // replace() is vectorized for 4 and 8 bytes only. - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); - test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); + test_replace(gen); test_reverse(gen); test_reverse(gen);