Vectorize is_sorted_until

benchmarks/CMakeLists.txt

@@ -107,6 +107,7 @@ add_benchmark(find_and_count src/find_and_count.cpp)
 add_benchmark(find_first_of src/find_first_of.cpp)
 add_benchmark(has_single_bit src/has_single_bit.cpp)
 add_benchmark(iota src/iota.cpp)
+add_benchmark(is_sorted_until src/is_sorted_until.cpp)
 add_benchmark(locale_classic src/locale_classic.cpp)
 add_benchmark(minmax_element src/minmax_element.cpp)
 add_benchmark(mismatch src/mismatch.cpp)

benchmarks/inc/utility.hpp

@@ -5,14 +5,15 @@
 
 #include <algorithm>
 #include <cstddef>
+#include <memory>
 #include <random>
 #include <vector>
 
-template <class Contained>
-std::vector<Contained> random_vector(size_t n) {
+template <class Contained, template <class> class Alloc = std::allocator>
+std::vector<Contained, Alloc<Contained>> random_vector(size_t n) {
     std::mt19937_64 prng;
 
-    std::vector<Contained> res(n);
+    std::vector<Contained, Alloc<Contained>> res(n);
 
 // Here, the type Contained can be char, int, aggregate<Data>, or non_trivial<Data> where Data is char or int.
 // (aggregate<Data> and non_trivial<Data> are defined in udt.hpp.)

benchmarks/src/is_sorted_until.cpp

@@ -0,0 +1,73 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <cstddef>
+#include <cstdint>
+#include <random>
+#include <type_traits>
+#include <vector>
+
+#include "skewed_allocator.hpp"
+#include "utility.hpp"
+
+enum class AlgType { Std, Rng };
+
+template <class T, AlgType Alg>
+void bm_is_sorted_until(benchmark::State& state) {
+    const std::size_t size     = static_cast<std::size_t>(state.range(0));
+    const std::size_t sort_pos = static_cast<std::size_t>(state.range(1));
+
+    std::vector<T, not_highly_aligned_allocator<T>> v;
+    if constexpr (std::is_integral_v<T>) {
+        v = random_vector<T, not_highly_aligned_allocator>(size);
+    } else if constexpr (std::is_floating_point_v<T>) {
+        v.resize(size, 0.0);
+        std::mt19937 gen;
+        std::normal_distribution<T> dis(0, 100000.0);
+        std::generate_n(v.begin(), size, [&dis, &gen] { return dis(gen); });
+    } else {
+        static_assert(false);
+    }
+
+    std::sort(v.begin(), v.begin() + sort_pos);
+
+    for (auto _ : state) {
+        benchmark::DoNotOptimize(v);
+        if constexpr (Alg == AlgType::Std) {
+            benchmark::DoNotOptimize(std::is_sorted_until(v.begin(), v.end()));
+        } else {
+            benchmark::DoNotOptimize(std::ranges::is_sorted_until(v));
+        }
+    }
+}
+
+void common_args(auto bm) {
+    bm->ArgPair(3000, 1800);
+}
+
+BENCHMARK(bm_is_sorted_until<std::int8_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int8_t, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int16_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int16_t, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int32_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int32_t, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int64_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::int64_t, AlgType::Rng>)->Apply(common_args);
+
+BENCHMARK(bm_is_sorted_until<std::uint8_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint8_t, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint16_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint16_t, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint32_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint32_t, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint64_t, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<std::uint64_t, AlgType::Rng>)->Apply(common_args);
+
+BENCHMARK(bm_is_sorted_until<float, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<float, AlgType::Rng>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<double, AlgType::Std>)->Apply(common_args);
+BENCHMARK(bm_is_sorted_until<double, AlgType::Rng>)->Apply(common_args);
+
+BENCHMARK_MAIN();

stl/inc/algorithm

@@ -74,6 +74,17 @@ __declspec(noalias) _Min_max_8u __stdcall __std_minmax_8u(const void* _First, co
 __declspec(noalias) _Min_max_f __stdcall __std_minmax_f(const void* _First, const void* _Last) noexcept;
 __declspec(noalias) _Min_max_d __stdcall __std_minmax_d(const void* _First, const void* _Last) noexcept;
 
+const void* __stdcall __std_is_sorted_until_1i(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_1u(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_2i(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_2u(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_4i(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_4u(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_8i(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_8u(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_f(const void* _First, const void* _Last, bool _Greater) noexcept;
+const void* __stdcall __std_is_sorted_until_d(const void* _First, const void* _Last, bool _Greater) noexcept;
+
 // TRANSITION, DevCom-10610477
 __declspec(noalias) void __stdcall __std_replace_4(
     void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept;
@@ -207,6 +218,43 @@ auto _Minmax_vectorized(_Ty* const _First, _Ty* const _Last) noexcept {
     }
 }
 
+template <class _Ty>
+_Ty* _Is_sorted_until_vectorized(_Ty* const _First, _Ty* const _Last, const bool _Greater) noexcept {
+    constexpr bool _Signed = is_signed_v<_Ty>;
+
+    if constexpr (is_same_v<remove_const_t<_Ty>, float>) {
+        return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_f(_First, _Last, _Greater)));
+    } else if constexpr (_Is_any_of_v<remove_const_t<_Ty>, double, long double>) {
+        return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_d(_First, _Last, _Greater)));
+    } else if constexpr (sizeof(_Ty) == 1) {
+        if constexpr (_Signed) {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_1i(_First, _Last, _Greater)));
+        } else {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_1u(_First, _Last, _Greater)));
+        }
+    } else if constexpr (sizeof(_Ty) == 2) {
+        if constexpr (_Signed) {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_2i(_First, _Last, _Greater)));
+        } else {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_2u(_First, _Last, _Greater)));
+        }
+    } else if constexpr (sizeof(_Ty) == 4) {
+        if constexpr (_Signed) {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_4i(_First, _Last, _Greater)));
+        } else {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_4u(_First, _Last, _Greater)));
+        }
+    } else if constexpr (sizeof(_Ty) == 8) {
+        if constexpr (_Signed) {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_8i(_First, _Last, _Greater)));
+        } else {
+            return const_cast<_Ty*>(static_cast<const _Ty*>(::__std_is_sorted_until_8u(_First, _Last, _Greater)));
+        }
+    } else {
+        _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size
+    }
+}
+
 template <class _Ty, class _TVal1, class _TVal2>
 __declspec(noalias) void _Replace_vectorized(
     _Ty* const _First, _Ty* const _Last, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept {
@@ -8284,6 +8332,25 @@ namespace ranges {
             return _First;
         }
 
+#if _USE_STD_VECTOR_ALGORITHMS
+        if constexpr (_Is_min_max_iterators_safe<_It> && sized_sentinel_for<_Se, _It> && is_same_v<_Pj, identity>) {
+            constexpr bool _Is_greater = _Is_predicate_greater<_It, _Pr>;
+            if constexpr (_Is_greater || _Is_predicate_less<_It, _Pr>) {
+                if (!_STD is_constant_evaluated()) {
+                    const auto _First_ptr = _STD _To_address(_First);
+                    const auto _Last_ptr  = _First_ptr + static_cast<size_t>(_Last - _First);
+                    const auto _Result    = _STD _Is_sorted_until_vectorized(_First_ptr, _Last_ptr, _Is_greater);
+
+                    if constexpr (is_pointer_v<_It>) {
+                        return _Result;
+                    } else {
+                        return _First + static_cast<iter_difference_t<_It>>(_Result - _First_ptr);
+                    }
+                }
+            }
+        }
+#endif // _USE_STD_VECTOR_ALGORITHMS
+
         for (auto _Prev = _First; ++_First != _Last; ++_Prev) {
             if (_STD invoke(_Pred, _STD invoke(_Proj, *_First), _STD invoke(_Proj, *_Prev))) {
                 break;
@@ -11152,6 +11219,29 @@ _NODISCARD _CONSTEXPR20 _FwdIt is_sorted_until(const _FwdIt _First, _FwdIt _Last
     _STD _Adl_verify_range(_First, _Last);
     auto _UFirst = _STD _Get_unwrapped(_First);
     auto _ULast  = _STD _Get_unwrapped(_Last);
+
+#if _USE_STD_VECTOR_ALGORITHMS
+    if constexpr (_Is_min_max_iterators_safe<decltype(_UFirst)>) {
+        constexpr bool _Is_greater = _Is_predicate_greater<decltype(_UFirst), _Pr>;
+        if constexpr (_Is_greater || _Is_predicate_less<decltype(_UFirst), _Pr>) {
+            if (!_STD _Is_constant_evaluated()) {
+                const auto _First_ptr = _STD _To_address(_UFirst);
+                const auto _Result =
+                    _STD _Is_sorted_until_vectorized(_First_ptr, _STD _To_address(_ULast), _Is_greater);
+
+                if constexpr (is_pointer_v<decltype(_UFirst)>) {
+                    _UFirst = _Result;
+                } else {
+                    _UFirst += static_cast<_Iter_diff_t<decltype(_UFirst)>>(_Result - _First_ptr);
+                }
+
+                _STD _Seek_wrapped(_Last, _UFirst);
+                return _Last;
+            }
+        }
+    }
+#endif // _USE_STD_VECTOR_ALGORITHMS
+
     if (_UFirst != _ULast) {
         for (auto _UNext = _UFirst; ++_UNext != _ULast; ++_UFirst) {
             if (_DEBUG_LT_PRED(_Pred, *_UNext, *_UFirst)) {

stl/inc/xutility

@@ -6981,24 +6981,37 @@ namespace ranges {
 } // namespace ranges
 #endif // _HAS_CXX20
 
-template <class _Iter, class _Pr, class _Elem = _Iter_value_t<_Iter>>
-constexpr bool _Is_min_max_optimization_safe = // Activate the vector algorithms for min_/max_element?
+template <class _Iter, class _Elem = _Iter_value_t<_Iter>>
+constexpr bool _Is_min_max_iterators_safe =
     _Iterator_is_contiguous<_Iter> // The iterator must be contiguous so we can get raw pointers.
     && !_Iterator_is_volatile<_Iter> // The iterator must not be volatile.
-    && conjunction_v<disjunction<
+    && disjunction_v<
 #if _USE_STD_VECTOR_FLOATING_ALGORITHMS
 #if defined(__LDBL_DIG__) && __LDBL_DIG__ == 18
-                         is_same<_Elem, float>, is_same<_Elem, double>,
+        is_same<_Elem, float>, is_same<_Elem, double>,
 #else // ^^^ 80-bit long double (not supported by MSVC in general, see GH-1316) / 64-bit long double vvv
-                         is_floating_point<_Elem>, // Element is floating-point or...
+        is_floating_point<_Elem>, // Element is floating-point or...
 #endif // ^^^ 64-bit long double ^^^
 #endif // _USE_STD_VECTOR_FLOATING_ALGORITHMS
-                         is_integral<_Elem>, is_pointer<_Elem>>, // ... integral or pointer type.
-        disjunction< // And either of the following:
+        is_integral<_Elem>, is_pointer<_Elem>>; // ... integral or pointer type.
+
+template <class _Iter, class _Pr>
+constexpr bool _Is_predicate_less = _Is_any_of_v<_Pr,
+#if _HAS_CXX20
+    _RANGES less,
+#endif // _HAS_CXX20
+    less<>, less<_Iter_value_t<_Iter>>>;
+
+template <class _Iter, class _Pr>
+constexpr bool _Is_predicate_greater = _Is_any_of_v<_Pr,
 #if _HAS_CXX20
-            is_same<_Pr, _RANGES less>, // predicate is ranges::less
+    _RANGES greater,
 #endif // _HAS_CXX20
-            is_same<_Pr, less<>>, is_same<_Pr, less<_Elem>>>>; // predicate is less
+    greater<>, greater<_Iter_value_t<_Iter>>>;
+
+template <class _Iter, class _Pr>
+constexpr bool _Is_min_max_optimization_safe = // Activate the vector algorithms for min_/max_element?
+    _Is_min_max_iterators_safe<_Iter> && _Is_predicate_less<_Iter, _Pr>;
 
 // Unlike the position-based vectorized implementation, the value-based vectorized implementation
 // does not always produce the expected results for floating-point types.
@@ -7016,7 +7029,7 @@ constexpr bool _Is_min_max_value_optimization_safe = // Activate the vector algo
 #ifndef _M_FP_FAST
     !is_floating_point_v<_Elem> &&
 #endif // ^^^ !defined(_M_FP_FAST) ^^^
-    _Is_min_max_optimization_safe<_Iter, _Pr, _Elem>;
+    _Is_min_max_optimization_safe<_Iter, _Pr>;
 
 template <class _FwdIt, class _Pr>
 constexpr _FwdIt _Max_element_unchecked(_FwdIt _First, _FwdIt _Last, _Pr _Pred) { // find largest element