Skip to content

Vectorize lexicographical_compare! #4552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Stephan T. Lavavej (StephanTLavavej) merged 14 commits into from
Apr 12, 2024
Merged

Vectorize lexicographical_compare! #4552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
e4bb843
Vectorize lexicographical_compare!
AlexGuteniev Apr 1, 2024
71f3c50
format
AlexGuteniev Apr 1, 2024
ea8f20e
hide C++20
AlexGuteniev Apr 1, 2024
a82b9d6
get em back
AlexGuteniev Apr 2, 2024
fb8f5bc
reduce invasiveness
AlexGuteniev Apr 8, 2024
c29ebb4
benchmark: fancy way to have common args
AlexGuteniev Apr 8, 2024
b4f5ca4
benchmark: explain signed / unsigned 8-bit integer
AlexGuteniev Apr 8, 2024
808ff8f
More precise comments in env.lst
AlexGuteniev Apr 9, 2024
26606be
avoid one comparison
AlexGuteniev Apr 9, 2024
a3bf3fa
Merge branch 'not_another_vector_algorithms_cpp_change' of https://gi…
AlexGuteniev Apr 9, 2024
8d839b2
Merge remote-tracking branch 'upstream/main' into not_another_vector_…
AlexGuteniev Apr 10, 2024
9e65f99
missed _Meow_vectorized
AlexGuteniev Apr 10, 2024
3990a81
Test `ranges::lexicographical_compare`.
StephanTLavavej Apr 11, 2024
dc9db99
Update test comments as more cases are allowed.
StephanTLavavej Apr 12, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions benchmarks/src/mismatch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,12 @@ using namespace std;

constexpr int64_t no_pos = -1;

template <class T>
enum class op {
mismatch,
lexi,
};

template <class T, op Op>
void bm(benchmark::State& state) {
vector<T> a(static_cast<size_t>(state.range(0)), T{'.'});
vector<T> b(static_cast<size_t>(state.range(0)), T{'.'});
Expand All @@ -22,15 +27,27 @@ void bm(benchmark::State& state) {
}

for (auto _ : state) {
benchmark::DoNotOptimize(ranges::mismatch(a, b));
if constexpr (Op == op::mismatch) {
benchmark::DoNotOptimize(ranges::mismatch(a, b));
} else if constexpr (Op == op::lexi) {
benchmark::DoNotOptimize(ranges::lexicographical_compare(a, b));
}
}
}

#define COMMON_ARGS Args({8, 3})->Args({24, 22})->Args({105, -1})->Args({4021, 3056})
void common_args(auto bm) {
bm->Args({8, 3})->Args({24, 22})->Args({105, -1})->Args({4021, 3056});
}

BENCHMARK(bm<uint8_t, op::mismatch>)->Apply(common_args);
BENCHMARK(bm<uint16_t, op::mismatch>)->Apply(common_args);
BENCHMARK(bm<uint32_t, op::mismatch>)->Apply(common_args);
BENCHMARK(bm<uint64_t, op::mismatch>)->Apply(common_args);

BENCHMARK(bm<uint8_t>)->COMMON_ARGS;
BENCHMARK(bm<uint16_t>)->COMMON_ARGS;
BENCHMARK(bm<uint32_t>)->COMMON_ARGS;
BENCHMARK(bm<uint64_t>)->COMMON_ARGS;
BENCHMARK(bm<uint8_t, op::lexi>)->Apply(common_args); // still optimized without vector algorithms using memcmp
BENCHMARK(bm<int8_t, op::lexi>)->Apply(common_args); // optimized with vector algorithms only
BENCHMARK(bm<uint16_t, op::lexi>)->Apply(common_args);
BENCHMARK(bm<uint32_t, op::lexi>)->Apply(common_args);
BENCHMARK(bm<uint64_t, op::lexi>)->Apply(common_args);

BENCHMARK_MAIN();
16 changes: 15 additions & 1 deletion stl/inc/algorithm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10971,8 +10971,22 @@ namespace ranges {
_Num2 = SIZE_MAX;
}

const int _Ans = _STD _Memcmp_count(_First1, _First2, (_STD min)(_Num1, _Num2));
const size_t _Num = (_STD min)(_Num1, _Num2);
#if _USE_STD_VECTOR_ALGORITHMS
const auto _First1_ptr = _STD to_address(_First1);
const auto _First2_ptr = _STD to_address(_First2);
const size_t _Pos = _Mismatch_vectorized<sizeof(*_First1_ptr)>(_First1_ptr, _First2_ptr, _Num);
if (_Pos == _Num2) {
return false;
} else if (_Pos == _Num1) {
return true;
} else {
return _STD invoke(_Pred, _First1_ptr[_Pos], _First2_ptr[_Pos]);
}
#else // ^^^ _USE_STD_VECTOR_ALGORITHMS / !_USE_STD_VECTOR_ALGORITHMS vvv
const int _Ans = _STD _Memcmp_count(_First1, _First2, _Num);
return _Memcmp_classification_pred{}(_Ans, 0) || (_Ans == 0 && _Num1 < _Num2);
#endif // ^^^ !_USE_STD_VECTOR_ALGORITHMS ^^^
}
}

Expand Down
53 changes: 45 additions & 8 deletions stl/inc/xutility
View file
Open in desktop
Comment thread
AlexGuteniev marked this conversation as resolved.
Original file line number Diff line number Diff line change
Expand Up @@ -5629,8 +5629,14 @@ namespace ranges {
#endif // _HAS_CXX20

template <class _Elem1, class _Elem2>
constexpr bool _Lex_compare_memcmp_classify_elements = conjunction_v<_Is_character_or_bool<_Elem1>,
_Is_character_or_bool<_Elem2>, is_unsigned<_Elem1>, is_unsigned<_Elem2>>;
constexpr bool _Lex_compare_memcmp_classify_elements =
#if _USE_STD_VECTOR_ALGORITHMS
is_integral_v<_Elem1> && is_integral_v<_Elem2> && sizeof(_Elem1) == sizeof(_Elem2)
&& is_unsigned_v<_Elem1> == is_unsigned_v<_Elem2>;
#else // ^^^ _USE_STD_VECTOR_ALGORITHMS / !_USE_STD_VECTOR_ALGORITHMS vvv
conjunction_v<_Is_character_or_bool<_Elem1>, _Is_character_or_bool<_Elem2>, is_unsigned<_Elem1>,
is_unsigned<_Elem2>>;
#endif // ^^^ !_USE_STD_VECTOR_ALGORITHMS ^^^

#ifdef __cpp_lib_byte
template <>
Expand Down Expand Up @@ -5702,10 +5708,24 @@ _NODISCARD _CONSTEXPR20 bool lexicographical_compare(
if (!_STD is_constant_evaluated())
#endif // _HAS_CXX20
{
const auto _Num1 = static_cast<size_t>(_ULast1 - _UFirst1);
const auto _Num2 = static_cast<size_t>(_ULast2 - _UFirst2);
const int _Ans = _STD _Memcmp_count(_UFirst1, _UFirst2, (_STD min)(_Num1, _Num2));
const auto _Num1 = static_cast<size_t>(_ULast1 - _UFirst1);
const auto _Num2 = static_cast<size_t>(_ULast2 - _UFirst2);
const size_t _Num = (_STD min)(_Num1, _Num2);
#if _USE_STD_VECTOR_ALGORITHMS
const auto _First1_ptr = _STD _To_address(_UFirst1);
const auto _First2_ptr = _STD _To_address(_UFirst2);
const size_t _Pos = _Mismatch_vectorized<sizeof(*_First1_ptr)>(_First1_ptr, _First2_ptr, _Num);
if (_Pos == _Num2) {
return false;
} else if (_Pos == _Num1) {
return true;
} else {
return _Pred(_First1_ptr[_Pos], _First2_ptr[_Pos]);
}
#else // ^^^ _USE_STD_VECTOR_ALGORITHMS / !_USE_STD_VECTOR_ALGORITHMS vvv
const int _Ans = _STD _Memcmp_count(_UFirst1, _UFirst2, _Num);
return _Memcmp_pred{}(_Ans, 0) || (_Ans == 0 && _Num1 < _Num2);
Comment thread
StephanTLavavej marked this conversation as resolved.
#endif // ^^^ !_USE_STD_VECTOR_ALGORITHMS ^^^
}
}

Expand Down Expand Up @@ -5802,14 +5822,31 @@ _NODISCARD constexpr auto lexicographical_compare_three_way(const _InIt1 _First1
using _Memcmp_pred = _Lex_compare_three_way_memcmp_classify<decltype(_UFirst1), decltype(_UFirst2), _Cmp>;
if constexpr (!is_void_v<_Memcmp_pred>) {
if (!_STD is_constant_evaluated()) {
const auto _Num1 = static_cast<size_t>(_ULast1 - _UFirst1);
const auto _Num2 = static_cast<size_t>(_ULast2 - _UFirst2);
const int _Ans = _STD _Memcmp_count(_UFirst1, _UFirst2, (_STD min)(_Num1, _Num2));
const auto _Num1 = static_cast<size_t>(_ULast1 - _UFirst1);
const auto _Num2 = static_cast<size_t>(_ULast2 - _UFirst2);
const size_t _Num = (_STD min)(_Num1, _Num2);
#if _USE_STD_VECTOR_ALGORITHMS
const auto _First1_ptr = _STD to_address(_UFirst1);
const auto _First2_ptr = _STD to_address(_UFirst2);
const size_t _Pos = _Mismatch_vectorized<sizeof(*_First1_ptr)>(_First1_ptr, _First2_ptr, _Num);
if (_Pos == _Num1) {
return _Pos == _Num2 ? strong_ordering::equal : strong_ordering::less;
} else if (_Pos == _Num2) {
return strong_ordering::greater;
} else {
const auto _Val1 = _First1_ptr[_Pos];
const auto _Val2 = _First2_ptr[_Pos];
__assume(_Val1 != _Val2); // avoid one comparison
return _Comp(_Val1, _Val2);
}
#else // ^^^ _USE_STD_VECTOR_ALGORITHMS / !_USE_STD_VECTOR_ALGORITHMS vvv
const int _Ans = _STD _Memcmp_count(_UFirst1, _UFirst2, _Num);
if (_Ans == 0) {
return _Num1 <=> _Num2;
} else {
return _Memcmp_pred{}(_Ans, 0);
}
#endif // ^^^ !_USE_STD_VECTOR_ALGORITHMS ^^^
}
}

Expand Down
3 changes: 3 additions & 0 deletions tests/std/tests/GH_000431_lex_compare_memcmp_classify/env.lst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,6 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

RUNALL_INCLUDE ..\char8_t_matrix.lst
RUNALL_CROSSLIST
* PM_CL="" # Test manual vectorization
* PM_CL="/D_USE_STD_VECTOR_ALGORITHMS=0" # Test memcmp optimization
44 changes: 25 additions & 19 deletions tests/std/tests/GH_000431_lex_compare_memcmp_classify/test.compile.pass.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,10 @@ void test_lex_compare_memcmp_classify_for_types() {
test_lex_compare_memcmp_classify_for_pred<expected_less, Type1, Type2, less<>>();
test_lex_compare_memcmp_classify_for_pred<expected_greater, Type1, Type2, greater<>>();

test_lex_compare_memcmp_classify_for_pred<void, Type1, Type2, less<int>>();
test_lex_compare_memcmp_classify_for_pred<void, Type1, Type2, greater<int>>();
using bigger_type = conditional_t<sizeof(Type1) == sizeof(int), long long, int>;

test_lex_compare_memcmp_classify_for_pred<void, Type1, Type2, less<bigger_type>>();
test_lex_compare_memcmp_classify_for_pred<void, Type1, Type2, greater<bigger_type>>();

test_lex_compare_memcmp_classify_for_pred<void, Type1, Type2, less<volatile Type1>>();
test_lex_compare_memcmp_classify_for_pred<void, Type1, Type2, greater<volatile Type1>>();
Expand Down Expand Up @@ -176,13 +178,15 @@ void test_lex_compare_memcmp_classify_for_types() {
#endif // _HAS_CXX20
}

constexpr bool vec_alg = _USE_STD_VECTOR_ALGORITHMS;

template <bool Expected, class Type1, class Type2>
void test_lex_compare_memcmp_classify_for_1byte_integrals() {
test_lex_compare_memcmp_classify_for_types<Expected, Type1, Type2>();

test_lex_compare_memcmp_classify_for_opaque_preds<is_unsigned_v<char>, Type1, Type2, char>();
test_lex_compare_memcmp_classify_for_opaque_preds<is_unsigned_v<char> || vec_alg, Type1, Type2, char>();
test_lex_compare_memcmp_classify_for_opaque_preds<true, Type1, Type2, unsigned char>();
test_lex_compare_memcmp_classify_for_opaque_preds<false, Type1, Type2, signed char>();
test_lex_compare_memcmp_classify_for_opaque_preds<vec_alg, Type1, Type2, signed char>();
#ifdef __cpp_lib_char8_t
test_lex_compare_memcmp_classify_for_opaque_preds<true, Type1, Type2, char8_t>();
#endif // __cpp_lib_char8_t
Expand Down Expand Up @@ -224,14 +228,14 @@ bool operator<(const user_struct&, const user_struct&) {
}

void lex_compare_memcmp_classify_test_cases() {
// Allow unsigned 1 byte integrals
test_lex_compare_memcmp_classify_for_1byte_integrals<is_unsigned_v<char>, char, char>();
// Test 1 byte integrals
test_lex_compare_memcmp_classify_for_1byte_integrals<is_unsigned_v<char> || vec_alg, char, char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<is_unsigned_v<char>, unsigned char, char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<is_unsigned_v<char>, char, unsigned char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<true, unsigned char, unsigned char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, signed char, signed char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, char, signed char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, signed char, char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<vec_alg, signed char, signed char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<vec_alg && is_signed_v<char>, char, signed char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<vec_alg && is_signed_v<char>, signed char, char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, unsigned char, signed char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, signed char, unsigned char>();
#ifdef __cpp_lib_char8_t
Expand All @@ -252,8 +256,8 @@ void lex_compare_memcmp_classify_test_cases() {
test_lex_compare_memcmp_classify_for_1byte_integrals<true, bool, char8_t>();
test_lex_compare_memcmp_classify_for_1byte_integrals<true, char8_t, bool>();
#endif // __cpp_lib_char8_t
test_lex_compare_memcmp_classify_for_1byte_integrals<false, char, bool>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, bool, char>();
test_lex_compare_memcmp_classify_for_1byte_integrals<vec_alg && is_unsigned_v<char>, char, bool>();
test_lex_compare_memcmp_classify_for_1byte_integrals<vec_alg && is_unsigned_v<char>, bool, char>();
Comment thread
StephanTLavavej marked this conversation as resolved.
test_lex_compare_memcmp_classify_for_1byte_integrals<false, signed char, bool>();
test_lex_compare_memcmp_classify_for_1byte_integrals<false, bool, signed char>();

Expand All @@ -278,13 +282,13 @@ void lex_compare_memcmp_classify_test_cases() {
test_lex_compare_memcmp_classify_for_types<true, byte, byte>();
#endif // __cpp_lib_byte

// Don't allow bigger integrals
// Test bigger integrals
test_lex_compare_memcmp_classify_for_types<false, unsigned char, int>();
test_lex_compare_memcmp_classify_for_types<false, int, unsigned char>();
test_lex_compare_memcmp_classify_for_types<false, int, int>();
test_lex_compare_memcmp_classify_for_types<false, unsigned int, unsigned int>();
test_lex_compare_memcmp_classify_for_types<false, short, short>();
test_lex_compare_memcmp_classify_for_types<false, unsigned short, unsigned short>();
test_lex_compare_memcmp_classify_for_types<vec_alg, int, int>();
test_lex_compare_memcmp_classify_for_types<vec_alg, unsigned int, unsigned int>();
test_lex_compare_memcmp_classify_for_types<vec_alg, short, short>();
test_lex_compare_memcmp_classify_for_types<vec_alg, unsigned short, unsigned short>();

// Don't allow pointers
test_lex_compare_memcmp_classify_for_types<false, int*, int*>();
Expand All @@ -298,9 +302,11 @@ void lex_compare_memcmp_classify_test_cases() {
test_lex_compare_memcmp_classify_for_pred<less<int>, char8_t, char8_t, _Char_traits_lt<char_traits<char8_t>>>();
#endif // __cpp_lib_char8_t

test_lex_compare_memcmp_classify_for_pred<void, wchar_t, wchar_t, _Char_traits_lt<char_traits<wchar_t>>>();
test_lex_compare_memcmp_classify_for_pred<void, char16_t, char16_t, _Char_traits_lt<char_traits<char16_t>>>();
test_lex_compare_memcmp_classify_for_pred<void, char32_t, char32_t, _Char_traits_lt<char_traits<char32_t>>>();
using vless = conditional_t<vec_alg, less<int>, void>;

test_lex_compare_memcmp_classify_for_pred<vless, wchar_t, wchar_t, _Char_traits_lt<char_traits<wchar_t>>>();
test_lex_compare_memcmp_classify_for_pred<vless, char16_t, char16_t, _Char_traits_lt<char_traits<char16_t>>>();
test_lex_compare_memcmp_classify_for_pred<vless, char32_t, char32_t, _Char_traits_lt<char_traits<char32_t>>>();

// Test different containers
#if _HAS_CXX20
Expand Down
Loading