[libc++] Optimize to_chars for bases 2, 8 and 16 - #223222
Open
davebayer wants to merge 1 commit into
Open
Conversation
|
@llvm/pr-subscribers-libcxx Author: David Bayer (davebayer) ChangesI've come up with an optimization for However, this optimization is dependent on the I've ran some benchmarks on my Intel i9-14900K with
For small types, it's not worth it, but for 32-bit and wider types, we can get some substantial improvements. Would you be interested in having this optimization when a If yes, then I'll rework the implementation to change the chars/register based on the platform's word size as well. I've used GPT-5.6 SOL to help me do the benchmarks & optimize the base 16 path. Full diff: https://github.com/llvm/llvm-project/pull/223222.diff 1 Files Affected:
diff --git a/libcxx/include/__charconv/to_chars_integral.h b/libcxx/include/__charconv/to_chars_integral.h
index 6d425139260b6..3fbbd6a8f8c3c 100644
--- a/libcxx/include/__charconv/to_chars_integral.h
+++ b/libcxx/include/__charconv/to_chars_integral.h
@@ -110,6 +110,11 @@ namespace __itoa {
template <unsigned _Base>
struct _LIBCPP_HIDDEN __integral;
+template <size_t _Np>
+struct _LIBCPP_HIDDEN __chars_storage {
+ char __data[_Np];
+};
+
template <>
struct _LIBCPP_HIDDEN __integral<2> {
template <typename _Tp>
@@ -126,21 +131,32 @@ struct _LIBCPP_HIDDEN __integral<2> {
int __n = __width(__value);
if (__n > __cap)
return {__last, errc::value_too_large};
-
__last = __first + __n;
char* __p = __last;
- const unsigned __divisor = 16;
- while (__value > __divisor) {
- unsigned __c = __value % __divisor;
+
+ constexpr auto __ncs_per_word = 8u;
+ constexpr auto __divisor = 1u << __ncs_per_word;
+ constexpr auto __dep_mask = uint64_t{0x0101'0101'0101'0101};
+ constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030};
+ while (__value >= __divisor) {
+ const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
+ auto __cs = __digits | __c0_offset_mask;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ __cs = __builtin_bswap64(__cs);
+#endif
__value /= __divisor;
- __p -= 4;
- std::copy_n(&__base_2_lut[4 * __c], 4, __p);
+ __p -= __ncs_per_word;
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ __builtin_memcpy(__p, __chars.__data, __ncs_per_word);
}
- do {
- unsigned __c = __value % 2;
- __value /= 2;
- *--__p = "01"[__c];
- } while (__value != 0);
+ const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
+ auto __cs = __digits | __c0_offset_mask;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ __cs = __builtin_bswap64(__cs);
+#endif
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ const auto __remaining = static_cast<size_t>(__p - __first);
+ __builtin_memcpy(__first, __chars.__data + __ncs_per_word - __remaining, __remaining);
return {__last, errc(0)};
}
};
@@ -161,21 +177,32 @@ struct _LIBCPP_HIDDEN __integral<8> {
int __n = __width(__value);
if (__n > __cap)
return {__last, errc::value_too_large};
+ __last = __first + __n;
+ char* __p = __last;
- __last = __first + __n;
- char* __p = __last;
- unsigned __divisor = 64;
- while (__value > __divisor) {
- unsigned __c = __value % __divisor;
+ constexpr auto __ncs_per_word = 8u;
+ constexpr auto __divisor = 1u << (3 * __ncs_per_word);
+ constexpr auto __dep_mask = uint64_t{0x0707'0707'0707'0707};
+ constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030};
+ while (__value >= __divisor) {
+ const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
+ auto __cs = __digits | __c0_offset_mask;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ __cs = __builtin_bswap64(__cs);
+#endif
__value /= __divisor;
- __p -= 2;
- std::copy_n(&__base_8_lut[2 * __c], 2, __p);
+ __p -= __ncs_per_word;
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ __builtin_memcpy(__p, __chars.__data, __ncs_per_word);
}
- do {
- unsigned __c = __value % 8;
- __value /= 8;
- *--__p = "01234567"[__c];
- } while (__value != 0);
+ const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
+ auto __cs = __digits | __c0_offset_mask;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ __cs = __builtin_bswap64(__cs);
+#endif
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ const auto __remaining = static_cast<size_t>(__p - __first);
+ __builtin_memcpy(__first, __chars.__data + __ncs_per_word - __remaining, __remaining);
return {__last, errc(0)};
}
};
@@ -197,21 +224,36 @@ struct _LIBCPP_HIDDEN __integral<16> {
if (__n > __cap)
return {__last, errc::value_too_large};
- __last = __first + __n;
- char* __p = __last;
- unsigned __divisor = 256;
- while (__value > __divisor) {
- unsigned __c = __value % __divisor;
+ __last = __first + __n;
+ char* __p = __last;
+
+ constexpr auto __ncs_per_word = 8u;
+ constexpr auto __divisor = uint64_t{1} << (4 * __ncs_per_word);
+ constexpr auto __dep_mask = uint64_t{0x0f0f'0f0f'0f0f'0f0f};
+ constexpr auto __add_six_mask = uint64_t{0x0606'0606'0606'0606};
+ constexpr auto __high_bit_mask = uint64_t{0x1010'1010'1010'1010};
+ constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030};
+ while (__value >= __divisor) {
+ const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
+ const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4;
+ auto __cs = __digits + __c0_offset_mask + __alpha * 0x27;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ __cs = __builtin_bswap64(__cs);
+#endif
__value /= __divisor;
- __p -= 2;
- std::copy_n(&__base_16_lut[2 * __c], 2, __p);
+ __p -= __ncs_per_word;
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ __builtin_memcpy(__p, __chars.__data, __ncs_per_word);
}
- if (__first != __last)
- do {
- unsigned __c = __value % 16;
- __value /= 16;
- *--__p = "0123456789abcdef"[__c];
- } while (__value != 0);
+ const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
+ const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4;
+ auto __cs = __digits + __c0_offset_mask + __alpha * 0x27;
+#ifdef _LIBCPP_LITTLE_ENDIAN
+ __cs = __builtin_bswap64(__cs);
+#endif
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ const auto __remaining = static_cast<size_t>(__p - __first);
+ __builtin_memcpy(__first, __chars.__data + __ncs_per_word - __remaining, __remaining);
return {__last, errc(0)};
}
};
|
You can test this locally with the following command:git-clang-format --diff origin/main HEAD --extensions h -- libcxx/include/__charconv/to_chars_integral.h --diff_from_common_commit
View the diff from clang-format here.diff --git a/libcxx/include/__charconv/to_chars_integral.h b/libcxx/include/__charconv/to_chars_integral.h
index 3fbbd6a8f..104319abe 100644
--- a/libcxx/include/__charconv/to_chars_integral.h
+++ b/libcxx/include/__charconv/to_chars_integral.h
@@ -140,7 +140,7 @@ struct _LIBCPP_HIDDEN __integral<2> {
constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030};
while (__value >= __divisor) {
const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
- auto __cs = __digits | __c0_offset_mask;
+ auto __cs = __digits | __c0_offset_mask;
#ifdef _LIBCPP_LITTLE_ENDIAN
__cs = __builtin_bswap64(__cs);
#endif
@@ -150,11 +150,11 @@ struct _LIBCPP_HIDDEN __integral<2> {
__builtin_memcpy(__p, __chars.__data, __ncs_per_word);
}
const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
- auto __cs = __digits | __c0_offset_mask;
+ auto __cs = __digits | __c0_offset_mask;
#ifdef _LIBCPP_LITTLE_ENDIAN
__cs = __builtin_bswap64(__cs);
#endif
- const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
const auto __remaining = static_cast<size_t>(__p - __first);
__builtin_memcpy(__first, __chars.__data + __ncs_per_word - __remaining, __remaining);
return {__last, errc(0)};
@@ -177,8 +177,8 @@ struct _LIBCPP_HIDDEN __integral<8> {
int __n = __width(__value);
if (__n > __cap)
return {__last, errc::value_too_large};
- __last = __first + __n;
- char* __p = __last;
+ __last = __first + __n;
+ char* __p = __last;
constexpr auto __ncs_per_word = 8u;
constexpr auto __divisor = 1u << (3 * __ncs_per_word);
@@ -186,7 +186,7 @@ struct _LIBCPP_HIDDEN __integral<8> {
constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030};
while (__value >= __divisor) {
const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
- auto __cs = __digits | __c0_offset_mask;
+ auto __cs = __digits | __c0_offset_mask;
#ifdef _LIBCPP_LITTLE_ENDIAN
__cs = __builtin_bswap64(__cs);
#endif
@@ -196,11 +196,11 @@ struct _LIBCPP_HIDDEN __integral<8> {
__builtin_memcpy(__p, __chars.__data, __ncs_per_word);
}
const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
- auto __cs = __digits | __c0_offset_mask;
+ auto __cs = __digits | __c0_offset_mask;
#ifdef _LIBCPP_LITTLE_ENDIAN
__cs = __builtin_bswap64(__cs);
#endif
- const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
+ const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs);
const auto __remaining = static_cast<size_t>(__p - __first);
__builtin_memcpy(__first, __chars.__data + __ncs_per_word - __remaining, __remaining);
return {__last, errc(0)};
@@ -235,8 +235,8 @@ struct _LIBCPP_HIDDEN __integral<16> {
constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030};
while (__value >= __divisor) {
const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
- const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4;
- auto __cs = __digits + __c0_offset_mask + __alpha * 0x27;
+ const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4;
+ auto __cs = __digits + __c0_offset_mask + __alpha * 0x27;
#ifdef _LIBCPP_LITTLE_ENDIAN
__cs = __builtin_bswap64(__cs);
#endif
@@ -246,8 +246,8 @@ struct _LIBCPP_HIDDEN __integral<16> {
__builtin_memcpy(__p, __chars.__data, __ncs_per_word);
}
const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask);
- const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4;
- auto __cs = __digits + __c0_offset_mask + __alpha * 0x27;
+ const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4;
+ auto __cs = __digits + __c0_offset_mask + __alpha * 0x27;
#ifdef _LIBCPP_LITTLE_ENDIAN
__cs = __builtin_bswap64(__cs);
#endif
|
davebayer
commented
Sep 13, 2026
Comment on lines
+142
to
+150
| const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask); | ||
| auto __cs = __digits | __c0_offset_mask; | ||
| #ifdef _LIBCPP_LITTLE_ENDIAN | ||
| __cs = __builtin_bswap64(__cs); | ||
| #endif | ||
| __value /= __divisor; | ||
| __p -= 4; | ||
| std::copy_n(&__base_2_lut[4 * __c], 4, __p); | ||
| __p -= __ncs_per_word; | ||
| const auto __chars = __builtin_bit_cast(__chars_storage<__ncs_per_word>, __cs); | ||
| __builtin_memcpy(__p, __chars.__data, __ncs_per_word); |
Contributor
Author
There was a problem hiding this comment.
The idea is to take bottom __ncs_per_word bits
0b0101'1010'0110'0110 -> 0b0110'0110
distribute each bit to start of a byte within a register
0b0110'0110 -> 0x0001'0100'0001'0100
add 0x30 ('0' character offset) to every byte (can be done using bit or)
0x0001'0100'0001'0100 + 0x3030'3030'3030'3030 -> 0x3031'3130'3031'3130
byteswap on little endian platforms
0x3031'3130'3031'3130 -> 0x3031'3130'3031'3130
and copy the result to memory
Comment on lines
+237
to
+239
| const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask); | ||
| const auto __alpha = ((__digits + __add_six_mask) & __high_bit_mask) >> 4; | ||
| auto __cs = __digits + __c0_offset_mask + __alpha * 0x27; |
Contributor
Author
There was a problem hiding this comment.
For base 16, '0' and 'a' have different offsets, so we need to account for that
Member
|
cc @eisenwave |
Zingam
reviewed
Sep 13, 2026
| constexpr auto __dep_mask = uint64_t{0x0101'0101'0101'0101}; | ||
| constexpr auto __c0_offset_mask = uint64_t{0x3030'3030'3030'3030}; | ||
| while (__value >= __divisor) { | ||
| const auto __digits = __builtin_elementwise_pdep(static_cast<uint64_t>(__value), __dep_mask); |
Contributor
There was a problem hiding this comment.
If __builtin_elementwise_pdep was introduced in Clang 23 as I think https://releases.llvm.org/23.1.0/tools/clang/docs/ReleaseNotes.html#non-comprehensive-list-of-changes-in-this-release we still need to support Clang 22 (officially) and some compiler like OpenXL which are on Clang 21 still.
For current support see: https://libcxx.llvm.org/#platform-and-compiler-support
Contributor
Author
There was a problem hiding this comment.
Absolutely, I just wanted to discuss first whether you want this thing in before spending the time on portability and more performance analysis
eisenwave
reviewed
Sep 13, 2026
eisenwave
left a comment
Contributor
There was a problem hiding this comment.
Seems like a reasonable optimization, but
- As Zingam mentioned, some of those builtins might need feature-tests.
__builtin_bit_castis also more recent thanstd::to_chars. - Any performance analysis should be cautious of the notoriously expensive early Zen BMI2 implementation, which is done in microcode. I imagine that attempting to use a
pdepinstruction on that architecture is strictly worse. https://godbolt.org/z/bvcjTo5sP https://godbolt.org/z/xE885Pzzc From what I can tell, we don't software-emulatepdepon Zen 2, so you're really getting that dramatic performance regression too. Anyway, that makes me think we want to keep the whole original implementation and only conditionally use the new one if it's not on Zen 2.
Contributor
|
Somewhat related to my feedback now: |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
4 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
I've come up with an optimization for
std::to_charsthat instead of character lookup computes the characters using SIMD within a register. The idea is to usestd::bit_decompressbehaviour to distribute the selected bits to the start of each byte in a register and add the0oracharacter offset using a single addition.However, this optimization is dependent on the
std::bit_decompressimplementation. For example, when BMI2 on x86 is available, it is compiled to a singlepdepinstruction.I've ran some benchmarks on my Intel i9-14900K with
-O3 -mbmi2flags using 8 characters/register and the results are:For small types, it's not worth it, but for 32-bit and wider types, we can get some substantial improvements.
Would you be interested in having this optimization when a
pext-like instruction is available?If yes, then I'll rework the implementation to change the chars/register based on the platform's word size as well.
Similar idea could be used to optimize
from_chars, too.I've used GPT-5.6 SOL to help me do the benchmarks & optimize the base 16 path.