From a7aa5fc637eccdb85318a1542061a21b9dfb0a65 Mon Sep 17 00:00:00 2001 From: cpplearner Date: Thu, 6 Apr 2023 15:26:41 +0800 Subject: [PATCH 1/2] Implement P2572R1 `std::format()` fill character allowances --- stl/inc/format | 62 +++++++++++++------ stl/inc/yvals_core.h | 1 + .../test.cpp | 2 + .../P0645R10_text_formatting_parsing/test.cpp | 6 ++ .../P0645R10_text_formatting_utf8/test.cpp | 14 +++++ 5 files changed, 65 insertions(+), 20 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 9c0121b7fe3..092af89e82e 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -168,38 +168,46 @@ concept _Parse_spec_callbacks = _Parse_align_callbacks<_Ty, _CharT> // clang-format on // _Decode_utf decodes UTF-8 or UTF-16 encoded unsigned char or wchar_t strings respectively -_NODISCARD constexpr const wchar_t* _Decode_utf(const wchar_t* _First, const wchar_t* _Last, char32_t& _Val) noexcept { +_NODISCARD constexpr const wchar_t* _Decode_utf( + const wchar_t* _First, const wchar_t* _Last, char32_t& _Val, bool& _Is_usv) noexcept { _STL_INTERNAL_CHECK(_First < _Last); _Val = static_cast(*_First); if (_Val < 0xD800) { + _Is_usv = true; return _First + 1; } else if (_Val <= 0xDBFF) { // 0xD800 <= _Val <= 0xDBFF: High surrogate if (_First + 1 == _Last) { - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _Last; } if (_First[1] < 0xDC00 || _First[1] > 0xDFFF) { // unpaired high surrogate - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } _Val = (_Val - 0xD800) << 10; _Val += _First[1] - 0xDC00; _Val += 0x10000; + _Is_usv = true; return _First + 2; } else if (_Val <= 0xDFFF) { // unpaired low surrogate - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } + _Is_usv = true; return _First + 1; } -_NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _Last, char32_t& _Val) noexcept { +_NODISCARD constexpr const char* _Decode_utf( + const char* _First, const char* _Last, char32_t& _Val, bool& _Is_usv) noexcept { _STL_INTERNAL_CHECK(_First < _Last); // Decode a UTF-8 encoded codepoint starting at _First and not exceeding _Last, returning // one past the end of the character decoded. Any invalid codepoints will result in @@ -216,6 +224,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La // we just sum the comparisons to get the number of trailing bytes. int _Num_bytes; if (_Val <= 0x7F) { + _Is_usv = true; return _First + 1; } else if (_Val >= 0xC2 && _Val <= 0xDF) { _Num_bytes = 2; @@ -225,7 +234,8 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La _Num_bytes = 4; } else { // definitely not valid - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } @@ -234,7 +244,8 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La // to check just the next byte here since we need to look for overlong sequences. // We want to return one past the end of a truncated sequence if everything is // otherwise valid, so we can't check if _First + _Num_bytes is off the end. - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _Last; } @@ -247,25 +258,29 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La // even though _Num_bytes would imply the next // codepoint starts at _First + 2, this is because // we don't consume trailing bytes of ill-formed subsequences - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } break; case 0xED: if (static_cast(_First[1]) > 0x9F) { - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } break; case 0xF0: if (static_cast(_First[1]) < 0x90) { - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } break; case 0xF4: if (static_cast(_First[1]) > 0x8F) { - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + 1; } break; @@ -291,20 +306,23 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La if (_First + _Idx >= _Last || static_cast(_First[_Idx]) < 0x80 || static_cast(_First[_Idx]) > 0xBF) { // truncated sequence - _Val = 0xFFFD; + _Val = 0xFFFD; + _Is_usv = false; return _First + _Idx; } // we know we're always in range due to the above check. _Val = (_Val << 6) | (static_cast(_First[_Idx]) & 0b11'1111u); } + _Is_usv = true; return _First + _Num_bytes; } _NODISCARD constexpr const char32_t* _Decode_utf( - const char32_t* _First, const char32_t* _Last, char32_t& _Val) noexcept { + const char32_t* _First, const char32_t* _Last, char32_t& _Val, bool& _Is_usv) noexcept { _STL_INTERNAL_CHECK(_First < _Last); (void) _Last; - _Val = *_First; + _Val = *_First; + _Is_usv = _Val < 0xD800 || (_Val > 0xDFFF && _Val <= 0x10FFFF); return _First + 1; } @@ -322,7 +340,8 @@ public: constexpr _Unicode_codepoint_iterator(const _CharT* _First_val, const _CharT* _Last_val) noexcept : _First(_First_val), _Last(_Last_val) { - _Next = _Decode_utf(_First, _Last, _Val); + bool _Is_usv; + _Next = _Decode_utf(_First, _Last, _Val, _Is_usv); } constexpr _Unicode_codepoint_iterator() = default; @@ -330,7 +349,8 @@ public: constexpr _Unicode_codepoint_iterator& operator++() noexcept { _First = _Next; if (_First != _Last) { - _Next = _Decode_utf(_First, _Last, _Val); + bool _Is_usv; + _Next = _Decode_utf(_First, _Last, _Val, _Is_usv); } return *this; @@ -916,9 +936,10 @@ private: _NODISCARD static constexpr int _Utf8_code_units_in_next_character( const char* const _First, const char* const _Last) noexcept { char32_t _Ch; - const auto _Next = _Decode_utf(_First, _Last, _Ch); + bool _Is_usv; + const auto _Next = _Decode_utf(_First, _Last, _Ch, _Is_usv); _STL_INTERNAL_CHECK(_Next - _First <= 4); - return static_cast(_Next - _First); + return _Is_usv ? static_cast(_Next - _First) : -1; } public: @@ -981,9 +1002,10 @@ public: _NODISCARD constexpr int _Units_in_next_character( const wchar_t* _First, const wchar_t* const _Last) const noexcept { char32_t _Ch; - const auto _Next = _Decode_utf(_First, _Last, _Ch); + bool _Is_usv; + const auto _Next = _Decode_utf(_First, _Last, _Ch, _Is_usv); _STL_INTERNAL_CHECK(_Next - _First <= 2); - return static_cast(_Next - _First); + return _Is_usv ? static_cast(_Next - _First) : -1; } }; diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index 893f73b9d08..ade4b4d1d13 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -285,6 +285,7 @@ // P2432R1 Fix istream_view // P2508R1 basic_format_string, format_string, wformat_string // P2520R0 move_iterator Should Be A Random-Access Iterator +// P2572R1 std::format Fill Character Allowances // P2588R3 barrier's Phase Completion Guarantees // P2602R2 Poison Pills Are Too Toxic // P2609R3 Relaxing Ranges Just A Smidge diff --git a/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp b/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp index 3c462e2937b..eac2577ce83 100644 --- a/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp +++ b/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp @@ -46,6 +46,8 @@ void test_parse_align() { {.expected_alignment = _Fmt_align::_Right, .expected_fill = "\x96\x7b"sv}); test_parse_helper(parse_align_fn, "\x92\x6e^X"sv, false, 3, {.expected_alignment = _Fmt_align::_Center, .expected_fill = "\x92\x6e"sv}); + + test_parse_helper(parse_align_fn, "\x92\x30 From e1de9d893dbf2d8262240d385cdaadc7f8a4d2b6 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 6 Apr 2023 14:18:48 -0700 Subject: [PATCH 2/2] Change out-param to struct, explain acronym. --- stl/inc/format | 101 +++++++++++++++++++++---------------------------- 1 file changed, 44 insertions(+), 57 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 092af89e82e..e65857a58e6 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -167,47 +167,47 @@ concept _Parse_spec_callbacks = _Parse_align_callbacks<_Ty, _CharT> }; // clang-format on +template +struct _Decode_utf_result { + const _CharT* _Next_ptr; + bool _Is_unicode_scalar_value; // Also _Is_usv below, see https://www.unicode.org/glossary/#unicode_scalar_value +}; + // _Decode_utf decodes UTF-8 or UTF-16 encoded unsigned char or wchar_t strings respectively -_NODISCARD constexpr const wchar_t* _Decode_utf( - const wchar_t* _First, const wchar_t* _Last, char32_t& _Val, bool& _Is_usv) noexcept { +_NODISCARD constexpr _Decode_utf_result _Decode_utf( + const wchar_t* _First, const wchar_t* _Last, char32_t& _Val) noexcept { _STL_INTERNAL_CHECK(_First < _Last); _Val = static_cast(*_First); if (_Val < 0xD800) { - _Is_usv = true; - return _First + 1; + return {_First + 1, true}; } else if (_Val <= 0xDBFF) { // 0xD800 <= _Val <= 0xDBFF: High surrogate if (_First + 1 == _Last) { - _Val = 0xFFFD; - _Is_usv = false; - return _Last; + _Val = 0xFFFD; + return {_Last, false}; } if (_First[1] < 0xDC00 || _First[1] > 0xDFFF) { // unpaired high surrogate - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } _Val = (_Val - 0xD800) << 10; _Val += _First[1] - 0xDC00; _Val += 0x10000; - _Is_usv = true; - return _First + 2; + return {_First + 2, true}; } else if (_Val <= 0xDFFF) { // unpaired low surrogate - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } - _Is_usv = true; - return _First + 1; + return {_First + 1, true}; } -_NODISCARD constexpr const char* _Decode_utf( - const char* _First, const char* _Last, char32_t& _Val, bool& _Is_usv) noexcept { +_NODISCARD constexpr _Decode_utf_result _Decode_utf( + const char* _First, const char* _Last, char32_t& _Val) noexcept { _STL_INTERNAL_CHECK(_First < _Last); // Decode a UTF-8 encoded codepoint starting at _First and not exceeding _Last, returning // one past the end of the character decoded. Any invalid codepoints will result in @@ -224,8 +224,7 @@ _NODISCARD constexpr const char* _Decode_utf( // we just sum the comparisons to get the number of trailing bytes. int _Num_bytes; if (_Val <= 0x7F) { - _Is_usv = true; - return _First + 1; + return {_First + 1, true}; } else if (_Val >= 0xC2 && _Val <= 0xDF) { _Num_bytes = 2; } else if (_Val >= 0xE0 && _Val <= 0xEF) { @@ -234,9 +233,8 @@ _NODISCARD constexpr const char* _Decode_utf( _Num_bytes = 4; } else { // definitely not valid - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } if (_First + 1 == _Last) { @@ -244,9 +242,8 @@ _NODISCARD constexpr const char* _Decode_utf( // to check just the next byte here since we need to look for overlong sequences. // We want to return one past the end of a truncated sequence if everything is // otherwise valid, so we can't check if _First + _Num_bytes is off the end. - _Val = 0xFFFD; - _Is_usv = false; - return _Last; + _Val = 0xFFFD; + return {_Last, false}; } switch (_Val) { @@ -258,30 +255,26 @@ _NODISCARD constexpr const char* _Decode_utf( // even though _Num_bytes would imply the next // codepoint starts at _First + 2, this is because // we don't consume trailing bytes of ill-formed subsequences - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } break; case 0xED: if (static_cast(_First[1]) > 0x9F) { - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } break; case 0xF0: if (static_cast(_First[1]) < 0x90) { - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } break; case 0xF4: if (static_cast(_First[1]) > 0x8F) { - _Val = 0xFFFD; - _Is_usv = false; - return _First + 1; + _Val = 0xFFFD; + return {_First + 1, false}; } break; } @@ -306,24 +299,22 @@ _NODISCARD constexpr const char* _Decode_utf( if (_First + _Idx >= _Last || static_cast(_First[_Idx]) < 0x80 || static_cast(_First[_Idx]) > 0xBF) { // truncated sequence - _Val = 0xFFFD; - _Is_usv = false; - return _First + _Idx; + _Val = 0xFFFD; + return {_First + _Idx, false}; } // we know we're always in range due to the above check. _Val = (_Val << 6) | (static_cast(_First[_Idx]) & 0b11'1111u); } - _Is_usv = true; - return _First + _Num_bytes; + return {_First + _Num_bytes, true}; } -_NODISCARD constexpr const char32_t* _Decode_utf( - const char32_t* _First, const char32_t* _Last, char32_t& _Val, bool& _Is_usv) noexcept { +_NODISCARD constexpr _Decode_utf_result _Decode_utf( + const char32_t* _First, const char32_t* _Last, char32_t& _Val) noexcept { _STL_INTERNAL_CHECK(_First < _Last); (void) _Last; - _Val = *_First; - _Is_usv = _Val < 0xD800 || (_Val > 0xDFFF && _Val <= 0x10FFFF); - return _First + 1; + _Val = *_First; + const bool _Is_usv = _Val < 0xD800 || (_Val > 0xDFFF && _Val <= 0x10FFFF); + return {_First + 1, _Is_usv}; } template @@ -340,8 +331,7 @@ public: constexpr _Unicode_codepoint_iterator(const _CharT* _First_val, const _CharT* _Last_val) noexcept : _First(_First_val), _Last(_Last_val) { - bool _Is_usv; - _Next = _Decode_utf(_First, _Last, _Val, _Is_usv); + _Next = _Decode_utf(_First, _Last, _Val)._Next_ptr; } constexpr _Unicode_codepoint_iterator() = default; @@ -349,8 +339,7 @@ public: constexpr _Unicode_codepoint_iterator& operator++() noexcept { _First = _Next; if (_First != _Last) { - bool _Is_usv; - _Next = _Decode_utf(_First, _Last, _Val, _Is_usv); + _Next = _Decode_utf(_First, _Last, _Val)._Next_ptr; } return *this; @@ -936,8 +925,7 @@ private: _NODISCARD static constexpr int _Utf8_code_units_in_next_character( const char* const _First, const char* const _Last) noexcept { char32_t _Ch; - bool _Is_usv; - const auto _Next = _Decode_utf(_First, _Last, _Ch, _Is_usv); + const auto [_Next, _Is_usv] = _Decode_utf(_First, _Last, _Ch); _STL_INTERNAL_CHECK(_Next - _First <= 4); return _Is_usv ? static_cast(_Next - _First) : -1; } @@ -1002,8 +990,7 @@ public: _NODISCARD constexpr int _Units_in_next_character( const wchar_t* _First, const wchar_t* const _Last) const noexcept { char32_t _Ch; - bool _Is_usv; - const auto _Next = _Decode_utf(_First, _Last, _Ch, _Is_usv); + const auto [_Next, _Is_usv] = _Decode_utf(_First, _Last, _Ch); _STL_INTERNAL_CHECK(_Next - _First <= 2); return _Is_usv ? static_cast(_Next - _First) : -1; }