diff --git a/stl/inc/format b/stl/inc/format index 9c0121b7fe3..e65857a58e6 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -167,39 +167,47 @@ concept _Parse_spec_callbacks = _Parse_align_callbacks<_Ty, _CharT> }; // clang-format on +template +struct _Decode_utf_result { + const _CharT* _Next_ptr; + bool _Is_unicode_scalar_value; // Also _Is_usv below, see https://www.unicode.org/glossary/#unicode_scalar_value +}; + // _Decode_utf decodes UTF-8 or UTF-16 encoded unsigned char or wchar_t strings respectively -_NODISCARD constexpr const wchar_t* _Decode_utf(const wchar_t* _First, const wchar_t* _Last, char32_t& _Val) noexcept { +_NODISCARD constexpr _Decode_utf_result _Decode_utf( + const wchar_t* _First, const wchar_t* _Last, char32_t& _Val) noexcept { _STL_INTERNAL_CHECK(_First < _Last); _Val = static_cast(*_First); if (_Val < 0xD800) { - return _First + 1; + return {_First + 1, true}; } else if (_Val <= 0xDBFF) { // 0xD800 <= _Val <= 0xDBFF: High surrogate if (_First + 1 == _Last) { _Val = 0xFFFD; - return _Last; + return {_Last, false}; } if (_First[1] < 0xDC00 || _First[1] > 0xDFFF) { // unpaired high surrogate _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } _Val = (_Val - 0xD800) << 10; _Val += _First[1] - 0xDC00; _Val += 0x10000; - return _First + 2; + return {_First + 2, true}; } else if (_Val <= 0xDFFF) { // unpaired low surrogate _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } - return _First + 1; + return {_First + 1, true}; } -_NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _Last, char32_t& _Val) noexcept { +_NODISCARD constexpr _Decode_utf_result _Decode_utf( + const char* _First, const char* _Last, char32_t& _Val) noexcept { _STL_INTERNAL_CHECK(_First < _Last); // Decode a UTF-8 encoded codepoint starting at _First and not exceeding _Last, returning // one past the end of the character decoded. Any invalid codepoints will result in @@ -216,7 +224,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La // we just sum the comparisons to get the number of trailing bytes. int _Num_bytes; if (_Val <= 0x7F) { - return _First + 1; + return {_First + 1, true}; } else if (_Val >= 0xC2 && _Val <= 0xDF) { _Num_bytes = 2; } else if (_Val >= 0xE0 && _Val <= 0xEF) { @@ -226,7 +234,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La } else { // definitely not valid _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } if (_First + 1 == _Last) { @@ -235,7 +243,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La // We want to return one past the end of a truncated sequence if everything is // otherwise valid, so we can't check if _First + _Num_bytes is off the end. _Val = 0xFFFD; - return _Last; + return {_Last, false}; } switch (_Val) { @@ -248,25 +256,25 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La // codepoint starts at _First + 2, this is because // we don't consume trailing bytes of ill-formed subsequences _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } break; case 0xED: if (static_cast(_First[1]) > 0x9F) { _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } break; case 0xF0: if (static_cast(_First[1]) < 0x90) { _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } break; case 0xF4: if (static_cast(_First[1]) > 0x8F) { _Val = 0xFFFD; - return _First + 1; + return {_First + 1, false}; } break; } @@ -292,20 +300,21 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La || static_cast(_First[_Idx]) > 0xBF) { // truncated sequence _Val = 0xFFFD; - return _First + _Idx; + return {_First + _Idx, false}; } // we know we're always in range due to the above check. _Val = (_Val << 6) | (static_cast(_First[_Idx]) & 0b11'1111u); } - return _First + _Num_bytes; + return {_First + _Num_bytes, true}; } -_NODISCARD constexpr const char32_t* _Decode_utf( +_NODISCARD constexpr _Decode_utf_result _Decode_utf( const char32_t* _First, const char32_t* _Last, char32_t& _Val) noexcept { _STL_INTERNAL_CHECK(_First < _Last); (void) _Last; - _Val = *_First; - return _First + 1; + _Val = *_First; + const bool _Is_usv = _Val < 0xD800 || (_Val > 0xDFFF && _Val <= 0x10FFFF); + return {_First + 1, _Is_usv}; } template @@ -322,7 +331,7 @@ public: constexpr _Unicode_codepoint_iterator(const _CharT* _First_val, const _CharT* _Last_val) noexcept : _First(_First_val), _Last(_Last_val) { - _Next = _Decode_utf(_First, _Last, _Val); + _Next = _Decode_utf(_First, _Last, _Val)._Next_ptr; } constexpr _Unicode_codepoint_iterator() = default; @@ -330,7 +339,7 @@ public: constexpr _Unicode_codepoint_iterator& operator++() noexcept { _First = _Next; if (_First != _Last) { - _Next = _Decode_utf(_First, _Last, _Val); + _Next = _Decode_utf(_First, _Last, _Val)._Next_ptr; } return *this; @@ -916,9 +925,9 @@ private: _NODISCARD static constexpr int _Utf8_code_units_in_next_character( const char* const _First, const char* const _Last) noexcept { char32_t _Ch; - const auto _Next = _Decode_utf(_First, _Last, _Ch); + const auto [_Next, _Is_usv] = _Decode_utf(_First, _Last, _Ch); _STL_INTERNAL_CHECK(_Next - _First <= 4); - return static_cast(_Next - _First); + return _Is_usv ? static_cast(_Next - _First) : -1; } public: @@ -981,9 +990,9 @@ public: _NODISCARD constexpr int _Units_in_next_character( const wchar_t* _First, const wchar_t* const _Last) const noexcept { char32_t _Ch; - const auto _Next = _Decode_utf(_First, _Last, _Ch); + const auto [_Next, _Is_usv] = _Decode_utf(_First, _Last, _Ch); _STL_INTERNAL_CHECK(_Next - _First <= 2); - return static_cast(_Next - _First); + return _Is_usv ? static_cast(_Next - _First) : -1; } }; diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index 893f73b9d08..ade4b4d1d13 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -285,6 +285,7 @@ // P2432R1 Fix istream_view // P2508R1 basic_format_string, format_string, wformat_string // P2520R0 move_iterator Should Be A Random-Access Iterator +// P2572R1 std::format Fill Character Allowances // P2588R3 barrier's Phase Completion Guarantees // P2602R2 Poison Pills Are Too Toxic // P2609R3 Relaxing Ranges Just A Smidge diff --git a/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp b/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp index 3c462e2937b..eac2577ce83 100644 --- a/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp +++ b/tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp @@ -46,6 +46,8 @@ void test_parse_align() { {.expected_alignment = _Fmt_align::_Right, .expected_fill = "\x96\x7b"sv}); test_parse_helper(parse_align_fn, "\x92\x6e^X"sv, false, 3, {.expected_alignment = _Fmt_align::_Center, .expected_fill = "\x92\x6e"sv}); + + test_parse_helper(parse_align_fn, "\x92\x30