Skip to content

Implement P2572R1 std::format() fill character allowances #3629

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Stephan T. Lavavej (StephanTLavavej) merged 2 commits into from
Apr 14, 2023
Merged

Implement P2572R1 std::format() fill character allowances #3629

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a7aa5fc
Implement P2572R1 `std::format()` fill character allowances
cpplearner Apr 6, 2023
e1de9d8
Change out-param to struct, explain acronym.
StephanTLavavej Apr 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 35 additions & 26 deletions stl/inc/format
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,39 +167,47 @@ concept _Parse_spec_callbacks = _Parse_align_callbacks<_Ty, _CharT>
};
// clang-format on

template <class _CharT>
struct _Decode_utf_result {
const _CharT* _Next_ptr;
bool _Is_unicode_scalar_value; // Also _Is_usv below, see https://www.unicode.org/glossary/#unicode_scalar_value
};

// _Decode_utf decodes UTF-8 or UTF-16 encoded unsigned char or wchar_t strings respectively
_NODISCARD constexpr const wchar_t* _Decode_utf(const wchar_t* _First, const wchar_t* _Last, char32_t& _Val) noexcept {
_NODISCARD constexpr _Decode_utf_result<wchar_t> _Decode_utf(
const wchar_t* _First, const wchar_t* _Last, char32_t& _Val) noexcept {
_STL_INTERNAL_CHECK(_First < _Last);
_Val = static_cast<char32_t>(*_First);
if (_Val < 0xD800) {
return _First + 1;
return {_First + 1, true};
} else if (_Val <= 0xDBFF) {
// 0xD800 <= _Val <= 0xDBFF: High surrogate
if (_First + 1 == _Last) {
_Val = 0xFFFD;
return _Last;
return {_Last, false};
}

if (_First[1] < 0xDC00 || _First[1] > 0xDFFF) {
// unpaired high surrogate
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}

_Val = (_Val - 0xD800) << 10;
_Val += _First[1] - 0xDC00;
_Val += 0x10000;
return _First + 2;
return {_First + 2, true};
} else if (_Val <= 0xDFFF) {
// unpaired low surrogate
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}

return _First + 1;
return {_First + 1, true};
}

_NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _Last, char32_t& _Val) noexcept {
_NODISCARD constexpr _Decode_utf_result<char> _Decode_utf(
const char* _First, const char* _Last, char32_t& _Val) noexcept {
_STL_INTERNAL_CHECK(_First < _Last);
// Decode a UTF-8 encoded codepoint starting at _First and not exceeding _Last, returning
// one past the end of the character decoded. Any invalid codepoints will result in
Expand All @@ -216,7 +224,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La
// we just sum the comparisons to get the number of trailing bytes.
int _Num_bytes;
if (_Val <= 0x7F) {
return _First + 1;
return {_First + 1, true};
} else if (_Val >= 0xC2 && _Val <= 0xDF) {
_Num_bytes = 2;
} else if (_Val >= 0xE0 && _Val <= 0xEF) {
Expand All @@ -226,7 +234,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La
} else {
// definitely not valid
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}

if (_First + 1 == _Last) {
Expand All @@ -235,7 +243,7 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La
// We want to return one past the end of a truncated sequence if everything is
// otherwise valid, so we can't check if _First + _Num_bytes is off the end.
_Val = 0xFFFD;
return _Last;
return {_Last, false};
}

switch (_Val) {
Expand All @@ -248,25 +256,25 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La
// codepoint starts at _First + 2, this is because
// we don't consume trailing bytes of ill-formed subsequences
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}
break;
case 0xED:
if (static_cast<unsigned char>(_First[1]) > 0x9F) {
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}
break;
case 0xF0:
if (static_cast<unsigned char>(_First[1]) < 0x90) {
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}
break;
case 0xF4:
if (static_cast<unsigned char>(_First[1]) > 0x8F) {
_Val = 0xFFFD;
return _First + 1;
return {_First + 1, false};
}
break;
}
Expand All @@ -292,20 +300,21 @@ _NODISCARD constexpr const char* _Decode_utf(const char* _First, const char* _La
|| static_cast<unsigned char>(_First[_Idx]) > 0xBF) {
// truncated sequence
_Val = 0xFFFD;
return _First + _Idx;
return {_First + _Idx, false};
}
// we know we're always in range due to the above check.
_Val = (_Val << 6) | (static_cast<unsigned char>(_First[_Idx]) & 0b11'1111u);
}
return _First + _Num_bytes;
return {_First + _Num_bytes, true};
}

_NODISCARD constexpr const char32_t* _Decode_utf(
_NODISCARD constexpr _Decode_utf_result<char32_t> _Decode_utf(
const char32_t* _First, const char32_t* _Last, char32_t& _Val) noexcept {
_STL_INTERNAL_CHECK(_First < _Last);
(void) _Last;
_Val = *_First;
return _First + 1;
_Val = *_First;
const bool _Is_usv = _Val < 0xD800 || (_Val > 0xDFFF && _Val <= 0x10FFFF);
return {_First + 1, _Is_usv};
}

template <class _CharT>
Expand All @@ -322,15 +331,15 @@ public:

constexpr _Unicode_codepoint_iterator(const _CharT* _First_val, const _CharT* _Last_val) noexcept
: _First(_First_val), _Last(_Last_val) {
_Next = _Decode_utf(_First, _Last, _Val);
_Next = _Decode_utf(_First, _Last, _Val)._Next_ptr;
}

constexpr _Unicode_codepoint_iterator() = default;

constexpr _Unicode_codepoint_iterator& operator++() noexcept {
_First = _Next;
if (_First != _Last) {
_Next = _Decode_utf(_First, _Last, _Val);
_Next = _Decode_utf(_First, _Last, _Val)._Next_ptr;
}

return *this;
Expand Down Expand Up @@ -916,9 +925,9 @@ private:
_NODISCARD static constexpr int _Utf8_code_units_in_next_character(
const char* const _First, const char* const _Last) noexcept {
char32_t _Ch;
const auto _Next = _Decode_utf(_First, _Last, _Ch);
const auto [_Next, _Is_usv] = _Decode_utf(_First, _Last, _Ch);
_STL_INTERNAL_CHECK(_Next - _First <= 4);
return static_cast<int>(_Next - _First);
return _Is_usv ? static_cast<int>(_Next - _First) : -1;
}

public:
Expand Down Expand Up @@ -981,9 +990,9 @@ public:
_NODISCARD constexpr int _Units_in_next_character(
const wchar_t* _First, const wchar_t* const _Last) const noexcept {
char32_t _Ch;
const auto _Next = _Decode_utf(_First, _Last, _Ch);
const auto [_Next, _Is_usv] = _Decode_utf(_First, _Last, _Ch);
_STL_INTERNAL_CHECK(_Next - _First <= 2);
return static_cast<int>(_Next - _First);
return _Is_usv ? static_cast<int>(_Next - _First) : -1;
}
};

Expand Down
1 change: 1 addition & 0 deletions stl/inc/yvals_core.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,7 @@
// P2432R1 Fix istream_view
// P2508R1 basic_format_string, format_string, wformat_string
// P2520R0 move_iterator<T*> Should Be A Random-Access Iterator
// P2572R1 std::format Fill Character Allowances
// P2588R3 barrier's Phase Completion Guarantees
// P2602R2 Poison Pills Are Too Toxic
// P2609R3 Relaxing Ranges Just A Smidge
Expand Down
2 changes: 2 additions & 0 deletions tests/std/tests/P0645R10_text_formatting_legacy_text_encoding/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ void test_parse_align() {
{.expected_alignment = _Fmt_align::_Right, .expected_fill = "\x96\x7b"sv});
test_parse_helper(parse_align_fn, "\x92\x6e^X"sv, false, 3,
{.expected_alignment = _Fmt_align::_Center, .expected_fill = "\x92\x6e"sv});

test_parse_helper(parse_align_fn, "\x92\x30<X"sv, true);
}

void test_width_estimation() {
Expand Down
6 changes: 6 additions & 0 deletions tests/std/tests/P0645R10_text_formatting_parsing/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,12 @@ bool test_parse_align() {
test_parse_helper(parse_align_fn, L"\U0001F3C8^X"sv, false, 3,
{.expected_alignment = _Fmt_align::_Center, .expected_fill = L"\U0001F3C8"sv});
}

// test invalid fill characters
{
test_parse_helper(parse_align_fn, L"\xD800<X"sv, true);
test_parse_helper(parse_align_fn, L"\xDC00<X"sv, true);
}
}

return true;
Expand Down
14 changes: 14 additions & 0 deletions tests/std/tests/P0645R10_text_formatting_utf8/test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,20 @@ void test_parse_align() {
test_parse_helper(parse_align_fn, "\U0001f3c8^X"sv, false, 5,
{.expected_alignment = _Fmt_align::_Center, .expected_fill = "\U0001f3c8"sv});
}

{
test_parse_helper(parse_align_fn, "\xC0\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xF5\x80\x80\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xE0\x80\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xED\xA0\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xF0\x80\x80\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xF4\x90\x80\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xE1\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xE1\x7F\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xE1\xC0\x80<X"sv, true);
test_parse_helper(parse_align_fn, "\xE1\x80\x7F<X"sv, true);
test_parse_helper(parse_align_fn, "\xE1\x80\xC0<X"sv, true);
}
}

template <class CharT>
Expand Down