diff --git a/stl/inc/format b/stl/inc/format index fdcf65e7908..cfe4426ff92 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -978,22 +978,46 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { #endif // ^^^ EDG workaround ^^^ } -inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4928 [format.string.std]/12 - 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, 0xFE10u, - 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, 0x20000u, - 0x2FFFEu, 0x30000u, 0x3FFFEu}; +// Generated per N4950 [format.string.std]/13, by tools/unicode_properties_parse/format_width_estimate_intervals.cpp +// in the https://github.com/microsoft/stl repository. +inline constexpr char32_t _Width_estimate_intervals_v2[] = { // + 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, + 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, + 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, + 0x26F6u, 0x26FAu, 0x26FBu, 0x26FDu, 0x26FEu, 0x2705u, 0x2706u, 0x270Au, 0x270Cu, 0x2728u, 0x2729u, 0x274Cu, 0x274Du, + 0x274Eu, 0x274Fu, 0x2753u, 0x2756u, 0x2757u, 0x2758u, 0x2795u, 0x2798u, 0x27B0u, 0x27B1u, 0x27BFu, 0x27C0u, 0x2B1Bu, + 0x2B1Du, 0x2B50u, 0x2B51u, 0x2B55u, 0x2B56u, 0x2E80u, 0x2E9Au, 0x2E9Bu, 0x2EF4u, 0x2F00u, 0x2FD6u, 0x2FF0u, 0x2FFCu, + 0x3000u, 0x303Fu, 0x3041u, 0x3097u, 0x3099u, 0x3100u, 0x3105u, 0x3130u, 0x3131u, 0x318Fu, 0x3190u, 0x31E4u, 0x31F0u, + 0x321Fu, 0x3220u, 0x3248u, 0x3250u, 0xA48Du, 0xA490u, 0xA4C7u, 0xA960u, 0xA97Du, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, + 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE53u, 0xFE54u, 0xFE67u, 0xFE68u, 0xFE6Cu, 0xFF01u, 0xFF61u, 0xFFE0u, 0xFFE7u, + 0x16FE0u, 0x16FE5u, 0x16FF0u, 0x16FF2u, 0x17000u, 0x187F8u, 0x18800u, 0x18CD6u, 0x18D00u, 0x18D09u, 0x1AFF0u, + 0x1AFF4u, 0x1AFF5u, 0x1AFFCu, 0x1AFFDu, 0x1AFFFu, 0x1B000u, 0x1B123u, 0x1B132u, 0x1B133u, 0x1B150u, 0x1B153u, + 0x1B155u, 0x1B156u, 0x1B164u, 0x1B168u, 0x1B170u, 0x1B2FCu, 0x1F004u, 0x1F005u, 0x1F0CFu, 0x1F0D0u, 0x1F18Eu, + 0x1F18Fu, 0x1F191u, 0x1F19Bu, 0x1F200u, 0x1F203u, 0x1F210u, 0x1F23Cu, 0x1F240u, 0x1F249u, 0x1F250u, 0x1F252u, + 0x1F260u, 0x1F266u, 0x1F300u, 0x1F650u, 0x1F680u, 0x1F6C6u, 0x1F6CCu, 0x1F6CDu, 0x1F6D0u, 0x1F6D3u, 0x1F6D5u, + 0x1F6D8u, 0x1F6DCu, 0x1F6E0u, 0x1F6EBu, 0x1F6EDu, 0x1F6F4u, 0x1F6FDu, 0x1F7E0u, 0x1F7ECu, 0x1F7F0u, 0x1F7F1u, + 0x1F900u, 0x1FA00u, 0x1FA70u, 0x1FA7Du, 0x1FA80u, 0x1FA89u, 0x1FA90u, 0x1FABEu, 0x1FABFu, 0x1FAC6u, 0x1FACEu, + 0x1FADCu, 0x1FAE0u, 0x1FAE9u, 0x1FAF0u, 0x1FAF9u, 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu}; _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { - // Computes the width estimation for Unicode characters from N4928 [format.string.std]/12 - int _Result = 1; - for (const auto& _Bound : _Width_estimate_intervals) { - if (_Ch < _Bound) { - return _Result; + // Computes the width estimation for Unicode characters from N4950 [format.string.std]/13 + // The two branches are functionally equivalent; `12` is chosen for performance here. + constexpr char32_t _Linear_search_threshold = _Width_estimate_intervals_v2[12]; + if (_Ch < _Linear_search_threshold) { + int _Result = 1; + for (const auto& _Bound : _Width_estimate_intervals_v2) { + if (_Ch < _Bound) { + return _Result; + } + _Result ^= 0b11u; // Flip between 1 and 2 on each iteration } - _Result ^= 0b11u; // Flip between 1 and 2 on each iteration + return 1; + } else { + const ptrdiff_t _Upper_bound_index = + _STD upper_bound(_Width_estimate_intervals_v2, _STD end(_Width_estimate_intervals_v2), _Ch) + - _Width_estimate_intervals_v2; + return 1 + (_Upper_bound_index & 1); } - - return 1; } template diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index c0ee672319d..aa8ad1a23ee 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -301,6 +301,7 @@ // P2602R2 Poison Pills Are Too Toxic // P2609R3 Relaxing Ranges Just A Smidge // P2655R3 common_reference_t Of reference_wrapper Should Be A Reference Type +// P2675R1 Improving std::format's Width Estimation // P2711R1 Making Multi-Param Constructors Of Views explicit // P2736R2 Referencing The Unicode Standard // P2770R0 Stashing Stashing Iterators For Proper Flattening diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 7b912c8a6b0..08702b8f33e 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -177,6 +177,9 @@ std/ranges/range.access/size.pass.cpp FAIL # libc++ doesn't implement P2652R2 "Disallowing User Specialization Of allocator_traits" std/utilities/memory/allocator.traits/allocate_at_least.pass.cpp FAIL +# libc++ doesn't implement P2675R1 "Improving std::format's Width Estimation" +std/utilities/format/format.functions/unicode.pass.cpp FAIL + # libc++ doesn't implement P2770R0 "Stashing stashing iterators for proper flattening" std/ranges/range.adaptors/range.join.view/end.pass.cpp FAIL std/ranges/range.adaptors/range.join.view/iterator/ctor.other.pass.cpp FAIL diff --git a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp index c8de404068b..db01cedbefa 100644 --- a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp +++ b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp @@ -69,7 +69,7 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\x58"), 1}, {TYPED_LITERAL(CharT, "x\x58"), 2}, - // test the boundaries of the intervals defined in n4885 [format.string.std]/11 + // Test N4950 [format.string.std]/13 {TYPED_LITERAL(CharT, "\u10ff\x58"), 2}, {TYPED_LITERAL(CharT, "\u1100\x58"), 3}, {TYPED_LITERAL(CharT, "\u115f\x58"), 3}, @@ -82,8 +82,6 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\u2e80\x58"), 3}, {TYPED_LITERAL(CharT, "\u303e\x58"), 3}, {TYPED_LITERAL(CharT, "\u303f\x58"), 2}, - {TYPED_LITERAL(CharT, "\u3040\x58"), 3}, - {TYPED_LITERAL(CharT, "\ua4cf\x58"), 3}, {TYPED_LITERAL(CharT, "\ua4d0\x58"), 2}, {TYPED_LITERAL(CharT, "\uabff\x58"), 2}, {TYPED_LITERAL(CharT, "\uac00\x58"), 3}, @@ -91,7 +89,7 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ud7a4\x58"), 2}, {TYPED_LITERAL(CharT, "\ud7ff\x58"), 2}, - // skip over the surrogate pair range (\ud800-\udfff) + // Skip over the surrogate pair range (\ud800-\udfff) {TYPED_LITERAL(CharT, "\ue000\x58"), 2}, {TYPED_LITERAL(CharT, "\uf8ff\x58"), 2}, @@ -104,10 +102,8 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ufe1a\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe2f\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe30\x58"), 3}, - {TYPED_LITERAL(CharT, "\ufe6f\x58"), 3}, {TYPED_LITERAL(CharT, "\ufe70\x58"), 2}, {TYPED_LITERAL(CharT, "\ufeff\x58"), 2}, - {TYPED_LITERAL(CharT, "\uff00\x58"), 3}, {TYPED_LITERAL(CharT, "\uff60\x58"), 3}, {TYPED_LITERAL(CharT, "\uff61\x58"), 2}, {TYPED_LITERAL(CharT, "\uffdf\x58"), 2}, @@ -143,8 +139,58 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ufe40\u2000\ufe40\x58"), 6}, {TYPED_LITERAL(CharT, "\ufe40\ufe40\u2000\x58"), 6}, {TYPED_LITERAL(CharT, "\ufe40\ufe40\ufe40\x58"), 7}, + {TYPED_LITERAL(CharT, "\u0061\u200D\x58"), 2}, {TYPED_LITERAL(CharT, "\u0061\U0001F1E6\U0001F1E7\u200D\U0001F1E8\u0062\x58"), 5}, + + // Test some codepoints affected by P2675R1 + // Used to be 2, now 1: + {TYPED_LITERAL(CharT, "\u2e9a\x58"), 2}, + {TYPED_LITERAL(CharT, "\u2ef4\u2eff\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2fd6\u2fef\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2ffc\u2fff\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3040\x58"), 2}, + {TYPED_LITERAL(CharT, "\u3097\u3098\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3100\u3104\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3130\x58"), 2}, + {TYPED_LITERAL(CharT, "\u318f\x58"), 2}, + {TYPED_LITERAL(CharT, "\u31e4\u31ef\x58"), 3}, + {TYPED_LITERAL(CharT, "\u321f\x58"), 2}, + {TYPED_LITERAL(CharT, "\u3248\u324f\x58"), 3}, + {TYPED_LITERAL(CharT, "\ua48d\ua48f\x58"), 3}, + {TYPED_LITERAL(CharT, "\ua4c7\ua4cf\x58"), 3}, + {TYPED_LITERAL(CharT, "\ufe53\x58"), 2}, + {TYPED_LITERAL(CharT, "\ufe67\x58"), 2}, + {TYPED_LITERAL(CharT, "\ufe6c\ufe6f\x58"), 3}, + {TYPED_LITERAL(CharT, "\uff00\x58"), 2}, + // Used to be 1, now 2: + {TYPED_LITERAL(CharT, "\u23f0\x58"), 3}, + {TYPED_LITERAL(CharT, "\u23f3\x58"), 3}, + {TYPED_LITERAL(CharT, "\u267f\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2693\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26a1\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26ce\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26d4\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26ea\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26f5\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26fa\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26fd\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2705\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2728\x58"), 3}, + {TYPED_LITERAL(CharT, "\u274c\x58"), 3}, + {TYPED_LITERAL(CharT, "\u274e\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2757\x58"), 3}, + {TYPED_LITERAL(CharT, "\u27b0\x58"), 3}, + {TYPED_LITERAL(CharT, "\u27bf\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2b50\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2b55\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001b132\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001b155\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f004\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f0cf\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f18e\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f6cc\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f7f0\x58"), 3}, }; for (const auto& test : test_cases) { diff --git a/tools/unicode_properties_parse/.gitignore b/tools/unicode_properties_parse/.gitignore index 94b365a6e24..4fb9d76d452 100644 --- a/tools/unicode_properties_parse/.gitignore +++ b/tools/unicode_properties_parse/.gitignore @@ -7,5 +7,6 @@ emoji-data.txt DerivedCoreProperties.txt DerivedGeneralCategory.txt +EastAsianWidth.txt GraphemeBreakProperty.txt GraphemeBreakTest.txt diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp new file mode 100644 index 00000000000..06946779b7d --- /dev/null +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// The following code generates data for `_Width_estimate_intervals_v2` in . + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +using namespace std; + +void verify(bool test, const char* msg, source_location loc = source_location::current()) { + if (!test) { + cerr << "Error at line " << loc.line() << ": " << msg << endl; + exit(EXIT_FAILURE); + } +} +constexpr const char* impl_assertion_failed = "impl assertion failed"; + +struct range_u { + uint32_t from; + uint32_t to; + constexpr range_u(uint32_t f, uint32_t t) : from(f), to(t) {} + constexpr explicit range_u(uint32_t v) : from(v), to(v) {} +}; + +enum class width_u : bool { is_1 = false, is_2 = true }; + +class table_u { +private: + // A valid Unicode code point won't exceed `max_u`. + static constexpr uint32_t max_u = 0x10'ffff; + vector table; + +public: + table_u() : table(max_u + 1, width_u::is_1) {} + + void fill_range(const range_u rng, const width_u width) { + const auto [from, to] = rng; + verify(from <= to, impl_assertion_failed); + verify(to <= max_u, impl_assertion_failed); + for (uint32_t u = from; u <= to; ++u) { + table[u] = width; + } + } + + void print_intervals() const { + // Print table for `_Width_estimate_intervals_v2`. + int c = 0; + width_u last = table[0]; + for (uint32_t u = 0; u <= max_u; ++u) { + if (table[u] != last) { + cout << "0x" << hex << uppercase << u << "u, "; + if (++c == 12) { + c = 0; + cout << endl; + } + } + last = table[u]; + } + cout << endl; + } + + void print_clusters_1_vs_2(const table_u& other) const { + vector cluster_table(max_u + 1, false); + for (uint32_t u = 0; u <= max_u; ++u) { + if (table[u] == width_u::is_1 && other.table[u] == width_u::is_2) { + cluster_table[u] = true; + } + } + + for (uint32_t u = 0; u <= max_u; ++u) { + if (cluster_table[u]) { + const uint32_t from = u; + uint32_t to = from; + while (to + 1 <= max_u && cluster_table[to + 1]) { + ++to; + } + if (from == to) { + cout << hex << uppercase << "U+" << from << endl; + } else { + cout << hex << uppercase << "U+" << from << "..U+" << to << endl; + } + u = to; + } + } + } +}; + +table_u get_table_cpp20() { + static constexpr range_u std_wide_ranges_cpp20[]{ + {0x1100, 0x115F}, + {0x2329, 0x232A}, + {0x2E80, 0x303E}, + {0x3040, 0xA4CF}, + {0xAC00, 0xD7A3}, + {0xF900, 0xFAFF}, + {0xFE10, 0xFE19}, + {0xFE30, 0xFE6F}, + {0xFF00, 0xFF60}, + {0xFFE0, 0xFFE6}, + {0x1F300, 0x1F64F}, + {0x1F900, 0x1F9FF}, + {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD}, + }; + + table_u table; + for (const range_u& rng : std_wide_ranges_cpp20) { + table.fill_range(rng, width_u::is_2); + } + return table; +} + +// Read data from "EastAsianWidth.txt". +// The latest version can be found at: +// https://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt +// The current implementation works for: +// https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// To make this function work, the file should not contain a BOM. +table_u read_from(ifstream& source) { + table_u table; + + // "The unassigned code points in the following blocks default to "W":" + static constexpr range_u default_wide_ranges[]{ + {0x4E00, 0x9FFF}, {0x3400, 0x4DBF}, {0xF900, 0xFAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}}; + for (const range_u& rng : default_wide_ranges) { + table.fill_range(rng, width_u::is_2); + } + + // Read explicitly assigned ranges. + // The lines that are not empty or pure comment are uniformly of the format "HEX(..HEX)?;(A|F|H|N|Na|W) #comment". + auto get_width = [](const string& str) { + if (str == "F" || str == "W") { + return width_u::is_2; + } else { + verify(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); + return width_u::is_1; + } + }; + auto get_value = [](const string& str) { + uint32_t value{}; + const auto [end_ptr, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); + verify(end_ptr == str.data() + str.size(), impl_assertion_failed); + verify(ec == errc{}, impl_assertion_failed); + return value; + }; + + verify(!!source, "invalid path"); + string line; + const regex reg(R"(([0-9A-Z]+)(\.\.[0-9A-Z]+)?;(A|F|H|N|Na|W) *#.*)"); + while (getline(source, line)) { + if (!line.empty() && !line.starts_with("#")) { + smatch match; + verify(regex_match(line, match, reg), "invalid line"); + verify(match[1].matched, impl_assertion_failed); + verify(match[3].matched, impl_assertion_failed); + const width_u width = get_width(match[3].str()); + const uint32_t from = get_value(match[1].str()); + if (match[2].matched) { + // range (HEX..HEX) + const string match2 = match[2].str(); + verify(match2.starts_with(".."), impl_assertion_failed); + table.fill_range({from, get_value(match2.substr(2))}, width); + } else { + // single character (HEX) + table.fill_range(range_u{from}, width); + } + } + } + + return table; +} + +table_u get_table_cpp23(ifstream& source) { + table_u table = read_from(source); + + // Override with ranges specified by the C++ standard. + static constexpr range_u std_wide_ranges_cpp23[]{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; + for (const range_u& rng : std_wide_ranges_cpp23) { + table.fill_range(rng, width_u::is_2); + } + + return table; +} + +int main() { + cout << "Old table:\n"; + const table_u old_table = get_table_cpp20(); + old_table.print_intervals(); + + cout << "\nNew table:\nInput path for EastAsianWidth.txt: "; + string path; + getline(cin, path); + ifstream source(path); + const table_u new_table = get_table_cpp23(source); + new_table.print_intervals(); + + cout << "\nWas 1, now 2:\n"; + old_table.print_clusters_1_vs_2(new_table); + cout << "\nWas 2, now 1:\n"; + new_table.print_clusters_1_vs_2(old_table); +}