From 249f79ecf331fc98075d3aa7e712407879dd37ca Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 04:24:14 +0800 Subject: [PATCH 01/35] add generator --- .../format_width_intervals_generate.cpp | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 tools/scripts/format_width_intervals_generate.cpp diff --git a/tools/scripts/format_width_intervals_generate.cpp b/tools/scripts/format_width_intervals_generate.cpp new file mode 100644 index 00000000000..a0de20b985a --- /dev/null +++ b/tools/scripts/format_width_intervals_generate.cpp @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// The following code reads data from https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// and generates interval table for `_Width_estimate_intervals` in . + +#include +#include +#include +#include +#include + +void _verify(bool must_true, int line, const char* msg) { + if (!must_true) { + std::cerr << "error at line " << line << ":" << msg << std::endl; + exit(EXIT_FAILURE); + } +} +#define verify(expr, msg) _verify((expr), __LINE__, (msg)) +static const char* impl_assertion_failed = "impl assertion failed"; + +struct range_u { + uint32_t from, to; + range_u(uint32_t f, uint32_t t) : from(f), to(t) {} + range_u(uint32_t v) : from(v), to(v) {} +}; + +const uint32_t max_u = 0x7fffff; // 838'8607 +using table_u = std::vector; // true: wide +table_u make_table() { + return table_u(max_u + 1, false); +} + +void fill_range(table_u& table, const ::range_u rng, bool iswide) { + const auto [from, to] = rng; + verify(from <= to && to <= max_u, impl_assertion_failed); + for (uint32_t u = from; u <= to; u++) { + table[u] = iswide; + } +} + +// For `_Width_estimate_intervals` in . +void print_intervals(const table_u& table) { + using namespace std; + bool last = table[0]; + cout << endl << "{ "; + for (uint32_t u = 0; u <= max_u; u++) { + if (table[u] != last) { + cout << "0x" << hex << uppercase << u << "u, "; + } + last = table[u]; + } + cout << "}"; +} + +// C++20. +table_u get_old_width_table() { + using namespace std; + const vector std_wide_ranges_old{ + {0x1100, 0x115F}, + {0x2329, 0x232A}, + {0x2E80, 0x303E}, + {0x3040, 0xA4CF}, + {0xAC00, 0xD7A3}, + {0xF900, 0xFAFF}, + {0xFE10, 0xFE19}, + {0xFE30, 0xFE6F}, + {0xFF00, 0xFF60}, + {0xFFE0, 0xFFE6}, + {0x1F300, 0x1F64F}, + {0x1F900, 0x1F9FF}, + {0x20000, 0x2FFFD}, + {0x30000, 0x3FFFD}, + }; + table_u table = make_table(); + for (const range_u rng : std_wide_ranges_old) { + fill_range(table, rng, true); + } + return table; +} + +// Read data from: +// https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +table_u consult_database(std::istream& source) { + using namespace std; + + table_u table = make_table(); + + // "The unassigned code points in the following blocks default to "W":" + const vector default_wide_ranges{ + {0x4E00, 0x9FFF}, {0x3400, 0x4DBF}, {0xF900, 0xFAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}}; + for (const range_u rng : default_wide_ranges) { + fill_range(table, rng, true); + } + + // Read explicitly assigned ranges. + auto is_wide = [](const string& str) { + if (str == "W" || str == "F") { + return true; + } else { + verify(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); + return false; + } + }; + auto get_value = [](const string& str) -> uint32_t { + uint32_t value{}; + auto [end, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); + verify(end == str.data() + str.size() && ec == errc{}, impl_assertion_failed); + return value; + }; + + verify(!!source, "invalid ifstream"); + string line; + const regex reg(R"(([0-9a-zA-Z]+)(\.\.[0-9a-zA-Z]+)?;(A|F|H|N|Na|W)\s*#.*)"); + while (getline(source, line)) { + if (!line.empty() && !line.starts_with("#")) { + smatch match; + verify(regex_match(line, match, reg), + R"(invalid line format (which must be: hex(..hex);A/F/H/N/Na/W #comment ))"); + verify(match[1].matched && match[3].matched, impl_assertion_failed); + bool iswide = is_wide(match[3].str()); + uint32_t from = get_value(match[1].str()); + if (match[2].matched) { + // range (hex..hex) + string match2 = match[2].str(); + verify(match2.starts_with(".."), impl_assertion_failed); + fill_range(table, {from, get_value(match2.substr(2))}, iswide); + } else { + // single character (hex) + fill_range(table, {from}, iswide); + } + } + } + + return table; +} + +// C++23. +table_u get_new_width_table(std::istream& source) { + using namespace std; + table_u table = consult_database(source); + + // Override with ranges specified by the standard. + const vector std_wide_ranges_new{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; + for (const range_u rng : std_wide_ranges_new) { + fill_range(table, rng, true); + } + + return table; +} + +// Confirm that we get the same result as in the annex in +// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2675r1.pdf +void compare_with_old(const table_u& table /*gotten from get_new_width_table*/) { + using namespace std; + + auto print_clusters = [](const table_u& table) { + for (uint32_t u = 0; u <= max_u; u++) { + if (table[u]) { + uint32_t from = u; + uint32_t to = from; + while (to + 1 <= max_u && table[to + 1]) { + ++to; + } + if (from == to) { + cout << hex << from << endl; + } else { + cout << hex << from << "-" << to << endl; + } + u = to; + } + } + }; + + const table_u old_table = get_old_width_table(); + table_u diff_table = make_table(); + cout << endl; + cout << "Was 1, now 2:\n"; + for (uint32_t u = 0; u <= max_u; u++) { + if (!old_table[u] && table[u]) { + diff_table[u] = true; + } + } + print_clusters(diff_table); + + diff_table = make_table(); // set back to false. + cout << "\nWas 2, now 1:\n"; + for (uint32_t u = 0; u <= max_u; u++) { + if (old_table[u] && !table[u]) { + diff_table[u] = true; + } + } + print_clusters(diff_table); +} + +int main() { + // print_intervals(get_old_width_table()); + // { 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, + // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, + // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, } + std::string source_path; + std::getline(std::cin, source_path); + std::ifstream source(source_path); + table_u new_table = get_new_width_table(source); + print_intervals(new_table); + // compare_with_old(new_table); +} From 26cb007f4437a84741f1830dd0fd1b14c1a1af99 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 04:37:23 +0800 Subject: [PATCH 02/35] update comments --- tools/scripts/format_width_intervals_generate.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/scripts/format_width_intervals_generate.cpp b/tools/scripts/format_width_intervals_generate.cpp index a0de20b985a..357c7038017 100644 --- a/tools/scripts/format_width_intervals_generate.cpp +++ b/tools/scripts/format_width_intervals_generate.cpp @@ -2,7 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // The following code reads data from https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt -// and generates interval table for `_Width_estimate_intervals` in . +// and generates data for `_Width_estimate_intervals` in . #include #include @@ -25,6 +25,8 @@ struct range_u { range_u(uint32_t v) : from(v), to(v) {} }; +// The current impl rely on all unicode not exceeding max_u. +// TODO maybe too wasteful. const uint32_t max_u = 0x7fffff; // 838'8607 using table_u = std::vector; // true: wide table_u make_table() { @@ -81,6 +83,7 @@ table_u get_old_width_table() { // Read data from: // https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// The content in `source` should be the same as this file, and should not contain a BOM. table_u consult_database(std::istream& source) { using namespace std; From 4a7d1a02dae375d077bad0c3b8af68b27ca421c2 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 06:16:34 +0800 Subject: [PATCH 03/35] improvements --- .../format_width_intervals_generate.cpp | 65 ++++++++++--------- 1 file changed, 35 insertions(+), 30 deletions(-) diff --git a/tools/scripts/format_width_intervals_generate.cpp b/tools/scripts/format_width_intervals_generate.cpp index 357c7038017..d58bdc44b6a 100644 --- a/tools/scripts/format_width_intervals_generate.cpp +++ b/tools/scripts/format_width_intervals_generate.cpp @@ -25,9 +25,8 @@ struct range_u { range_u(uint32_t v) : from(v), to(v) {} }; -// The current impl rely on all unicode not exceeding max_u. -// TODO maybe too wasteful. -const uint32_t max_u = 0x7fffff; // 838'8607 +// The largest possible unicode won't exceed max_u. +const uint32_t max_u = 0x1f'ffff; using table_u = std::vector; // true: wide table_u make_table() { return table_u(max_u + 1, false); @@ -44,21 +43,26 @@ void fill_range(table_u& table, const ::range_u rng, bool iswide) { // For `_Width_estimate_intervals` in . void print_intervals(const table_u& table) { using namespace std; - bool last = table[0]; - cout << endl << "{ "; + cout << endl; + const int perline = 12; + int c = 0; + bool last = table[0]; for (uint32_t u = 0; u <= max_u; u++) { if (table[u] != last) { cout << "0x" << hex << uppercase << u << "u, "; + if (++c == perline) { + c = 0; + cout << endl; + } } last = table[u]; } - cout << "}"; + cout << endl; } -// C++20. -table_u get_old_width_table() { +table_u get_width_table_cpp20() { using namespace std; - const vector std_wide_ranges_old{ + const vector std_wide_ranges_cpp20{ {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x303E}, @@ -75,16 +79,15 @@ table_u get_old_width_table() { {0x30000, 0x3FFFD}, }; table_u table = make_table(); - for (const range_u rng : std_wide_ranges_old) { + for (const range_u rng : std_wide_ranges_cpp20) { fill_range(table, rng, true); } return table; } -// Read data from: -// https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// Read data from: https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt // The content in `source` should be the same as this file, and should not contain a BOM. -table_u consult_database(std::istream& source) { +table_u read_from_database(std::istream& source) { using namespace std; table_u table = make_table(); @@ -97,7 +100,7 @@ table_u consult_database(std::istream& source) { } // Read explicitly assigned ranges. - auto is_wide = [](const string& str) { + auto is_wide = [](const string& str) -> bool { if (str == "W" || str == "F") { return true; } else { @@ -138,14 +141,13 @@ table_u consult_database(std::istream& source) { return table; } -// C++23. -table_u get_new_width_table(std::istream& source) { +table_u get_width_table_cpp23(std::istream& source) { using namespace std; - table_u table = consult_database(source); + table_u table = read_from_database(source); - // Override with ranges specified by the standard. - const vector std_wide_ranges_new{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; - for (const range_u rng : std_wide_ranges_new) { + // Override with ranges specified by the C++ standard. + const vector std_wide_ranges_cpp23{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; + for (const range_u rng : std_wide_ranges_cpp23) { fill_range(table, rng, true); } @@ -154,7 +156,7 @@ table_u get_new_width_table(std::istream& source) { // Confirm that we get the same result as in the annex in // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2675r1.pdf -void compare_with_old(const table_u& table /*gotten from get_new_width_table*/) { +void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23*/) { using namespace std; auto print_clusters = [](const table_u& table) { @@ -175,7 +177,7 @@ void compare_with_old(const table_u& table /*gotten from get_new_width_table*/) } }; - const table_u old_table = get_old_width_table(); + const table_u old_table = get_width_table_cpp20(); table_u diff_table = make_table(); cout << endl; cout << "Was 1, now 2:\n"; @@ -186,7 +188,7 @@ void compare_with_old(const table_u& table /*gotten from get_new_width_table*/) } print_clusters(diff_table); - diff_table = make_table(); // set back to false. + diff_table = make_table(); // Reset all bits. cout << "\nWas 2, now 1:\n"; for (uint32_t u = 0; u <= max_u; u++) { if (old_table[u] && !table[u]) { @@ -197,14 +199,17 @@ void compare_with_old(const table_u& table /*gotten from get_new_width_table*/) } int main() { - // print_intervals(get_old_width_table()); - // { 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, - // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, - // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, } + // print_intervals(get_width_table_cpp20()); + // 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, + // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, + // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, + std::string source_path; std::getline(std::cin, source_path); std::ifstream source(source_path); - table_u new_table = get_new_width_table(source); - print_intervals(new_table); - // compare_with_old(new_table); + table_u table = get_width_table_cpp23(source); + print_intervals(table); + compare_with_cpp20(table); + + return 0; } From 09c253b09d6a924949bd1faf6db6a2f8ab0418a8 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 06:55:10 +0800 Subject: [PATCH 04/35] update _Width_estimate_intervals and _Unicode_width_estimate --- stl/inc/format | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/stl/inc/format b/stl/inc/format index fdcf65e7908..1e6d705d2f8 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -978,13 +978,35 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { #endif // ^^^ EDG workaround ^^^ } +#if !_HAS_CXX23 inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4928 [format.string.std]/12 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu}; +#else // ^^^ !_HAS_CXX23 / _HAS_CXX23 vvv +inline constexpr char32_t _Width_estimate_intervals[] = { // Generated from "format_width_intervals_generate.cpp" + 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, + 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, + 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, + 0x26F6u, 0x26FAu, 0x26FBu, 0x26FDu, 0x26FEu, 0x2705u, 0x2706u, 0x270Au, 0x270Cu, 0x2728u, 0x2729u, 0x274Cu, 0x274Du, + 0x274Eu, 0x274Fu, 0x2753u, 0x2756u, 0x2757u, 0x2758u, 0x2795u, 0x2798u, 0x27B0u, 0x27B1u, 0x27BFu, 0x27C0u, 0x2B1Bu, + 0x2B1Du, 0x2B50u, 0x2B51u, 0x2B55u, 0x2B56u, 0x2E80u, 0x2E9Au, 0x2E9Bu, 0x2EF4u, 0x2F00u, 0x2FD6u, 0x2FF0u, 0x2FFCu, + 0x3000u, 0x303Fu, 0x3041u, 0x3097u, 0x3099u, 0x3100u, 0x3105u, 0x3130u, 0x3131u, 0x318Fu, 0x3190u, 0x31E4u, 0x31F0u, + 0x321Fu, 0x3220u, 0x3248u, 0x3250u, 0xA48Du, 0xA490u, 0xA4C7u, 0xA960u, 0xA97Du, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, + 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE53u, 0xFE54u, 0xFE67u, 0xFE68u, 0xFE6Cu, 0xFF01u, 0xFF61u, 0xFFE0u, 0xFFE7u, + 0x16FE0u, 0x16FE5u, 0x16FF0u, 0x16FF2u, 0x17000u, 0x187F8u, 0x18800u, 0x18CD6u, 0x18D00u, 0x18D09u, 0x1AFF0u, + 0x1AFF4u, 0x1AFF5u, 0x1AFFCu, 0x1AFFDu, 0x1AFFFu, 0x1B000u, 0x1B123u, 0x1B132u, 0x1B133u, 0x1B150u, 0x1B153u, + 0x1B155u, 0x1B156u, 0x1B164u, 0x1B168u, 0x1B170u, 0x1B2FCu, 0x1F004u, 0x1F005u, 0x1F0CFu, 0x1F0D0u, 0x1F18Eu, + 0x1F18Fu, 0x1F191u, 0x1F19Bu, 0x1F200u, 0x1F203u, 0x1F210u, 0x1F23Cu, 0x1F240u, 0x1F249u, 0x1F250u, 0x1F252u, + 0x1F260u, 0x1F266u, 0x1F300u, 0x1F650u, 0x1F680u, 0x1F6C6u, 0x1F6CCu, 0x1F6CDu, 0x1F6D0u, 0x1F6D3u, 0x1F6D5u, + 0x1F6D8u, 0x1F6DCu, 0x1F6E0u, 0x1F6EBu, 0x1F6EDu, 0x1F6F4u, 0x1F6FDu, 0x1F7E0u, 0x1F7ECu, 0x1F7F0u, 0x1F7F1u, + 0x1F900u, 0x1FA00u, 0x1FA70u, 0x1FA7Du, 0x1FA80u, 0x1FA89u, 0x1FA90u, 0x1FABEu, 0x1FABFu, 0x1FAC6u, 0x1FACEu, + 0x1FADCu, 0x1FAE0u, 0x1FAE9u, 0x1FAF0u, 0x1FAF9u, 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu}; +#endif // ^^^ _HAS_CXX23 ^^^ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { // Computes the width estimation for Unicode characters from N4928 [format.string.std]/12 +#if !_HAS_CXX23 int _Result = 1; for (const auto& _Bound : _Width_estimate_intervals) { if (_Ch < _Bound) { @@ -994,6 +1016,23 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { } return 1; +#else // ^^^ !_HAS_CXX23 / _HAS_CXX23 vvv + if (_Ch < 0x26ACu) { + int _Result = 1; + for (const auto& _Bound : _Width_estimate_intervals) { + if (_Ch < _Bound) { + return _Result; + } + _Result ^= 0b11u; // Flip between 1 and 2 on each iteration + } + + return 1; + } else { + int _Index = + upper_bound(_Width_estimate_intervals, end(_Width_estimate_intervals), _Ch) - _Width_estimate_intervals; + return 1 + (_Index & 1); + } +#endif // ^^^ _HAS_CXX23 ^^^ } template From 361580d7ff90dbfdde46d510e6d48842d48d74fd Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 13:43:46 +0800 Subject: [PATCH 05/35] review feedback: drop `#if _HAS_CXX23` improve generator source code --- stl/inc/format | 28 ++----- tools/unicode_properties_parse/.gitignore | 1 + .../format_width_estimate_intervals.cpp} | 73 +++++++++---------- 3 files changed, 40 insertions(+), 62 deletions(-) rename tools/{scripts/format_width_intervals_generate.cpp => unicode_properties_parse/format_width_estimate_intervals.cpp} (74%) diff --git a/stl/inc/format b/stl/inc/format index 1e6d705d2f8..6fdfd1334d8 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -978,13 +978,8 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { #endif // ^^^ EDG workaround ^^^ } -#if !_HAS_CXX23 -inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4928 [format.string.std]/12 - 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, 0xFE10u, - 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, 0x20000u, - 0x2FFFEu, 0x30000u, 0x3FFFEu}; -#else // ^^^ !_HAS_CXX23 / _HAS_CXX23 vvv -inline constexpr char32_t _Width_estimate_intervals[] = { // Generated from "format_width_intervals_generate.cpp" +inline constexpr char32_t _Width_estimate_intervals[] = { + // Per N4928 [format.string.std]/12 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, @@ -1002,22 +997,10 @@ inline constexpr char32_t _Width_estimate_intervals[] = { // Generated from "for 0x1F6D8u, 0x1F6DCu, 0x1F6E0u, 0x1F6EBu, 0x1F6EDu, 0x1F6F4u, 0x1F6FDu, 0x1F7E0u, 0x1F7ECu, 0x1F7F0u, 0x1F7F1u, 0x1F900u, 0x1FA00u, 0x1FA70u, 0x1FA7Du, 0x1FA80u, 0x1FA89u, 0x1FA90u, 0x1FABEu, 0x1FABFu, 0x1FAC6u, 0x1FACEu, 0x1FADCu, 0x1FAE0u, 0x1FAE9u, 0x1FAF0u, 0x1FAF9u, 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu}; -#endif // ^^^ _HAS_CXX23 ^^^ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { // Computes the width estimation for Unicode characters from N4928 [format.string.std]/12 -#if !_HAS_CXX23 - int _Result = 1; - for (const auto& _Bound : _Width_estimate_intervals) { - if (_Ch < _Bound) { - return _Result; - } - _Result ^= 0b11u; // Flip between 1 and 2 on each iteration - } - - return 1; -#else // ^^^ !_HAS_CXX23 / _HAS_CXX23 vvv - if (_Ch < 0x26ACu) { + if (_Ch < 0x25FDu) { int _Result = 1; for (const auto& _Bound : _Width_estimate_intervals) { if (_Ch < _Bound) { @@ -1028,11 +1011,10 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { return 1; } else { - int _Index = + int _Upper_bound_index = upper_bound(_Width_estimate_intervals, end(_Width_estimate_intervals), _Ch) - _Width_estimate_intervals; - return 1 + (_Index & 1); + return 1 + (_Upper_bound_index & 1); } -#endif // ^^^ _HAS_CXX23 ^^^ } template diff --git a/tools/unicode_properties_parse/.gitignore b/tools/unicode_properties_parse/.gitignore index 94b365a6e24..c586bd6edee 100644 --- a/tools/unicode_properties_parse/.gitignore +++ b/tools/unicode_properties_parse/.gitignore @@ -7,5 +7,6 @@ emoji-data.txt DerivedCoreProperties.txt DerivedGeneralCategory.txt +EastAsianWidth.txt # format_width_estimate_intervals.cpp GraphemeBreakProperty.txt GraphemeBreakTest.txt diff --git a/tools/scripts/format_width_intervals_generate.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp similarity index 74% rename from tools/scripts/format_width_intervals_generate.cpp rename to tools/unicode_properties_parse/format_width_estimate_intervals.cpp index d58bdc44b6a..ebb3aba2b99 100644 --- a/tools/scripts/format_width_intervals_generate.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -1,17 +1,17 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// The following code reads data from https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt -// and generates data for `_Width_estimate_intervals` in . +// The following code generates data for `_Width_estimate_intervals` in . #include +#include #include #include #include #include -void _verify(bool must_true, int line, const char* msg) { - if (!must_true) { +void _verify(bool test, int line, const char* msg) { + if (!test) { std::cerr << "error at line " << line << ":" << msg << std::endl; exit(EXIT_FAILURE); } @@ -32,25 +32,16 @@ table_u make_table() { return table_u(max_u + 1, false); } -void fill_range(table_u& table, const ::range_u rng, bool iswide) { - const auto [from, to] = rng; - verify(from <= to && to <= max_u, impl_assertion_failed); - for (uint32_t u = from; u <= to; u++) { - table[u] = iswide; - } -} - -// For `_Width_estimate_intervals` in . +// For `_Width_estimate_intervals`. void print_intervals(const table_u& table) { using namespace std; cout << endl; - const int perline = 12; - int c = 0; - bool last = table[0]; + int c = 0; + bool last = table[0]; for (uint32_t u = 0; u <= max_u; u++) { if (table[u] != last) { cout << "0x" << hex << uppercase << u << "u, "; - if (++c == perline) { + if (++c == 12) { c = 0; cout << endl; } @@ -60,6 +51,14 @@ void print_intervals(const table_u& table) { cout << endl; } +void fill_range(table_u& table, const ::range_u rng, bool is_wide) { + const auto [from, to] = rng; + verify(from <= to && to <= max_u, impl_assertion_failed); + for (uint32_t u = from; u <= to; u++) { + table[u] = is_wide; + } +} + table_u get_width_table_cpp20() { using namespace std; const vector std_wide_ranges_cpp20{ @@ -85,9 +84,9 @@ table_u get_width_table_cpp20() { return table; } -// Read data from: https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt -// The content in `source` should be the same as this file, and should not contain a BOM. -table_u read_from_database(std::istream& source) { +// Read data from a file with the same content as in: https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// The file should not contain a BOM. +table_u read_from_source(std::ifstream& source) { using namespace std; table_u table = make_table(); @@ -100,7 +99,8 @@ table_u read_from_database(std::istream& source) { } // Read explicitly assigned ranges. - auto is_wide = [](const string& str) -> bool { + // The lines that are not empty or pure comment are uniformly of the format "hex(..hex)?;(A|F|H|N|Na|W) #comment". + auto test_wide = [](const string& str) -> bool { if (str == "W" || str == "F") { return true; } else { @@ -122,18 +122,18 @@ table_u read_from_database(std::istream& source) { if (!line.empty() && !line.starts_with("#")) { smatch match; verify(regex_match(line, match, reg), - R"(invalid line format (which must be: hex(..hex);A/F/H/N/Na/W #comment ))"); + R"(invalid line format (which must be: hex(..hex)?;(A|F|H|N|Na|W) #comment ))"); verify(match[1].matched && match[3].matched, impl_assertion_failed); - bool iswide = is_wide(match[3].str()); + bool is_wide = test_wide(match[3].str()); uint32_t from = get_value(match[1].str()); if (match[2].matched) { // range (hex..hex) string match2 = match[2].str(); verify(match2.starts_with(".."), impl_assertion_failed); - fill_range(table, {from, get_value(match2.substr(2))}, iswide); + fill_range(table, {from, get_value(match2.substr(2))}, is_wide); } else { // single character (hex) - fill_range(table, {from}, iswide); + fill_range(table, {from}, is_wide); } } } @@ -141,9 +141,9 @@ table_u read_from_database(std::istream& source) { return table; } -table_u get_width_table_cpp23(std::istream& source) { +table_u get_width_table_cpp23(std::ifstream& source) { using namespace std; - table_u table = read_from_database(source); + table_u table = read_from_source(source); // Override with ranges specified by the C++ standard. const vector std_wide_ranges_cpp23{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; @@ -168,9 +168,9 @@ void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23 ++to; } if (from == to) { - cout << hex << from << endl; + cout << hex << uppercase << from << endl; } else { - cout << hex << from << "-" << to << endl; + cout << hex << uppercase << from << ".." << to << endl; } u = to; } @@ -179,8 +179,7 @@ void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23 const table_u old_table = get_width_table_cpp20(); table_u diff_table = make_table(); - cout << endl; - cout << "Was 1, now 2:\n"; + cout << "\nWas 1, now 2:\n"; for (uint32_t u = 0; u <= max_u; u++) { if (!old_table[u] && table[u]) { diff_table[u] = true; @@ -200,16 +199,12 @@ void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23 int main() { // print_intervals(get_width_table_cpp20()); - // 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, - // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, - // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, + // 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, + // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, + // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, - std::string source_path; - std::getline(std::cin, source_path); - std::ifstream source(source_path); + std::ifstream source("EastAsianWidth.txt"); table_u table = get_width_table_cpp23(source); print_intervals(table); compare_with_cpp20(table); - - return 0; } From 077b3febb936e2104dbc188d8d5880480ef79d86 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 13:48:53 +0800 Subject: [PATCH 06/35] nits --- stl/inc/format | 3 +-- .../format_width_estimate_intervals.cpp | 25 +++++++++---------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 6fdfd1334d8..d3e93615f3d 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -978,8 +978,7 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { #endif // ^^^ EDG workaround ^^^ } -inline constexpr char32_t _Width_estimate_intervals[] = { - // Per N4928 [format.string.std]/12 +inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4928 [format.string.std]/12 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index ebb3aba2b99..0126e0d9dee 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -59,7 +59,7 @@ void fill_range(table_u& table, const ::range_u rng, bool is_wide) { } } -table_u get_width_table_cpp20() { +table_u get_table_cpp20() { using namespace std; const vector std_wide_ranges_cpp20{ {0x1100, 0x115F}, @@ -84,8 +84,8 @@ table_u get_width_table_cpp20() { return table; } -// Read data from a file with the same content as in: https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt -// The file should not contain a BOM. +// Read data from a file with the same content as in https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// To make this function work, the file should not contain a BOM. table_u read_from_source(std::ifstream& source) { using namespace std; @@ -101,7 +101,7 @@ table_u read_from_source(std::ifstream& source) { // Read explicitly assigned ranges. // The lines that are not empty or pure comment are uniformly of the format "hex(..hex)?;(A|F|H|N|Na|W) #comment". auto test_wide = [](const string& str) -> bool { - if (str == "W" || str == "F") { + if (str == "F" || str == "W") { return true; } else { verify(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); @@ -121,8 +121,7 @@ table_u read_from_source(std::ifstream& source) { while (getline(source, line)) { if (!line.empty() && !line.starts_with("#")) { smatch match; - verify(regex_match(line, match, reg), - R"(invalid line format (which must be: hex(..hex)?;(A|F|H|N|Na|W) #comment ))"); + verify(regex_match(line, match, reg), "invalid line"); verify(match[1].matched && match[3].matched, impl_assertion_failed); bool is_wide = test_wide(match[3].str()); uint32_t from = get_value(match[1].str()); @@ -141,7 +140,7 @@ table_u read_from_source(std::ifstream& source) { return table; } -table_u get_width_table_cpp23(std::ifstream& source) { +table_u get_table_cpp23(std::ifstream& source) { using namespace std; table_u table = read_from_source(source); @@ -156,7 +155,7 @@ table_u get_width_table_cpp23(std::ifstream& source) { // Confirm that we get the same result as in the annex in // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2675r1.pdf -void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23*/) { +void compare_with_cpp20(const table_u& table /*gotten from get_table_cpp23*/) { using namespace std; auto print_clusters = [](const table_u& table) { @@ -177,9 +176,9 @@ void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23 } }; - const table_u old_table = get_width_table_cpp20(); + const table_u old_table = get_table_cpp20(); table_u diff_table = make_table(); - cout << "\nWas 1, now 2:\n"; + cout << "\nwas 1, now 2:\n"; for (uint32_t u = 0; u <= max_u; u++) { if (!old_table[u] && table[u]) { diff_table[u] = true; @@ -188,7 +187,7 @@ void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23 print_clusters(diff_table); diff_table = make_table(); // Reset all bits. - cout << "\nWas 2, now 1:\n"; + cout << "\nwas 2, now 1:\n"; for (uint32_t u = 0; u <= max_u; u++) { if (old_table[u] && !table[u]) { diff_table[u] = true; @@ -198,13 +197,13 @@ void compare_with_cpp20(const table_u& table /*gotten from get_width_table_cpp23 } int main() { - // print_intervals(get_width_table_cpp20()); + // print_intervals(get_table_cpp20()); // 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, std::ifstream source("EastAsianWidth.txt"); - table_u table = get_width_table_cpp23(source); + table_u table = get_table_cpp23(source); print_intervals(table); compare_with_cpp20(table); } From b5a6561842354817d12a5e4d60f3e9af9f352ba0 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 14:51:04 +0800 Subject: [PATCH 07/35] add cast --- stl/inc/format | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index d3e93615f3d..89cbfc1690d 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -1010,8 +1010,8 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { return 1; } else { - int _Upper_bound_index = - upper_bound(_Width_estimate_intervals, end(_Width_estimate_intervals), _Ch) - _Width_estimate_intervals; + int _Upper_bound_index = static_cast( + upper_bound(_Width_estimate_intervals, end(_Width_estimate_intervals), _Ch) - _Width_estimate_intervals); return 1 + (_Upper_bound_index & 1); } } From 7750b050ee9641be80acd6a3a47a7eacc12c4e90 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 15:29:24 +0800 Subject: [PATCH 08/35] citation updates Co-authored-by: A. Jiang --- stl/inc/format | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 89cbfc1690d..c3f7258f1cb 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -978,7 +978,7 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { #endif // ^^^ EDG workaround ^^^ } -inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4928 [format.string.std]/12 +inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4950 [format.string.std]/13 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, @@ -998,7 +998,7 @@ inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4928 [format.s 0x1FADCu, 0x1FAE0u, 0x1FAE9u, 0x1FAF0u, 0x1FAF9u, 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu}; _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { - // Computes the width estimation for Unicode characters from N4928 [format.string.std]/12 + // Computes the width estimation for Unicode characters from N4950 [format.string.std]/13 if (_Ch < 0x25FDu) { int _Result = 1; for (const auto& _Bound : _Width_estimate_intervals) { From a447d995cb675f11841af38881201e098cbff52b Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Sun, 23 Jul 2023 17:35:29 +0800 Subject: [PATCH 09/35] test case fix --- tests/std/tests/P0645R10_text_formatting_utf8/test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp index c8de404068b..c049ad8399f 100644 --- a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp +++ b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp @@ -82,8 +82,8 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\u2e80\x58"), 3}, {TYPED_LITERAL(CharT, "\u303e\x58"), 3}, {TYPED_LITERAL(CharT, "\u303f\x58"), 2}, - {TYPED_LITERAL(CharT, "\u3040\x58"), 3}, - {TYPED_LITERAL(CharT, "\ua4cf\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3040\x58"), 2}, + {TYPED_LITERAL(CharT, "\ua4cf\x58"), 2}, {TYPED_LITERAL(CharT, "\ua4d0\x58"), 2}, {TYPED_LITERAL(CharT, "\uabff\x58"), 2}, {TYPED_LITERAL(CharT, "\uac00\x58"), 3}, @@ -104,10 +104,10 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ufe1a\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe2f\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe30\x58"), 3}, - {TYPED_LITERAL(CharT, "\ufe6f\x58"), 3}, + {TYPED_LITERAL(CharT, "\ufe6f\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe70\x58"), 2}, {TYPED_LITERAL(CharT, "\ufeff\x58"), 2}, - {TYPED_LITERAL(CharT, "\uff00\x58"), 3}, + {TYPED_LITERAL(CharT, "\uff00\x58"), 2}, {TYPED_LITERAL(CharT, "\uff60\x58"), 3}, {TYPED_LITERAL(CharT, "\uff61\x58"), 2}, {TYPED_LITERAL(CharT, "\uffdf\x58"), 2}, From 383925f39985bc0be8de7ccc73e28b024659fc27 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Mon, 24 Jul 2023 15:51:39 +0800 Subject: [PATCH 10/35] update "format_width_estimate_interval.cpp" --- stl/inc/format | 4 ++- tools/unicode_properties_parse/.gitignore | 2 +- .../format_width_estimate_intervals.cpp | 35 +++++++++++-------- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index c3f7258f1cb..8070b6b4473 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -978,7 +978,9 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { #endif // ^^^ EDG workaround ^^^ } -inline constexpr char32_t _Width_estimate_intervals[] = { // Per N4950 [format.string.std]/13 +// Generated per N4950 [format.string.std]/13, by tools/unicode_properties_parse/format_width_estimate_intervals.cpp +// in the https://github.com/microsoft/stl repository. +inline constexpr char32_t _Width_estimate_intervals[] = { // 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, diff --git a/tools/unicode_properties_parse/.gitignore b/tools/unicode_properties_parse/.gitignore index c586bd6edee..4fb9d76d452 100644 --- a/tools/unicode_properties_parse/.gitignore +++ b/tools/unicode_properties_parse/.gitignore @@ -7,6 +7,6 @@ emoji-data.txt DerivedCoreProperties.txt DerivedGeneralCategory.txt -EastAsianWidth.txt # format_width_estimate_intervals.cpp +EastAsianWidth.txt GraphemeBreakProperty.txt GraphemeBreakTest.txt diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 0126e0d9dee..b138b8ac721 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -12,7 +12,7 @@ void _verify(bool test, int line, const char* msg) { if (!test) { - std::cerr << "error at line " << line << ":" << msg << std::endl; + std::cerr << "Error at line " << line << ": " << msg << std::endl; exit(EXIT_FAILURE); } } @@ -25,8 +25,8 @@ struct range_u { range_u(uint32_t v) : from(v), to(v) {} }; -// The largest possible unicode won't exceed max_u. -const uint32_t max_u = 0x1f'ffff; +// A valid Unicode code point won't exceed `max_u`. +const uint32_t max_u = 0x10'ffff; using table_u = std::vector; // true: wide table_u make_table() { return table_u(max_u + 1, false); @@ -84,7 +84,11 @@ table_u get_table_cpp20() { return table; } -// Read data from a file with the same content as in https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt +// Read data from "EastAsianWidth.txt". +// The latest version can be found at: +// https://www.unicode.org/Public/UCD/latest/ucd/EastAsianWidth.txt +// The current implementation works for: +// https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt // To make this function work, the file should not contain a BOM. table_u read_from_source(std::ifstream& source) { using namespace std; @@ -99,7 +103,7 @@ table_u read_from_source(std::ifstream& source) { } // Read explicitly assigned ranges. - // The lines that are not empty or pure comment are uniformly of the format "hex(..hex)?;(A|F|H|N|Na|W) #comment". + // The lines that are not empty or pure comment are uniformly of the format "HEX(..HEX)?;(A|F|H|N|Na|W) #comment". auto test_wide = [](const string& str) -> bool { if (str == "F" || str == "W") { return true; @@ -117,7 +121,7 @@ table_u read_from_source(std::ifstream& source) { verify(!!source, "invalid ifstream"); string line; - const regex reg(R"(([0-9a-zA-Z]+)(\.\.[0-9a-zA-Z]+)?;(A|F|H|N|Na|W)\s*#.*)"); + const regex reg(R"(([0-9A-Z]+)(\.\.[0-9A-Z]+)?;(A|F|H|N|Na|W) *#.*)"); while (getline(source, line)) { if (!line.empty() && !line.starts_with("#")) { smatch match; @@ -126,12 +130,12 @@ table_u read_from_source(std::ifstream& source) { bool is_wide = test_wide(match[3].str()); uint32_t from = get_value(match[1].str()); if (match[2].matched) { - // range (hex..hex) + // range (HEX..HEX) string match2 = match[2].str(); verify(match2.starts_with(".."), impl_assertion_failed); fill_range(table, {from, get_value(match2.substr(2))}, is_wide); } else { - // single character (hex) + // single character (HEX) fill_range(table, {from}, is_wide); } } @@ -153,7 +157,7 @@ table_u get_table_cpp23(std::ifstream& source) { return table; } -// Confirm that we get the same result as in the annex in +// Confirm that we get the same result (under UCD version 15.0.0) as in the annex in // https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2675r1.pdf void compare_with_cpp20(const table_u& table /*gotten from get_table_cpp23*/) { using namespace std; @@ -167,9 +171,9 @@ void compare_with_cpp20(const table_u& table /*gotten from get_table_cpp23*/) { ++to; } if (from == to) { - cout << hex << uppercase << from << endl; + cout << hex << uppercase << "U+" << from << endl; } else { - cout << hex << uppercase << from << ".." << to << endl; + cout << hex << uppercase << "U+" << from << "..U+" << to << endl; } u = to; } @@ -178,7 +182,7 @@ void compare_with_cpp20(const table_u& table /*gotten from get_table_cpp23*/) { const table_u old_table = get_table_cpp20(); table_u diff_table = make_table(); - cout << "\nwas 1, now 2:\n"; + cout << "\nWas 1, now 2:\n"; for (uint32_t u = 0; u <= max_u; u++) { if (!old_table[u] && table[u]) { diff_table[u] = true; @@ -187,7 +191,7 @@ void compare_with_cpp20(const table_u& table /*gotten from get_table_cpp23*/) { print_clusters(diff_table); diff_table = make_table(); // Reset all bits. - cout << "\nwas 2, now 1:\n"; + cout << "\nWas 2, now 1:\n"; for (uint32_t u = 0; u <= max_u; u++) { if (old_table[u] && !table[u]) { diff_table[u] = true; @@ -202,7 +206,10 @@ int main() { // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, - std::ifstream source("EastAsianWidth.txt"); + std::cout << "Input path for EastAsianWidth.txt: "; + std::string path; + getline(std::cin, path); + std::ifstream source(path); table_u table = get_table_cpp23(source); print_intervals(table); compare_with_cpp20(table); From 8ef6bfce7e79fbdf49ecedcda70e71208084749f Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Mon, 24 Jul 2023 16:03:54 +0800 Subject: [PATCH 11/35] "std/utilities/format/format.functions/unicode.pass.cpp FAIL" --- tests/libcxx/expected_results.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 7b912c8a6b0..08702b8f33e 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -177,6 +177,9 @@ std/ranges/range.access/size.pass.cpp FAIL # libc++ doesn't implement P2652R2 "Disallowing User Specialization Of allocator_traits" std/utilities/memory/allocator.traits/allocate_at_least.pass.cpp FAIL +# libc++ doesn't implement P2675R1 "Improving std::format's Width Estimation" +std/utilities/format/format.functions/unicode.pass.cpp FAIL + # libc++ doesn't implement P2770R0 "Stashing stashing iterators for proper flattening" std/ranges/range.adaptors/range.join.view/end.pass.cpp FAIL std/ranges/range.adaptors/range.join.view/iterator/ctor.other.pass.cpp FAIL From 359ef006353f5c54bdc7ec5725517facc65d93e5 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Tue, 25 Jul 2023 09:19:07 +0800 Subject: [PATCH 12/35] refactor format_width_estimate_intervals.cpp --- .../format_width_estimate_intervals.cpp | 186 ++++++++---------- 1 file changed, 87 insertions(+), 99 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index b138b8ac721..1c59d8daa0c 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -25,43 +25,70 @@ struct range_u { range_u(uint32_t v) : from(v), to(v) {} }; -// A valid Unicode code point won't exceed `max_u`. -const uint32_t max_u = 0x10'ffff; -using table_u = std::vector; // true: wide -table_u make_table() { - return table_u(max_u + 1, false); -} +enum class width_u : bool { is_1 = false, is_2 = true }; + +class table_u { + // A valid Unicode code point won't exceed `max_u`. + static constexpr uint32_t max_u = 0x10'ffff; + std::vector table; + +public: + table_u() : table(max_u + 1, width_u::is_1) {} + void fill_range(const range_u rng, const width_u width) { + const auto [from, to] = rng; + verify(from <= to && to <= max_u, impl_assertion_failed); + for (uint32_t u = from; u <= to; u++) { + table[u] = width; + } + } -// For `_Width_estimate_intervals`. -void print_intervals(const table_u& table) { - using namespace std; - cout << endl; - int c = 0; - bool last = table[0]; - for (uint32_t u = 0; u <= max_u; u++) { - if (table[u] != last) { - cout << "0x" << hex << uppercase << u << "u, "; - if (++c == 12) { - c = 0; - cout << endl; + void print_intervals() const { + // Print table for `_Width_estimate_intervals`. + using namespace std; + int c = 0; + width_u last = table[0]; + for (uint32_t u = 0; u <= max_u; u++) { + if (table[u] != last) { + cout << "0x" << hex << uppercase << u << "u, "; + if (++c == 12) { + c = 0; + cout << endl; + } } + last = table[u]; } - last = table[u]; + cout << endl; } - cout << endl; -} -void fill_range(table_u& table, const ::range_u rng, bool is_wide) { - const auto [from, to] = rng; - verify(from <= to && to <= max_u, impl_assertion_failed); - for (uint32_t u = from; u <= to; u++) { - table[u] = is_wide; + void print_clusters_1_vs_2(const table_u& other) const { + using namespace std; + vector cluster_table(max_u + 1, false); + for (uint32_t u = 0; u <= max_u; u++) { + if (table[u] == width_u::is_1 && other.table[u] == width_u::is_2) { + cluster_table[u] = true; + } + } + + for (uint32_t u = 0; u <= max_u; u++) { + if (cluster_table[u]) { + uint32_t from = u; + uint32_t to = from; + while (to + 1 <= max_u && cluster_table[to + 1]) { + ++to; + } + if (from == to) { + cout << hex << uppercase << "U+" << from << endl; + } else { + cout << hex << uppercase << "U+" << from << "..U+" << to << endl; + } + u = to; + } + } } -} +}; table_u get_table_cpp20() { - using namespace std; - const vector std_wide_ranges_cpp20{ + const range_u std_wide_ranges_cpp20[]{ {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x303E}, @@ -77,9 +104,10 @@ table_u get_table_cpp20() { {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}, }; - table_u table = make_table(); + + table_u table; for (const range_u rng : std_wide_ranges_cpp20) { - fill_range(table, rng, true); + table.fill_range(rng, width_u::is_2); } return table; } @@ -90,26 +118,25 @@ table_u get_table_cpp20() { // The current implementation works for: // https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt // To make this function work, the file should not contain a BOM. -table_u read_from_source(std::ifstream& source) { +table_u read_from(std::ifstream& source) { using namespace std; - - table_u table = make_table(); + table_u table; // "The unassigned code points in the following blocks default to "W":" - const vector default_wide_ranges{ + const range_u default_wide_ranges[]{ {0x4E00, 0x9FFF}, {0x3400, 0x4DBF}, {0xF900, 0xFAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}}; for (const range_u rng : default_wide_ranges) { - fill_range(table, rng, true); + table.fill_range(rng, width_u::is_2); } // Read explicitly assigned ranges. // The lines that are not empty or pure comment are uniformly of the format "HEX(..HEX)?;(A|F|H|N|Na|W) #comment". - auto test_wide = [](const string& str) -> bool { + auto get_width = [](const string& str) -> width_u { if (str == "F" || str == "W") { - return true; + return width_u::is_2; } else { verify(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); - return false; + return width_u::is_1; } }; auto get_value = [](const string& str) -> uint32_t { @@ -119,7 +146,7 @@ table_u read_from_source(std::ifstream& source) { return value; }; - verify(!!source, "invalid ifstream"); + verify(!!source, "invalid path"); string line; const regex reg(R"(([0-9A-Z]+)(\.\.[0-9A-Z]+)?;(A|F|H|N|Na|W) *#.*)"); while (getline(source, line)) { @@ -127,16 +154,16 @@ table_u read_from_source(std::ifstream& source) { smatch match; verify(regex_match(line, match, reg), "invalid line"); verify(match[1].matched && match[3].matched, impl_assertion_failed); - bool is_wide = test_wide(match[3].str()); - uint32_t from = get_value(match[1].str()); + const width_u width = get_width(match[3].str()); + const uint32_t from = get_value(match[1].str()); if (match[2].matched) { // range (HEX..HEX) string match2 = match[2].str(); verify(match2.starts_with(".."), impl_assertion_failed); - fill_range(table, {from, get_value(match2.substr(2))}, is_wide); + table.fill_range({from, get_value(match2.substr(2))}, width); } else { // single character (HEX) - fill_range(table, {from}, is_wide); + table.fill_range({from}, width); } } } @@ -145,72 +172,33 @@ table_u read_from_source(std::ifstream& source) { } table_u get_table_cpp23(std::ifstream& source) { - using namespace std; - table_u table = read_from_source(source); + table_u table = read_from(source); // Override with ranges specified by the C++ standard. - const vector std_wide_ranges_cpp23{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; + const range_u std_wide_ranges_cpp23[]{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; for (const range_u rng : std_wide_ranges_cpp23) { - fill_range(table, rng, true); + table.fill_range(rng, width_u::is_2); } return table; } -// Confirm that we get the same result (under UCD version 15.0.0) as in the annex in -// https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2023/p2675r1.pdf -void compare_with_cpp20(const table_u& table /*gotten from get_table_cpp23*/) { +int main() { using namespace std; - auto print_clusters = [](const table_u& table) { - for (uint32_t u = 0; u <= max_u; u++) { - if (table[u]) { - uint32_t from = u; - uint32_t to = from; - while (to + 1 <= max_u && table[to + 1]) { - ++to; - } - if (from == to) { - cout << hex << uppercase << "U+" << from << endl; - } else { - cout << hex << uppercase << "U+" << from << "..U+" << to << endl; - } - u = to; - } - } - }; - + cout << "Old table:\n"; const table_u old_table = get_table_cpp20(); - table_u diff_table = make_table(); - cout << "\nWas 1, now 2:\n"; - for (uint32_t u = 0; u <= max_u; u++) { - if (!old_table[u] && table[u]) { - diff_table[u] = true; - } - } - print_clusters(diff_table); + old_table.print_intervals(); - diff_table = make_table(); // Reset all bits. - cout << "\nWas 2, now 1:\n"; - for (uint32_t u = 0; u <= max_u; u++) { - if (old_table[u] && !table[u]) { - diff_table[u] = true; - } - } - print_clusters(diff_table); -} + cout << "\nNew table:\nInput path for EastAsianWidth.txt: "; + string path; + getline(cin, path); + ifstream source(path); + table_u new_table = get_table_cpp23(source); + new_table.print_intervals(); -int main() { - // print_intervals(get_table_cpp20()); - // 0x1100u, 0x1160u, 0x2329u, 0x232Bu, 0x2E80u, 0x303Fu, 0x3040u, 0xA4D0u, 0xAC00u, 0xD7A4u, 0xF900u, 0xFB00u, - // 0xFE10u, 0xFE1Au, 0xFE30u, 0xFE70u, 0xFF00u, 0xFF61u, 0xFFE0u, 0xFFE7u, 0x1F300u, 0x1F650u, 0x1F900u, 0x1FA00u, - // 0x20000u, 0x2FFFEu, 0x30000u, 0x3FFFEu, - - std::cout << "Input path for EastAsianWidth.txt: "; - std::string path; - getline(std::cin, path); - std::ifstream source(path); - table_u table = get_table_cpp23(source); - print_intervals(table); - compare_with_cpp20(table); + cout << "\nWas 1, now 2:\n"; + old_table.print_clusters_1_vs_2(new_table); + cout << "\nWas 2, now 1:\n"; + new_table.print_clusters_1_vs_2(old_table); } From 75134a38a84317cb7d526b5766fc82e027ac173b Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Tue, 25 Jul 2023 09:41:09 +0800 Subject: [PATCH 13/35] Apply suggestions from code review Co-authored-by: A. Jiang --- .../format_width_estimate_intervals.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 1c59d8daa0c..b76cfb5cd52 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -10,14 +10,14 @@ #include #include -void _verify(bool test, int line, const char* msg) { +void verify_impl(bool test, int line, const char* msg) { if (!test) { std::cerr << "Error at line " << line << ": " << msg << std::endl; exit(EXIT_FAILURE); } } -#define verify(expr, msg) _verify((expr), __LINE__, (msg)) -static const char* impl_assertion_failed = "impl assertion failed"; +#define verify(expr, msg) verify_impl((expr), __LINE__, (msg)) +constexpr const char* impl_assertion_failed = "impl assertion failed"; struct range_u { uint32_t from, to; From e6020d4804e27d1139ca1da39987d14ad50167a8 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Wed, 26 Jul 2023 00:16:48 +0800 Subject: [PATCH 14/35] avoid magic number --- stl/inc/format | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 8070b6b4473..b0162fe6f02 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -1001,7 +1001,8 @@ inline constexpr char32_t _Width_estimate_intervals[] = { // _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { // Computes the width estimation for Unicode characters from N4950 [format.string.std]/13 - if (_Ch < 0x25FDu) { + constexpr char32_t _Limit = _Width_estimate_intervals[12]; + if (_Ch < _Limit) { int _Result = 1; for (const auto& _Bound : _Width_estimate_intervals) { if (_Ch < _Bound) { @@ -1009,7 +1010,6 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { } _Result ^= 0b11u; // Flip between 1 and 2 on each iteration } - return 1; } else { int _Upper_bound_index = static_cast( From c16684a955ae945bcff3473996ac322e287a3e6d Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:05:06 -0700 Subject: [PATCH 15/35] List P2675R1 as implemented in C++20 mode. --- stl/inc/yvals_core.h | 1 + 1 file changed, 1 insertion(+) diff --git a/stl/inc/yvals_core.h b/stl/inc/yvals_core.h index c0ee672319d..aa8ad1a23ee 100644 --- a/stl/inc/yvals_core.h +++ b/stl/inc/yvals_core.h @@ -301,6 +301,7 @@ // P2602R2 Poison Pills Are Too Toxic // P2609R3 Relaxing Ranges Just A Smidge // P2655R3 common_reference_t Of reference_wrapper Should Be A Reference Type +// P2675R1 Improving std::format's Width Estimation // P2711R1 Making Multi-Param Constructors Of Views explicit // P2736R2 Referencing The Unicode Standard // P2770R0 Stashing Stashing Iterators For Proper Flattening From d035d597fe1327e86f461ae27f7c3cb2601f9199 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:13:32 -0700 Subject: [PATCH 16/35] Avoid ABI issues: `_Width_estimate_intervals` => `_Width_estimate_intervals_v2` --- stl/inc/format | 11 ++++++----- .../format_width_estimate_intervals.cpp | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index b0162fe6f02..3d0d2222212 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -980,7 +980,7 @@ _NODISCARD constexpr bool _Is_execution_charset_self_synchronizing() { // Generated per N4950 [format.string.std]/13, by tools/unicode_properties_parse/format_width_estimate_intervals.cpp // in the https://github.com/microsoft/stl repository. -inline constexpr char32_t _Width_estimate_intervals[] = { // +inline constexpr char32_t _Width_estimate_intervals_v2[] = { // 0x1100u, 0x1160u, 0x231Au, 0x231Cu, 0x2329u, 0x232Bu, 0x23E9u, 0x23EDu, 0x23F0u, 0x23F1u, 0x23F3u, 0x23F4u, 0x25FDu, 0x25FFu, 0x2614u, 0x2616u, 0x2648u, 0x2654u, 0x267Fu, 0x2680u, 0x2693u, 0x2694u, 0x26A1u, 0x26A2u, 0x26AAu, 0x26ACu, 0x26BDu, 0x26BFu, 0x26C4u, 0x26C6u, 0x26CEu, 0x26CFu, 0x26D4u, 0x26D5u, 0x26EAu, 0x26EBu, 0x26F2u, 0x26F4u, 0x26F5u, @@ -1001,10 +1001,10 @@ inline constexpr char32_t _Width_estimate_intervals[] = { // _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { // Computes the width estimation for Unicode characters from N4950 [format.string.std]/13 - constexpr char32_t _Limit = _Width_estimate_intervals[12]; + constexpr char32_t _Limit = _Width_estimate_intervals_v2[12]; if (_Ch < _Limit) { int _Result = 1; - for (const auto& _Bound : _Width_estimate_intervals) { + for (const auto& _Bound : _Width_estimate_intervals_v2) { if (_Ch < _Bound) { return _Result; } @@ -1012,8 +1012,9 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { } return 1; } else { - int _Upper_bound_index = static_cast( - upper_bound(_Width_estimate_intervals, end(_Width_estimate_intervals), _Ch) - _Width_estimate_intervals); + int _Upper_bound_index = + static_cast(upper_bound(_Width_estimate_intervals_v2, end(_Width_estimate_intervals_v2), _Ch) + - _Width_estimate_intervals_v2); return 1 + (_Upper_bound_index & 1); } } diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index b76cfb5cd52..92822f876c3 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// The following code generates data for `_Width_estimate_intervals` in . +// The following code generates data for `_Width_estimate_intervals_v2` in . #include #include @@ -43,7 +43,7 @@ class table_u { } void print_intervals() const { - // Print table for `_Width_estimate_intervals`. + // Print table for `_Width_estimate_intervals_v2`. using namespace std; int c = 0; width_u last = table[0]; From 6d31de1a5532f0d13d81b8248d4588015b1a260a Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:23:27 -0700 Subject: [PATCH 17/35] `_STD` qualify `upper_bound` and `end`. --- stl/inc/format | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/format b/stl/inc/format index 3d0d2222212..92e2732b4b0 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -1013,7 +1013,7 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { return 1; } else { int _Upper_bound_index = - static_cast(upper_bound(_Width_estimate_intervals_v2, end(_Width_estimate_intervals_v2), _Ch) + static_cast(_STD upper_bound(_Width_estimate_intervals_v2, _STD end(_Width_estimate_intervals_v2), _Ch) - _Width_estimate_intervals_v2); return 1 + (_Upper_bound_index & 1); } From 25dc1c5735cf47326b47b729d4dc3bf9c62d7d22 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:25:41 -0700 Subject: [PATCH 18/35] `_Upper_bound_index` can be `const ptrdiff_t`. --- stl/inc/format | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 92e2732b4b0..7896128a82f 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -1012,9 +1012,9 @@ _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { } return 1; } else { - int _Upper_bound_index = - static_cast(_STD upper_bound(_Width_estimate_intervals_v2, _STD end(_Width_estimate_intervals_v2), _Ch) - - _Width_estimate_intervals_v2); + const ptrdiff_t _Upper_bound_index = + _STD upper_bound(_Width_estimate_intervals_v2, _STD end(_Width_estimate_intervals_v2), _Ch) + - _Width_estimate_intervals_v2; return 1 + (_Upper_bound_index & 1); } } From 0335f7fcf82129403ff724fdaf67fb8f07d50902 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:36:31 -0700 Subject: [PATCH 19/35] Define data members on separate lines. --- .../format_width_estimate_intervals.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 92822f876c3..9fa55291dad 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -20,7 +20,8 @@ void verify_impl(bool test, int line, const char* msg) { constexpr const char* impl_assertion_failed = "impl assertion failed"; struct range_u { - uint32_t from, to; + uint32_t from; + uint32_t to; range_u(uint32_t f, uint32_t t) : from(f), to(t) {} range_u(uint32_t v) : from(v), to(v) {} }; From 9ee2ed293bdc4e30058abf8a61ec76629dbe2e96 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:39:55 -0700 Subject: [PATCH 20/35] Lift using-directive to file scope; all shall love std and despair. --- .../format_width_estimate_intervals.cpp | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 9fa55291dad..af9f4deacb9 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -9,10 +9,11 @@ #include #include #include +using namespace std; void verify_impl(bool test, int line, const char* msg) { if (!test) { - std::cerr << "Error at line " << line << ": " << msg << std::endl; + cerr << "Error at line " << line << ": " << msg << endl; exit(EXIT_FAILURE); } } @@ -31,7 +32,7 @@ enum class width_u : bool { is_1 = false, is_2 = true }; class table_u { // A valid Unicode code point won't exceed `max_u`. static constexpr uint32_t max_u = 0x10'ffff; - std::vector table; + vector table; public: table_u() : table(max_u + 1, width_u::is_1) {} @@ -45,7 +46,6 @@ class table_u { void print_intervals() const { // Print table for `_Width_estimate_intervals_v2`. - using namespace std; int c = 0; width_u last = table[0]; for (uint32_t u = 0; u <= max_u; u++) { @@ -62,7 +62,6 @@ class table_u { } void print_clusters_1_vs_2(const table_u& other) const { - using namespace std; vector cluster_table(max_u + 1, false); for (uint32_t u = 0; u <= max_u; u++) { if (table[u] == width_u::is_1 && other.table[u] == width_u::is_2) { @@ -119,8 +118,7 @@ table_u get_table_cpp20() { // The current implementation works for: // https://www.unicode.org/Public/15.0.0/ucd/EastAsianWidth.txt // To make this function work, the file should not contain a BOM. -table_u read_from(std::ifstream& source) { - using namespace std; +table_u read_from(ifstream& source) { table_u table; // "The unassigned code points in the following blocks default to "W":" @@ -172,7 +170,7 @@ table_u read_from(std::ifstream& source) { return table; } -table_u get_table_cpp23(std::ifstream& source) { +table_u get_table_cpp23(ifstream& source) { table_u table = read_from(source); // Override with ranges specified by the C++ standard. @@ -185,8 +183,6 @@ table_u get_table_cpp23(std::ifstream& source) { } int main() { - using namespace std; - cout << "Old table:\n"; const table_u old_table = get_table_cpp20(); old_table.print_intervals(); From 071fa7b3bc5d2c35e148d22b935bbd8576f4b4d5 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:42:03 -0700 Subject: [PATCH 21/35] `verify` => `VERIFY` because it's a macro. --- .../format_width_estimate_intervals.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index af9f4deacb9..0b55aefd44f 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -17,7 +17,7 @@ void verify_impl(bool test, int line, const char* msg) { exit(EXIT_FAILURE); } } -#define verify(expr, msg) verify_impl((expr), __LINE__, (msg)) +#define VERIFY(expr, msg) verify_impl((expr), __LINE__, (msg)) constexpr const char* impl_assertion_failed = "impl assertion failed"; struct range_u { @@ -38,7 +38,7 @@ class table_u { table_u() : table(max_u + 1, width_u::is_1) {} void fill_range(const range_u rng, const width_u width) { const auto [from, to] = rng; - verify(from <= to && to <= max_u, impl_assertion_failed); + VERIFY(from <= to && to <= max_u, impl_assertion_failed); for (uint32_t u = from; u <= to; u++) { table[u] = width; } @@ -134,31 +134,31 @@ table_u read_from(ifstream& source) { if (str == "F" || str == "W") { return width_u::is_2; } else { - verify(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); + VERIFY(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); return width_u::is_1; } }; auto get_value = [](const string& str) -> uint32_t { uint32_t value{}; auto [end, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); - verify(end == str.data() + str.size() && ec == errc{}, impl_assertion_failed); + VERIFY(end == str.data() + str.size() && ec == errc{}, impl_assertion_failed); return value; }; - verify(!!source, "invalid path"); + VERIFY(!!source, "invalid path"); string line; const regex reg(R"(([0-9A-Z]+)(\.\.[0-9A-Z]+)?;(A|F|H|N|Na|W) *#.*)"); while (getline(source, line)) { if (!line.empty() && !line.starts_with("#")) { smatch match; - verify(regex_match(line, match, reg), "invalid line"); - verify(match[1].matched && match[3].matched, impl_assertion_failed); + VERIFY(regex_match(line, match, reg), "invalid line"); + VERIFY(match[1].matched && match[3].matched, impl_assertion_failed); const width_u width = get_width(match[3].str()); const uint32_t from = get_value(match[1].str()); if (match[2].matched) { // range (HEX..HEX) string match2 = match[2].str(); - verify(match2.starts_with(".."), impl_assertion_failed); + VERIFY(match2.starts_with(".."), impl_assertion_failed); table.fill_range({from, get_value(match2.substr(2))}, width); } else { // single character (HEX) From 5fbd710bdb4247d467968d55a0d561cf2b576b23 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:52:34 -0700 Subject: [PATCH 22/35] Adjust included headers. --- .../format_width_estimate_intervals.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 0b55aefd44f..e7b6a9d83df 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -4,10 +4,13 @@ // The following code generates data for `_Width_estimate_intervals_v2` in . #include -#include +#include +#include #include #include #include +#include +#include #include using namespace std; From 6f207a0f60757d2d4ed8f6392864e0d9aed3d704 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:54:26 -0700 Subject: [PATCH 23/35] Prefer preincrement: `u++` => `++u` --- .../format_width_estimate_intervals.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index e7b6a9d83df..32dd5baaa32 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -42,7 +42,7 @@ class table_u { void fill_range(const range_u rng, const width_u width) { const auto [from, to] = rng; VERIFY(from <= to && to <= max_u, impl_assertion_failed); - for (uint32_t u = from; u <= to; u++) { + for (uint32_t u = from; u <= to; ++u) { table[u] = width; } } @@ -51,7 +51,7 @@ class table_u { // Print table for `_Width_estimate_intervals_v2`. int c = 0; width_u last = table[0]; - for (uint32_t u = 0; u <= max_u; u++) { + for (uint32_t u = 0; u <= max_u; ++u) { if (table[u] != last) { cout << "0x" << hex << uppercase << u << "u, "; if (++c == 12) { @@ -66,13 +66,13 @@ class table_u { void print_clusters_1_vs_2(const table_u& other) const { vector cluster_table(max_u + 1, false); - for (uint32_t u = 0; u <= max_u; u++) { + for (uint32_t u = 0; u <= max_u; ++u) { if (table[u] == width_u::is_1 && other.table[u] == width_u::is_2) { cluster_table[u] = true; } } - for (uint32_t u = 0; u <= max_u; u++) { + for (uint32_t u = 0; u <= max_u; ++u) { if (cluster_table[u]) { uint32_t from = u; uint32_t to = from; From 3474a226cf93cef7d4ebf2b871304623b483d7ae Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:58:56 -0700 Subject: [PATCH 24/35] These lambdas can use implicit return types. --- .../format_width_estimate_intervals.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 32dd5baaa32..0a3d5d9936c 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -133,7 +133,7 @@ table_u read_from(ifstream& source) { // Read explicitly assigned ranges. // The lines that are not empty or pure comment are uniformly of the format "HEX(..HEX)?;(A|F|H|N|Na|W) #comment". - auto get_width = [](const string& str) -> width_u { + auto get_width = [](const string& str) { if (str == "F" || str == "W") { return width_u::is_2; } else { @@ -141,7 +141,7 @@ table_u read_from(ifstream& source) { return width_u::is_1; } }; - auto get_value = [](const string& str) -> uint32_t { + auto get_value = [](const string& str) { uint32_t value{}; auto [end, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); VERIFY(end == str.data() + str.size() && ec == errc{}, impl_assertion_failed); From d3ba486cb1fece2f087af0bd54a223e586d4e251 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 17:59:17 -0700 Subject: [PATCH 25/35] Separately verify conditions. --- .../format_width_estimate_intervals.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 0a3d5d9936c..894d424a939 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -41,7 +41,8 @@ class table_u { table_u() : table(max_u + 1, width_u::is_1) {} void fill_range(const range_u rng, const width_u width) { const auto [from, to] = rng; - VERIFY(from <= to && to <= max_u, impl_assertion_failed); + VERIFY(from <= to, impl_assertion_failed); + VERIFY(to <= max_u, impl_assertion_failed); for (uint32_t u = from; u <= to; ++u) { table[u] = width; } @@ -144,7 +145,8 @@ table_u read_from(ifstream& source) { auto get_value = [](const string& str) { uint32_t value{}; auto [end, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); - VERIFY(end == str.data() + str.size() && ec == errc{}, impl_assertion_failed); + VERIFY(end == str.data() + str.size(), impl_assertion_failed); + VERIFY(ec == errc{}, impl_assertion_failed); return value; }; @@ -155,7 +157,8 @@ table_u read_from(ifstream& source) { if (!line.empty() && !line.starts_with("#")) { smatch match; VERIFY(regex_match(line, match, reg), "invalid line"); - VERIFY(match[1].matched && match[3].matched, impl_assertion_failed); + VERIFY(match[1].matched, impl_assertion_failed); + VERIFY(match[3].matched, impl_assertion_failed); const width_u width = get_width(match[3].str()); const uint32_t from = get_value(match[1].str()); if (match[2].matched) { From 19d7b95bf13d4a14a33d99372649e7dc82b23cdb Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:02:10 -0700 Subject: [PATCH 26/35] Iterate in-place with range-for. --- .../format_width_estimate_intervals.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 894d424a939..eade61aff09 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -110,7 +110,7 @@ table_u get_table_cpp20() { }; table_u table; - for (const range_u rng : std_wide_ranges_cpp20) { + for (const range_u& rng : std_wide_ranges_cpp20) { table.fill_range(rng, width_u::is_2); } return table; @@ -128,7 +128,7 @@ table_u read_from(ifstream& source) { // "The unassigned code points in the following blocks default to "W":" const range_u default_wide_ranges[]{ {0x4E00, 0x9FFF}, {0x3400, 0x4DBF}, {0xF900, 0xFAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}}; - for (const range_u rng : default_wide_ranges) { + for (const range_u& rng : default_wide_ranges) { table.fill_range(rng, width_u::is_2); } @@ -181,7 +181,7 @@ table_u get_table_cpp23(ifstream& source) { // Override with ranges specified by the C++ standard. const range_u std_wide_ranges_cpp23[]{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; - for (const range_u rng : std_wide_ranges_cpp23) { + for (const range_u& rng : std_wide_ranges_cpp23) { table.fill_range(rng, width_u::is_2); } From 21c77a80cb221a261028a7f71f4ced0a0f7ae44a Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:12:57 -0700 Subject: [PATCH 27/35] Use `explicit range_u(uint32_t)` to avoid mistakes. --- .../format_width_estimate_intervals.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index eade61aff09..9c6d8ecd2df 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -27,7 +27,7 @@ struct range_u { uint32_t from; uint32_t to; range_u(uint32_t f, uint32_t t) : from(f), to(t) {} - range_u(uint32_t v) : from(v), to(v) {} + explicit range_u(uint32_t v) : from(v), to(v) {} }; enum class width_u : bool { is_1 = false, is_2 = true }; @@ -168,7 +168,7 @@ table_u read_from(ifstream& source) { table.fill_range({from, get_value(match2.substr(2))}, width); } else { // single character (HEX) - table.fill_range({from}, width); + table.fill_range(range_u{from}, width); } } } From 2d186c5b94fec26aa658305228f7e8d169806ced Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:18:16 -0700 Subject: [PATCH 28/35] Upgrade tables to `static constexpr`. --- .../format_width_estimate_intervals.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 9c6d8ecd2df..187441fd121 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -26,8 +26,8 @@ constexpr const char* impl_assertion_failed = "impl assertion failed"; struct range_u { uint32_t from; uint32_t to; - range_u(uint32_t f, uint32_t t) : from(f), to(t) {} - explicit range_u(uint32_t v) : from(v), to(v) {} + constexpr range_u(uint32_t f, uint32_t t) : from(f), to(t) {} + constexpr explicit range_u(uint32_t v) : from(v), to(v) {} }; enum class width_u : bool { is_1 = false, is_2 = true }; @@ -92,7 +92,7 @@ class table_u { }; table_u get_table_cpp20() { - const range_u std_wide_ranges_cpp20[]{ + static constexpr range_u std_wide_ranges_cpp20[]{ {0x1100, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x303E}, @@ -126,7 +126,7 @@ table_u read_from(ifstream& source) { table_u table; // "The unassigned code points in the following blocks default to "W":" - const range_u default_wide_ranges[]{ + static constexpr range_u default_wide_ranges[]{ {0x4E00, 0x9FFF}, {0x3400, 0x4DBF}, {0xF900, 0xFAFF}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}}; for (const range_u& rng : default_wide_ranges) { table.fill_range(rng, width_u::is_2); @@ -180,7 +180,7 @@ table_u get_table_cpp23(ifstream& source) { table_u table = read_from(source); // Override with ranges specified by the C++ standard. - const range_u std_wide_ranges_cpp23[]{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; + static constexpr range_u std_wide_ranges_cpp23[]{{0x4DC0, 0x4DFF}, {0x1F300, 0x1F5FF}, {0x1F900, 0x1F9FF}}; for (const range_u& rng : std_wide_ranges_cpp23) { table.fill_range(rng, width_u::is_2); } From d36246df0b89bb448fd8b1115a793c4e8d726be5 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:19:50 -0700 Subject: [PATCH 29/35] Explicitly say `private:`. --- .../unicode_properties_parse/format_width_estimate_intervals.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 187441fd121..be5f399eed7 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -33,6 +33,7 @@ struct range_u { enum class width_u : bool { is_1 = false, is_2 = true }; class table_u { +private: // A valid Unicode code point won't exceed `max_u`. static constexpr uint32_t max_u = 0x10'ffff; vector table; From 308471dfacacb9bd404f384c774442e834897f28 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:23:01 -0700 Subject: [PATCH 30/35] Add newline between function definitions. --- .../unicode_properties_parse/format_width_estimate_intervals.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index be5f399eed7..383bb288e7e 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -40,6 +40,7 @@ class table_u { public: table_u() : table(max_u + 1, width_u::is_1) {} + void fill_range(const range_u rng, const width_u width) { const auto [from, to] = rng; VERIFY(from <= to, impl_assertion_failed); From b001724c101ce53f10a7c60815068e6cc8b8fd41 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:35:28 -0700 Subject: [PATCH 31/35] Avoid shadowing: `end` => `end_ptr` --- .../format_width_estimate_intervals.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index 383bb288e7e..b248e2d6564 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -146,8 +146,8 @@ table_u read_from(ifstream& source) { }; auto get_value = [](const string& str) { uint32_t value{}; - auto [end, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); - VERIFY(end == str.data() + str.size(), impl_assertion_failed); + auto [end_ptr, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); + VERIFY(end_ptr == str.data() + str.size(), impl_assertion_failed); VERIFY(ec == errc{}, impl_assertion_failed); return value; }; From 2371dab47bc49499b60433204b8c63f19e40e59b Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 25 Jul 2023 18:36:05 -0700 Subject: [PATCH 32/35] Add `const`. --- .../format_width_estimate_intervals.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index b248e2d6564..f8b4bbbd69d 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -146,7 +146,7 @@ table_u read_from(ifstream& source) { }; auto get_value = [](const string& str) { uint32_t value{}; - auto [end_ptr, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); + const auto [end_ptr, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); VERIFY(end_ptr == str.data() + str.size(), impl_assertion_failed); VERIFY(ec == errc{}, impl_assertion_failed); return value; @@ -165,7 +165,7 @@ table_u read_from(ifstream& source) { const uint32_t from = get_value(match[1].str()); if (match[2].matched) { // range (HEX..HEX) - string match2 = match[2].str(); + const string match2 = match[2].str(); VERIFY(match2.starts_with(".."), impl_assertion_failed); table.fill_range({from, get_value(match2.substr(2))}, width); } else { From 06cabc260dad0590f985a49c2a9bc9b1703bcb89 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Thu, 27 Jul 2023 01:51:14 +0800 Subject: [PATCH 33/35] verify_impl&VERIFY -> verify; add const --- .../format_width_estimate_intervals.cpp | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp index f8b4bbbd69d..06946779b7d 100644 --- a/tools/unicode_properties_parse/format_width_estimate_intervals.cpp +++ b/tools/unicode_properties_parse/format_width_estimate_intervals.cpp @@ -9,18 +9,18 @@ #include #include #include +#include #include #include #include using namespace std; -void verify_impl(bool test, int line, const char* msg) { +void verify(bool test, const char* msg, source_location loc = source_location::current()) { if (!test) { - cerr << "Error at line " << line << ": " << msg << endl; + cerr << "Error at line " << loc.line() << ": " << msg << endl; exit(EXIT_FAILURE); } } -#define VERIFY(expr, msg) verify_impl((expr), __LINE__, (msg)) constexpr const char* impl_assertion_failed = "impl assertion failed"; struct range_u { @@ -43,8 +43,8 @@ class table_u { void fill_range(const range_u rng, const width_u width) { const auto [from, to] = rng; - VERIFY(from <= to, impl_assertion_failed); - VERIFY(to <= max_u, impl_assertion_failed); + verify(from <= to, impl_assertion_failed); + verify(to <= max_u, impl_assertion_failed); for (uint32_t u = from; u <= to; ++u) { table[u] = width; } @@ -77,8 +77,8 @@ class table_u { for (uint32_t u = 0; u <= max_u; ++u) { if (cluster_table[u]) { - uint32_t from = u; - uint32_t to = from; + const uint32_t from = u; + uint32_t to = from; while (to + 1 <= max_u && cluster_table[to + 1]) { ++to; } @@ -140,33 +140,33 @@ table_u read_from(ifstream& source) { if (str == "F" || str == "W") { return width_u::is_2; } else { - VERIFY(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); + verify(str == "A" || str == "H" || str == "N" || str == "Na", impl_assertion_failed); return width_u::is_1; } }; auto get_value = [](const string& str) { uint32_t value{}; const auto [end_ptr, ec] = from_chars(str.data(), str.data() + str.size(), value, 16); - VERIFY(end_ptr == str.data() + str.size(), impl_assertion_failed); - VERIFY(ec == errc{}, impl_assertion_failed); + verify(end_ptr == str.data() + str.size(), impl_assertion_failed); + verify(ec == errc{}, impl_assertion_failed); return value; }; - VERIFY(!!source, "invalid path"); + verify(!!source, "invalid path"); string line; const regex reg(R"(([0-9A-Z]+)(\.\.[0-9A-Z]+)?;(A|F|H|N|Na|W) *#.*)"); while (getline(source, line)) { if (!line.empty() && !line.starts_with("#")) { smatch match; - VERIFY(regex_match(line, match, reg), "invalid line"); - VERIFY(match[1].matched, impl_assertion_failed); - VERIFY(match[3].matched, impl_assertion_failed); + verify(regex_match(line, match, reg), "invalid line"); + verify(match[1].matched, impl_assertion_failed); + verify(match[3].matched, impl_assertion_failed); const width_u width = get_width(match[3].str()); const uint32_t from = get_value(match[1].str()); if (match[2].matched) { // range (HEX..HEX) const string match2 = match[2].str(); - VERIFY(match2.starts_with(".."), impl_assertion_failed); + verify(match2.starts_with(".."), impl_assertion_failed); table.fill_range({from, get_value(match2.substr(2))}, width); } else { // single character (HEX) @@ -199,7 +199,7 @@ int main() { string path; getline(cin, path); ifstream source(path); - table_u new_table = get_table_cpp23(source); + const table_u new_table = get_table_cpp23(source); new_table.print_intervals(); cout << "\nWas 1, now 2:\n"; From 5ac83464bd54656c2eca9c3b5148a52ba5a55b88 Mon Sep 17 00:00:00 2001 From: achabense <60953653+achabense@users.noreply.github.com> Date: Thu, 27 Jul 2023 11:39:17 +0800 Subject: [PATCH 34/35] review feedback; add comments for `_Unicode_width_estimate`; add test cases: all 2->1 cases; and some 1->2 cases --- stl/inc/format | 5 +- .../P0645R10_text_formatting_utf8/test.cpp | 58 +++++++++++++++++-- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/stl/inc/format b/stl/inc/format index 7896128a82f..cfe4426ff92 100644 --- a/stl/inc/format +++ b/stl/inc/format @@ -1001,8 +1001,9 @@ inline constexpr char32_t _Width_estimate_intervals_v2[] = { // _NODISCARD constexpr int _Unicode_width_estimate(const char32_t _Ch) noexcept { // Computes the width estimation for Unicode characters from N4950 [format.string.std]/13 - constexpr char32_t _Limit = _Width_estimate_intervals_v2[12]; - if (_Ch < _Limit) { + // The two branches are functionally equivalent; `12` is chosen for performance here. + constexpr char32_t _Linear_search_threshold = _Width_estimate_intervals_v2[12]; + if (_Ch < _Linear_search_threshold) { int _Result = 1; for (const auto& _Bound : _Width_estimate_intervals_v2) { if (_Ch < _Bound) { diff --git a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp index c049ad8399f..0ddf3e2b0b7 100644 --- a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp +++ b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp @@ -69,7 +69,7 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\x58"), 1}, {TYPED_LITERAL(CharT, "x\x58"), 2}, - // test the boundaries of the intervals defined in n4885 [format.string.std]/11 + // Test the boundaries of the intervals defined in n4885 [format.string.std]/11 {TYPED_LITERAL(CharT, "\u10ff\x58"), 2}, {TYPED_LITERAL(CharT, "\u1100\x58"), 3}, {TYPED_LITERAL(CharT, "\u115f\x58"), 3}, @@ -82,8 +82,6 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\u2e80\x58"), 3}, {TYPED_LITERAL(CharT, "\u303e\x58"), 3}, {TYPED_LITERAL(CharT, "\u303f\x58"), 2}, - {TYPED_LITERAL(CharT, "\u3040\x58"), 2}, - {TYPED_LITERAL(CharT, "\ua4cf\x58"), 2}, {TYPED_LITERAL(CharT, "\ua4d0\x58"), 2}, {TYPED_LITERAL(CharT, "\uabff\x58"), 2}, {TYPED_LITERAL(CharT, "\uac00\x58"), 3}, @@ -91,7 +89,7 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ud7a4\x58"), 2}, {TYPED_LITERAL(CharT, "\ud7ff\x58"), 2}, - // skip over the surrogate pair range (\ud800-\udfff) + // Skip over the surrogate pair range (\ud800-\udfff) {TYPED_LITERAL(CharT, "\ue000\x58"), 2}, {TYPED_LITERAL(CharT, "\uf8ff\x58"), 2}, @@ -104,10 +102,8 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ufe1a\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe2f\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe30\x58"), 3}, - {TYPED_LITERAL(CharT, "\ufe6f\x58"), 2}, {TYPED_LITERAL(CharT, "\ufe70\x58"), 2}, {TYPED_LITERAL(CharT, "\ufeff\x58"), 2}, - {TYPED_LITERAL(CharT, "\uff00\x58"), 2}, {TYPED_LITERAL(CharT, "\uff60\x58"), 3}, {TYPED_LITERAL(CharT, "\uff61\x58"), 2}, {TYPED_LITERAL(CharT, "\uffdf\x58"), 2}, @@ -143,8 +139,58 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\ufe40\u2000\ufe40\x58"), 6}, {TYPED_LITERAL(CharT, "\ufe40\ufe40\u2000\x58"), 6}, {TYPED_LITERAL(CharT, "\ufe40\ufe40\ufe40\x58"), 7}, + {TYPED_LITERAL(CharT, "\u0061\u200D\x58"), 2}, {TYPED_LITERAL(CharT, "\u0061\U0001F1E6\U0001F1E7\u200D\U0001F1E8\u0062\x58"), 5}, + + // Test some codepoints affected by P2675R1 + // Used to be 2, now 1: + {TYPED_LITERAL(CharT, "\u2e9a\x58"), 2}, + {TYPED_LITERAL(CharT, "\u2ef4\u2eff\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2fd6\u2fef\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2ffc\u2fff\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3040\x58"), 2}, + {TYPED_LITERAL(CharT, "\u3097\u3098\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3100\u3104\x58"), 3}, + {TYPED_LITERAL(CharT, "\u3130\x58"), 2}, + {TYPED_LITERAL(CharT, "\u318f\x58"), 2}, + {TYPED_LITERAL(CharT, "\u31e4\u31ef\x58"), 3}, + {TYPED_LITERAL(CharT, "\u321f\x58"), 2}, + {TYPED_LITERAL(CharT, "\u3248\u324f\x58"), 3}, + {TYPED_LITERAL(CharT, "\ua48d\ua48f\x58"), 3}, + {TYPED_LITERAL(CharT, "\ua4c7\ua4cf\x58"), 3}, + {TYPED_LITERAL(CharT, "\ufe53\x58"), 2}, + {TYPED_LITERAL(CharT, "\ufe67\x58"), 2}, + {TYPED_LITERAL(CharT, "\ufe6c\ufe6f\x58"), 3}, + {TYPED_LITERAL(CharT, "\uff00\x58"), 2}, + // Used to be 1, now 2: + {TYPED_LITERAL(CharT, "\u23f0\x58"), 3}, + {TYPED_LITERAL(CharT, "\u23f3\x58"), 3}, + {TYPED_LITERAL(CharT, "\u267f\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2693\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26a1\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26ce\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26d4\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26ea\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26f5\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26fa\x58"), 3}, + {TYPED_LITERAL(CharT, "\u26fd\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2705\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2728\x58"), 3}, + {TYPED_LITERAL(CharT, "\u274c\x58"), 3}, + {TYPED_LITERAL(CharT, "\u274e\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2757\x58"), 3}, + {TYPED_LITERAL(CharT, "\u27b0\x58"), 3}, + {TYPED_LITERAL(CharT, "\u27bf\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2b50\x58"), 3}, + {TYPED_LITERAL(CharT, "\u2b55\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001b132\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001b155\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f004\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f0cf\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f18e\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f6cc\x58"), 3}, + {TYPED_LITERAL(CharT, "\U0001f7f0\x58"), 3}, }; for (const auto& test : test_cases) { From b6080d59dab843616f89d2abdee1ff3e1634f82f Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Thu, 27 Jul 2023 13:05:10 -0700 Subject: [PATCH 35/35] Update Standardese citation, drop outdated explanation. --- tests/std/tests/P0645R10_text_formatting_utf8/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp index 0ddf3e2b0b7..db01cedbefa 100644 --- a/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp +++ b/tests/std/tests/P0645R10_text_formatting_utf8/test.cpp @@ -69,7 +69,7 @@ void test_width_estimation() { {TYPED_LITERAL(CharT, "\x58"), 1}, {TYPED_LITERAL(CharT, "x\x58"), 2}, - // Test the boundaries of the intervals defined in n4885 [format.string.std]/11 + // Test N4950 [format.string.std]/13 {TYPED_LITERAL(CharT, "\u10ff\x58"), 2}, {TYPED_LITERAL(CharT, "\u1100\x58"), 3}, {TYPED_LITERAL(CharT, "\u115f\x58"), 3},