From 898777a69c30cbc8d564b92262db306f51c2dd8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 26 Apr 2025 14:16:38 +0200 Subject: [PATCH 01/13] ``: `regex_traits::transform_primary` should yield primary sort keys appropriate for the imbued locale --- stl/CMakeLists.txt | 1 + stl/inc/locale | 5 + stl/inc/regex | 85 +++++++++--- stl/src/regex.cpp | 124 ++++++++++++++++++ .../GH_005204_regex_collating_ranges/test.cpp | 80 +++++++++-- tests/tr1/tests/regex1/test.cpp | 2 +- tests/tr1/tests/regex2/test.cpp | 2 +- 7 files changed, 270 insertions(+), 29 deletions(-) create mode 100644 stl/src/regex.cpp diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index 968d98987ec..079350673d3 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -213,6 +213,7 @@ set(IMPLIB_SOURCES ${CMAKE_CURRENT_LIST_DIR}/src/locale0_implib.cpp ${CMAKE_CURRENT_LIST_DIR}/src/nothrow.cpp ${CMAKE_CURRENT_LIST_DIR}/src/print.cpp + ${CMAKE_CURRENT_LIST_DIR}/src/regex.cpp ${CMAKE_CURRENT_LIST_DIR}/src/sharedmutex.cpp ${CMAKE_CURRENT_LIST_DIR}/src/stacktrace.cpp ${CMAKE_CURRENT_LIST_DIR}/src/syserror_import_lib.cpp diff --git a/stl/inc/locale b/stl/inc/locale index 41963efa509..bb04cd80148 100644 --- a/stl/inc/locale +++ b/stl/inc/locale @@ -94,6 +94,9 @@ inline size_t __CRTDECL _LStrxfrm(_Out_writes_(_Last1 - _First1) _Post_readable_ } #endif // defined(_CRTBLD) +template +class _Regex_traits; + _EXPORT_STD template class collate : public locale::facet { // facet for ordering sequences of elements public: @@ -185,6 +188,8 @@ protected: private: _Locinfo::_Collvec _Coll; // used by _LStrcoll and _XStrxfrm + + friend class _Regex_traits<_Elem>; }; #ifdef __clang__ diff --git a/stl/inc/regex b/stl/inc/regex index b2023ebd87c..271909ad739 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -53,6 +53,15 @@ _STL_DISABLE_CLANG_WARNINGS #endif // ^^^ !defined(_DEBUG) ^^^ #endif // !defined(_ENHANCED_REGEX_VISUALIZER) +extern "C" { +_STD size_t __stdcall __std_regex_transform_primary_char( + _Out_writes_(_Last1 - _First1) _Post_readable_size_(return) char* _First1, char* _Last1, + _In_reads_(_Last2 - _First2) const char* _First2, const char* _Last2, _In_opt_ const _Collvec*) noexcept; +_STD size_t __stdcall __std_regex_transform_primary_wchar_t( + _Out_writes_(_Last1 - _First1) _Post_readable_size_(return) wchar_t* _First1, wchar_t* _Last1, + _In_reads_(_Last2 - _First2) const wchar_t* _First2, const wchar_t* _Last2, _In_opt_ const _Collvec*) noexcept; +} // extern "C" + _STD_BEGIN enum _Meta_type : int { // meta character representations for parser @@ -267,6 +276,18 @@ struct _Regex_traits_base { // base of all regular expression traits using char_class_type = ctype_base::mask; }; +inline size_t _Regex_transform_primary(_Out_writes_(_Last1 - _First1) _Post_readable_size_(return) char* _First1, + char* _Last1, _In_reads_(_Last2 - _First2) const char* _First2, const char* _Last2, + _In_opt_ const _Locinfo::_Collvec* _Vector) noexcept { + return __std_regex_transform_primary_char(_First1, _Last1, _First2, _Last2, _Vector); +} + +inline size_t _Regex_transform_primary(_Out_writes_(_Last1 - _First1) _Post_readable_size_(return) wchar_t* _First1, + wchar_t* _Last1, _In_reads_(_Last2 - _First2) const wchar_t* _First2, const wchar_t* _Last2, + _In_opt_ const _Locinfo::_Collvec* _Vector) noexcept { + return __std_regex_transform_primary_wchar_t(_First1, _Last1, _First2, _Last2, _Vector); +} + template class _Regex_traits : public _Regex_traits_base { // base class for regular expression traits public: @@ -312,13 +333,35 @@ public: string_type transform_primary(_FwdIt _First, _FwdIt _Last) const { // apply locale-specific case-insensitive transformation string_type _Res; - - if (_First != _Last) { // non-empty string, transform it - vector<_Elem> _Temp(_First, _Last); - - _Getctype()->tolower(_Temp.data(), _Temp.data() + _Temp.size()); - _Res = _Getcoll()->transform(_Temp.data(), _Temp.data() + _Temp.size()); +#ifdef _CPPRTTI + if (_First != _Last) { + const collate<_Elem>* _Coll = _Getcoll(); + const auto& _Coll_type = typeid(*_Coll); + // TRANSITION, ABI: GH-5394: locale creates collate objects of type collate, not collate_byname + // Depending on the resolution of LWG-2338, comparison to typeid(collate) might also become + // required by the standard + if (_Coll_type == typeid(collate_byname<_Elem>) || _Coll_type == typeid(collate<_Elem>)) { + // non-empty string with known collate facet, transform it + size_t _Count; + const string_type _Src(_First, _Last); + const auto _Src_first = _Src.data(); + const auto _Src_last = _Src_first + _Src.size(); + + for (_Count = _Src.size(); _Res.size() < _Count;) { + _Res.resize(_Count); + _Count = _STD _Regex_transform_primary( + &_Res[0], &_Res[0] + _Count, _Src_first, _Src_last, &_Coll->_Coll); + + if (_Count == static_cast(-1)) { + // return empty string in case of error + _Count = 0; + break; + } + } + _Res.resize(_Count); + } } +#endif // defined(_CPPRTTI) return _Res; } @@ -4181,26 +4224,30 @@ _Prs_ret _Parser<_FwdIt, _Elem, _RxTraits>::_Do_ex_class2( _Elem* const _Coll_elem_first = &_Coll_elem.front(); const _Elem* const _Coll_elem_last = _Coll_elem_first + _Size; + + if (_Size == 1 && _End_arg == _Meta_dot) { + // process single-element collating elements like individual characters + _Val = *_Coll_elem_first; + return _Prs_chr; + } + + if (_Flags & regex_constants::icase) { + for (auto _Current = _Coll_elem_first; _Current != _Coll_elem_last; ++_Current) { + *_Current = _Traits.translate_nocase(*_Current); + } + } else if (_Flags & regex_constants::collate) { + for (auto _Current = _Coll_elem_first; _Current != _Coll_elem_last; ++_Current) { + *_Current = _Traits.translate(*_Current); + } + } + if (_End_arg == _Meta_equal) { // process equivalence _Nfa._Add_equiv2(_Coll_elem_first, _Coll_elem_last); return _Prs_set; } else { // process collating element - if (_Size == 1) { - _Val = *_Coll_elem_first; - return _Prs_chr; - } // Character ranges with multi-character bounds cannot be represented in NFA nodes yet (see GH-5391). // Provisionally treat multi-character collating elements as character sets. - if (_Flags & regex_constants::icase) { - for (auto _Current = _Coll_elem_first; _Current != _Coll_elem_last; ++_Current) { - *_Current = _Traits.translate_nocase(*_Current); - } - } else if (_Flags & regex_constants::collate) { - for (auto _Current = _Coll_elem_first; _Current != _Coll_elem_last; ++_Current) { - *_Current = _Traits.translate(*_Current); - } - } _Nfa._Add_coll2(_Coll_elem_first, _Coll_elem_last); return _Prs_set; } diff --git a/stl/src/regex.cpp b/stl/src/regex.cpp new file mode 100644 index 00000000000..860776f1eb1 --- /dev/null +++ b/stl/src/regex.cpp @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include <__msvc_xlocinfo_types.hpp> +#include +#include +#include +#include +#include + +#include + +#undef _ENFORCE_ONLY_CORE_HEADERS +#include "awint.hpp" + +extern "C" { + +// derived from xstrxfrm.cpp +size_t __stdcall __std_regex_transform_primary_char( + _Out_writes_(end1 - string1) _Post_readable_size_(return) char* string1, char* end1, + _In_reads_(end2 - string2) const char* string2, const char* end2, _In_opt_ const _Collvec* ploc) noexcept { + size_t n1 = end1 - string1; + size_t n2 = end2 - string2; + size_t retval = static_cast(-1); + UINT codepage; + const wchar_t* locale_name; + + if (ploc == nullptr) { + locale_name = ___lc_locale_name_func()[LC_COLLATE]; + codepage = ___lc_collate_cp_func(); + } else { + locale_name = ploc->_LocaleName; + codepage = ploc->_Page; + } + + if (locale_name == nullptr && codepage == CP_ACP) { + if (n2 <= n1) { + memcpy(string1, string2, n2); + } + retval = n2; + } else { + // Inquire size of dst string in BYTES + const int dstlen = __crtLCMapStringA(locale_name, + LCMAP_SORTKEY | LINGUISTIC_IGNORECASE | LINGUISTIC_IGNOREDIACRITIC | NORM_IGNOREKANATYPE | NORM_IGNOREWIDTH, + string2, static_cast(n2), nullptr, 0, codepage, TRUE); + + if (dstlen != 0) { + retval = dstlen; + + // if not enough room, return amount needed + if (dstlen <= static_cast(n1)) { + // Map src string to dst string + __crtLCMapStringA(locale_name, + LCMAP_SORTKEY | LINGUISTIC_IGNORECASE | LINGUISTIC_IGNOREDIACRITIC | NORM_IGNOREKANATYPE + | NORM_IGNOREWIDTH, + string2, static_cast(n2), string1, static_cast(n1), codepage, TRUE); + } + } + } + + return retval; +} + +// derived from xwcsxfrm.cpp +size_t __stdcall __std_regex_transform_primary_wchar_t( + _Out_writes_(end1 - string1) _Post_readable_size_(return) wchar_t* string1, wchar_t* end1, + _In_reads_(end2 - string2) const wchar_t* string2, const wchar_t* end2, _In_opt_ const _Collvec* ploc) noexcept { + size_t n1 = end1 - string1; + size_t n2 = end2 - string2; + size_t size = static_cast(-1); + const wchar_t* locale_name; + + if (ploc == nullptr) { + locale_name = ___lc_locale_name_func()[LC_COLLATE]; + } else { + locale_name = ploc->_LocaleName; + } + + if (locale_name == nullptr) { + if (n2 <= n1) { + memcpy(string1, string2, n2 * sizeof(wchar_t)); + } + size = n2; + } else { + // When using LCMAP_SORTKEY, LCMapStringW handles BYTES not wide + // chars. We use a byte buffer to hold bytes and then convert the + // byte string to a wide char string and return this so it can be + // compared using wcscmp(). User's buffer is n1 wide chars, so + // use an internal buffer of n1 bytes. + + auto bbuffer = _malloc_crt_t(unsigned char, n1); + + if (bbuffer) { +#pragma warning(push) +#pragma warning(disable : 6386) // PREfast doesn't understand LCMAP_SORTKEY + size = __crtLCMapStringW(locale_name, + LCMAP_SORTKEY | LINGUISTIC_IGNORECASE | LINGUISTIC_IGNOREDIACRITIC | NORM_IGNOREKANATYPE + | NORM_IGNOREWIDTH, + string2, static_cast(n2), reinterpret_cast(bbuffer.get()), static_cast(n1)); +#pragma warning(pop) + + if (size == 0) { + // buffer not big enough, get size required. + size = __crtLCMapStringW(locale_name, + LCMAP_SORTKEY | LINGUISTIC_IGNORECASE | LINGUISTIC_IGNOREDIACRITIC | NORM_IGNOREKANATYPE + | NORM_IGNOREWIDTH, + string2, static_cast(n2), nullptr, 0); + + if (size == 0) { + size = INT_MAX; // default error + } + } else { + // string successfully mapped, convert to wide char + + for (size_t i = 0; i < size; ++i) { + string1[i] = static_cast(bbuffer.get()[i]); + } + } + } + } + + return size; +} +} // extern "C" diff --git a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp index c7c017c9778..b37b0221d39 100644 --- a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp +++ b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp @@ -535,36 +535,100 @@ void test_gh_994() { gh_994_should_throw("[a-[.cs.]]", error_range); gh_994_should_throw("[[.cs.]-[.dzs.]]", error_range); -#ifndef SKIP_COLLATE_TESTS g_regexTester.should_throw("[[=a=]-c]", error_range); g_regexTester.should_throw("[c-[=z=]]", error_range); g_regexTester.should_throw("[[=a=]-[=z=]]", error_range); g_regexTester.should_match("a", "[[=a=]]"); - g_regexTester.should_match("A", "[[=a=]]"); + g_regexTester.should_not_match("A", "[[=a=]]"); g_regexTester.should_not_match("b", "[[=a=]]"); g_regexTester.should_not_match("B", "[[=a=]]"); - g_regexTester.should_match("z", "[[=Z=]]"); + g_regexTester.should_not_match("z", "[[=Z=]]"); g_regexTester.should_match("Z", "[[=Z=]]"); g_regexTester.should_not_match("b", "[[=Z=]]"); g_regexTester.should_not_match("B", "[[=Z=]]"); + g_regexTester.should_match("a", "[[=a=]]", icase); + g_regexTester.should_match("A", "[[=a=]]", icase); + g_regexTester.should_not_match("b", "[[=a=]]", icase); + g_regexTester.should_not_match("B", "[[=a=]]", icase); + g_regexTester.should_match("z", "[[=Z=]]", icase); + g_regexTester.should_match("Z", "[[=Z=]]", icase); + g_regexTester.should_not_match("b", "[[=Z=]]", icase); + g_regexTester.should_not_match("B", "[[=Z=]]", icase); + g_regexTester.should_match("ab", "[[=a=]]b"); - g_regexTester.should_match("Ab", "[[=a=]]b"); + g_regexTester.should_not_match("Ab", "[[=a=]]b"); g_regexTester.should_not_match("Ab", "[[=a=]]B"); g_regexTester.should_not_match("b", "[[=a=]]b"); g_regexTester.should_not_match("aab", "[[=a=]]b"); g_regexTester.should_not_match("B", "[[=a=]]b"); - - g_regexTester.should_match("AaAaaAaab", "[[=a=]]*b"); + g_regexTester.should_not_match("ab", "[[=A=]]b"); + g_regexTester.should_match("Ab", "[[=A=]]b"); + g_regexTester.should_not_match("Ab", "[[=A=]]B"); + g_regexTester.should_not_match("b", "[[=A=]]b"); + g_regexTester.should_not_match("AAb", "[[=A=]]b"); + g_regexTester.should_not_match("B", "[[=A=]]b"); + + g_regexTester.should_match("ab", "[[=a=]]b", icase); + g_regexTester.should_match("Ab", "[[=a=]]b", icase); + g_regexTester.should_match("Ab", "[[=a=]]B", icase); + g_regexTester.should_not_match("b", "[[=a=]]b", icase); + g_regexTester.should_not_match("aab", "[[=a=]]b", icase); + g_regexTester.should_not_match("B", "[[=a=]]b", icase); + g_regexTester.should_match("ab", "[[=A=]]b", icase); + g_regexTester.should_match("Ab", "[[=A=]]b", icase); + g_regexTester.should_match("Ab", "[[=A=]]B", icase); + g_regexTester.should_not_match("b", "[[=A=]]b", icase); + g_regexTester.should_not_match("AAb", "[[=A=]]b", icase); + g_regexTester.should_not_match("B", "[[=A=]]b", icase); + + g_regexTester.should_not_match("AaAaaAaab", "[[=a=]]*b"); g_regexTester.should_not_match("AaAaaAaab", "[[=a=]]*c"); - g_regexTester.should_match("AaAabcaAaad", "[[=a=]bc]*d"); -#endif // !defined(SKIP_COLLATE_TESTS) + g_regexTester.should_not_match("AaAabcaAaad", "[[=a=]bc]*d"); + g_regexTester.should_match("AaAaaAaab", "[[=a=]]*b", icase); + g_regexTester.should_not_match("AaAaaAaab", "[[=a=]]*c", icase); + g_regexTester.should_match("AaAabcaAaad", "[[=a=]bc]*d", icase); +} + +void test_gh_5435() { + // GH-5345: : Equivalence classes have unexpected behavior with std::wregex + { + test_wregex_locale eq_a_regex(&g_regexTester, L"^[[=a=]]*b$", "en-US"); + eq_a_regex.should_search_match(L"A\u00c0ab", L"A\u00c0ab"); // U+00C0 LATIN CAPITAL LETTER A WITH GRAVE + eq_a_regex.should_search_fail(L"Ab\u00c0ab"); // U+00C0 LATIN CAPITAL LETTER A WITH GRAVE + eq_a_regex.should_search_match(L"A\u00e0ab", L"A\u00e0ab"); // U+00E0 LATIN SMALL LETTER A WITH GRAVE + eq_a_regex.should_search_match(L"A\u00c1ab", L"A\u00c1ab"); // U+00C1 LATIN CAPITAL LETTER A WITH ACUTE + eq_a_regex.should_search_match(L"A\u00e1ab", L"A\u00e1ab"); // U+00E1 LATIN SMALL LETTER A WITH ACUTE + eq_a_regex.should_search_match(L"A\u00c2ab", L"A\u00c2ab"); // U+00C2 LATIN CAPITAL LETTER A WITH CIRCUMFLEX + eq_a_regex.should_search_match(L"A\u00e2ab", L"A\u00e2ab"); // U+00E2 LATIN SMALL LETTER A WITH CIRCUMFLEX + eq_a_regex.should_search_match(L"A\u00c3ab", L"A\u00c3ab"); // U+00C3 LATIN CAPITAL LETTER A WITH TILDE + eq_a_regex.should_search_match(L"A\u00e3ab", L"A\u00e3ab"); // U+00E3 LATIN SMALL LETTER A WITH TILDE + eq_a_regex.should_search_match(L"A\u00c4ab", L"A\u00c4ab"); // U+00C4 LATIN CAPITAL LETTER A WITH DIAERESIS + eq_a_regex.should_search_match(L"A\u00e4ab", L"A\u00e4ab"); // U+00E4 LATIN SMALL LETTER A WITH DIAERESIS + eq_a_regex.should_search_match(L"A\u00c5ab", L"A\u00c5ab"); // U+00C5 LATIN CAPITAL LETTER A WITH RING ABOVE + eq_a_regex.should_search_match(L"A\u00e5ab", L"A\u00e5ab"); // U+00E5 LATIN SMALL LETTER A WITH RING ABOVE + } + { + test_wregex_locale eq_e_regex(&g_regexTester, L"^[[=e=]]*b$", "en-US"); + eq_e_regex.should_search_match(L"e\u00c8Eb", L"e\u00c8Eb"); // U+00C8 LATIN CAPITAL LETTER E WITH GRAVE + eq_e_regex.should_search_fail(L"eb\u00c8Eb"); // U+00C8 LATIN CAPITAL LETTER E WITH GRAVE + eq_e_regex.should_search_match(L"e\u00e8Eb", L"e\u00e8Eb"); // U+00E8 LATIN SMALL LETTER E WITH GRAVE + eq_e_regex.should_search_match(L"e\u00c9Eb", L"e\u00c9Eb"); // U+00C9 LATIN CAPITAL LETTER E WITH ACUTE + eq_e_regex.should_search_match(L"e\u00e9Eb", L"e\u00e9Eb"); // U+00E9 LATIN SMALL LETTER E WITH ACUTE + eq_e_regex.should_search_match(L"e\u00caEb", L"e\u00caEb"); // U+00CA LATIN CAPITAL LETTER E WITH CIRCUMFLEX + eq_e_regex.should_search_match(L"e\u00eaEb", L"e\u00eaEb"); // U+00EA LATIN SMALL LETTER A WITH CIRCUMFLEX + eq_e_regex.should_search_match(L"e\u00cbEb", L"e\u00cbEb"); // U+00CB LATIN CAPITAL LETTER E WITH DIAERESIS + eq_e_regex.should_search_match(L"e\u00ebEb", L"e\u00ebEb"); // U+00EB LATIN SMALL LETTER A WITH DIAERESIS + eq_e_regex.should_search_fail(L"e\u00ccEb"); // U+00CC LATIN CAPITAL LETTER I WITH GRAVE + eq_e_regex.should_search_fail(L"e\u00ecEb"); // U+00EC LATIN SMALL LETTER I WITH GRAVE + } } int main() { test_collating_ranges_german(); test_gh_994(); + test_gh_5435(); return g_regexTester.result(); } diff --git a/tests/tr1/tests/regex1/test.cpp b/tests/tr1/tests/regex1/test.cpp index a9f4689b176..37d24c61fd8 100644 --- a/tests/tr1/tests/regex1/test.cpp +++ b/tests/tr1/tests/regex1/test.cpp @@ -174,7 +174,7 @@ static void test_traits() { // test template regex_traits CHECK(v0.transform(carr, carr + xlen(carr)) != v0.transform(carr0, carr0 + xlen(carr0))); CHECK(v0.transform(carr, carr + xlen(carr)) < v0.transform(carr1, carr1 + xlen(carr1))); CHECK(v0.transform_primary(carr, carr + xlen(carr)) == v0.transform_primary(carr, carr + xlen(carr))); - CHECK(v0.transform_primary(carr, carr + xlen(carr)) == v0.transform_primary(carr0, carr0 + xlen(carr0))); + CHECK(v0.transform_primary(carr, carr + xlen(carr)) != v0.transform_primary(carr0, carr0 + xlen(carr0))); CHECK(v0.transform_primary(carr0, carr0 + xlen(carr0)) < v0.transform_primary(carr1, carr1 + xlen(carr1))); for (size_t i = 0; i < sizeof(class_names) / sizeof(*class_names); ++i) { diff --git a/tests/tr1/tests/regex2/test.cpp b/tests/tr1/tests/regex2/test.cpp index 44cb5632126..994b33b2717 100644 --- a/tests/tr1/tests/regex2/test.cpp +++ b/tests/tr1/tests/regex2/test.cpp @@ -588,7 +588,7 @@ static const regex_test tests[] = { {__LINE__, T("[[:xdigit:]]"), T("g"), "0", ALL}, {__LINE__, T("[[:xdigit:]]"), T("1"), "1 0 1", ALL}, {__LINE__, T("[[:xdigit:]]"), T(" "), "0", ALL}, - {__LINE__, T("[[=x=]]"), T("X"), "1 0 1", ALL}, + {__LINE__, T("[[=x=]]"), T("X"), "0", ALL}, {__LINE__, T("[[=x=]]"), T("x"), "1 0 1", ALL}, // character class ranges From c74e118efb2302ffd9813db8a33c58db428476d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 26 Apr 2025 21:26:07 +0200 Subject: [PATCH 02/13] fix unused variable warnings when RTTI is disabled --- stl/inc/regex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/stl/inc/regex b/stl/inc/regex index 271909ad739..7ba37eb5bcb 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -361,7 +361,10 @@ public: _Res.resize(_Count); } } -#endif // defined(_CPPRTTI) +#else // ^^^ defined(_CPPRTTI) / !defined(_CPPRTTI) vvv + (void) _First; + (void) _Last; +#endif // !defined(_CPPRTTI) return _Res; } From 413b3060909a62ee47d5a78df032ca4861d10a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 26 Apr 2025 21:26:34 +0200 Subject: [PATCH 03/13] add scary comments --- stl/inc/yvals.h | 3 ++- stl/src/awint.hpp | 5 +++++ stl/src/regex.cpp | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/stl/inc/yvals.h b/stl/inc/yvals.h index c8cfa973615..88392ae8fb4 100644 --- a/stl/inc/yvals.h +++ b/stl/inc/yvals.h @@ -3,7 +3,8 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// This header is used to compile the import library (via locale0_implib.cpp => locale0.cpp => xfacet => yvals.h). +// This header is used to compile the import library +// (via locale0_implib.cpp => locale0.cpp => xfacet => yvals.h and regex.cpp => awint.hpp => yvals.h). // MAJOR LIMITATIONS apply to what can be included here! // Before editing this file, read: /docs/import_library.md diff --git a/stl/src/awint.hpp b/stl/src/awint.hpp index da1e26e470a..e5d753c93ba 100644 --- a/stl/src/awint.hpp +++ b/stl/src/awint.hpp @@ -2,6 +2,11 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // Internal definitions for A&W Win32 wrapper routines. + +// This file is compiled into the import library (via regex.cpp => awint.hpp). +// MAJOR LIMITATIONS apply to what can be included here! +// Before editing this file, read: /docs/import_library.md + #pragma once #include diff --git a/stl/src/regex.cpp b/stl/src/regex.cpp index 860776f1eb1..414787ce800 100644 --- a/stl/src/regex.cpp +++ b/stl/src/regex.cpp @@ -1,6 +1,10 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// This file is compiled into the import library. +// MAJOR LIMITATIONS apply to what can be included here! +// Before editing this file, read: /docs/import_library.md + #include <__msvc_xlocinfo_types.hpp> #include #include From 4697464b23d873c8d4b2d82eaf43605e6556aabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Thu, 1 May 2025 13:40:20 +0200 Subject: [PATCH 04/13] update expected_results.txt --- tests/libcxx/expected_results.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/libcxx/expected_results.txt b/tests/libcxx/expected_results.txt index 64b389eaf67..c0fff7a9161 100644 --- a/tests/libcxx/expected_results.txt +++ b/tests/libcxx/expected_results.txt @@ -830,7 +830,6 @@ std/re/re.alg/re.alg.search/basic.pass.cpp FAIL std/re/re.alg/re.alg.search/ecma.pass.cpp FAIL std/re/re.alg/re.alg.search/extended.pass.cpp FAIL std/re/re.traits/lookup_collatename.pass.cpp FAIL -std/re/re.traits/transform_primary.pass.cpp FAIL # Not analyzed, likely STL bugs. Various assertions. std/numerics/complex.number/complex.ops/complex_divide_complex.pass.cpp FAIL From 4549c6bdbe8eb69bc3b62e50f6e75d7a68cf8946 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 14:45:57 -0700 Subject: [PATCH 05/13] Add regex.cpp to the MSBuild machinery. --- stl/msbuild/stl_base/stl.files.settings.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/stl/msbuild/stl_base/stl.files.settings.targets b/stl/msbuild/stl_base/stl.files.settings.targets index 9b7db156955..0669a4c0a59 100644 --- a/stl/msbuild/stl_base/stl.files.settings.targets +++ b/stl/msbuild/stl_base/stl.files.settings.targets @@ -158,6 +158,7 @@ SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception $(CrtRoot)\github\stl\src\locale0_implib.cpp; $(CrtRoot)\github\stl\src\nothrow.cpp; $(CrtRoot)\github\stl\src\print.cpp; + $(CrtRoot)\github\stl\src\regex.cpp; $(CrtRoot)\github\stl\src\sharedmutex.cpp; $(CrtRoot)\github\stl\src\stacktrace.cpp; $(CrtRoot)\github\stl\src\syserror_import_lib.cpp; From 5c750f1c867d68309c0e41c9dcbc935fad3cb23c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 14:48:37 -0700 Subject: [PATCH 06/13] Extended friendship is magic. --- stl/inc/locale | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/locale b/stl/inc/locale index 52893426dbc..7e75f4d7634 100644 --- a/stl/inc/locale +++ b/stl/inc/locale @@ -193,7 +193,7 @@ protected: private: _Locinfo::_Collvec _Coll; // used by _LStrcoll and _XStrxfrm - friend class _Regex_traits<_Elem>; + friend _Regex_traits<_Elem>; }; #ifdef __clang__ From e5d9137788c368b887f9642353752d169c7b37ab Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 15:11:30 -0700 Subject: [PATCH 07/13] Comment cleanups. --- stl/inc/regex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 7ba37eb5bcb..dc9d60bf2b1 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -337,9 +337,9 @@ public: if (_First != _Last) { const collate<_Elem>* _Coll = _Getcoll(); const auto& _Coll_type = typeid(*_Coll); - // TRANSITION, ABI: GH-5394: locale creates collate objects of type collate, not collate_byname + // TRANSITION, ABI: GH-5394: locale creates collate objects of type collate, not collate_byname. // Depending on the resolution of LWG-2338, comparison to typeid(collate) might also become - // required by the standard + // required by the standard. if (_Coll_type == typeid(collate_byname<_Elem>) || _Coll_type == typeid(collate<_Elem>)) { // non-empty string with known collate facet, transform it size_t _Count; @@ -364,7 +364,7 @@ public: #else // ^^^ defined(_CPPRTTI) / !defined(_CPPRTTI) vvv (void) _First; (void) _Last; -#endif // !defined(_CPPRTTI) +#endif // ^^^ !defined(_CPPRTTI) ^^^ return _Res; } From cf5cbec9eae32c35784cfd1b54d5b5fb86aaa288 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 15:17:05 -0700 Subject: [PATCH 08/13] Avoid uninitialized `_Count`, transform `for` to `while`. --- stl/inc/regex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index dc9d60bf2b1..8e8bc1dd76f 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -342,12 +342,12 @@ public: // required by the standard. if (_Coll_type == typeid(collate_byname<_Elem>) || _Coll_type == typeid(collate<_Elem>)) { // non-empty string with known collate facet, transform it - size_t _Count; const string_type _Src(_First, _Last); const auto _Src_first = _Src.data(); const auto _Src_last = _Src_first + _Src.size(); - for (_Count = _Src.size(); _Res.size() < _Count;) { + size_t _Count = _Src.size(); + while (_Res.size() < _Count) { _Res.resize(_Count); _Count = _STD _Regex_transform_primary( &_Res[0], &_Res[0] + _Count, _Src_first, _Src_last, &_Coll->_Coll); From a303ef261291fa261d9c6cf1c969422a237d7b8d Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 15:23:35 -0700 Subject: [PATCH 09/13] Guard entrypoints to separately compiled code with `#ifdef _CPPRTTI`. --- stl/inc/regex | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stl/inc/regex b/stl/inc/regex index 8e8bc1dd76f..d38c3d7c8ca 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -53,6 +53,7 @@ _STL_DISABLE_CLANG_WARNINGS #endif // ^^^ !defined(_DEBUG) ^^^ #endif // !defined(_ENHANCED_REGEX_VISUALIZER) +#ifdef _CPPRTTI extern "C" { _STD size_t __stdcall __std_regex_transform_primary_char( _Out_writes_(_Last1 - _First1) _Post_readable_size_(return) char* _First1, char* _Last1, @@ -61,6 +62,7 @@ _STD size_t __stdcall __std_regex_transform_primary_wchar_t( _Out_writes_(_Last1 - _First1) _Post_readable_size_(return) wchar_t* _First1, wchar_t* _Last1, _In_reads_(_Last2 - _First2) const wchar_t* _First2, const wchar_t* _Last2, _In_opt_ const _Collvec*) noexcept; } // extern "C" +#endif // ^^^ defined(_CPPRTTI) ^^^ _STD_BEGIN @@ -276,6 +278,7 @@ struct _Regex_traits_base { // base of all regular expression traits using char_class_type = ctype_base::mask; }; +#ifdef _CPPRTTI inline size_t _Regex_transform_primary(_Out_writes_(_Last1 - _First1) _Post_readable_size_(return) char* _First1, char* _Last1, _In_reads_(_Last2 - _First2) const char* _First2, const char* _Last2, _In_opt_ const _Locinfo::_Collvec* _Vector) noexcept { @@ -287,6 +290,7 @@ inline size_t _Regex_transform_primary(_Out_writes_(_Last1 - _First1) _Post_read _In_opt_ const _Locinfo::_Collvec* _Vector) noexcept { return __std_regex_transform_primary_wchar_t(_First1, _Last1, _First2, _Last2, _Vector); } +#endif // ^^^ defined(_CPPRTTI) ^^^ template class _Regex_traits : public _Regex_traits_base { // base class for regular expression traits From 510562e9a65c90a2ffba867a305e4bf9c984349c Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 16:08:37 -0700 Subject: [PATCH 10/13] Apply GH 5431's fix: `INT_MAX` => `static_cast(-1)` --- stl/src/regex.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/src/regex.cpp b/stl/src/regex.cpp index 414787ce800..6ecd7744cd7 100644 --- a/stl/src/regex.cpp +++ b/stl/src/regex.cpp @@ -111,7 +111,7 @@ size_t __stdcall __std_regex_transform_primary_wchar_t( string2, static_cast(n2), nullptr, 0); if (size == 0) { - size = INT_MAX; // default error + size = static_cast(-1); // default error } } else { // string successfully mapped, convert to wide char From 2d4d244bcbf946e6daed1bbfe1619301b2503ceb Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 16:19:16 -0700 Subject: [PATCH 11/13] Fix digit transposition. --- tests/std/tests/GH_005204_regex_collating_ranges/test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp index b37b0221d39..c1e67187f50 100644 --- a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp +++ b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp @@ -592,7 +592,7 @@ void test_gh_994() { } void test_gh_5435() { - // GH-5345: : Equivalence classes have unexpected behavior with std::wregex + // GH-5435: : Equivalence classes have unexpected behavior with std::wregex { test_wregex_locale eq_a_regex(&g_regexTester, L"^[[=a=]]*b$", "en-US"); eq_a_regex.should_search_match(L"A\u00c0ab", L"A\u00c0ab"); // U+00C0 LATIN CAPITAL LETTER A WITH GRAVE From a66c773137aef6c4a522a0a5ca9db877fadb3ba3 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Wed, 7 May 2025 16:29:30 -0700 Subject: [PATCH 12/13] Fix Unicode comments. --- tests/std/tests/GH_005204_regex_collating_ranges/test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp index c1e67187f50..3d9986bb252 100644 --- a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp +++ b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp @@ -617,9 +617,9 @@ void test_gh_5435() { eq_e_regex.should_search_match(L"e\u00c9Eb", L"e\u00c9Eb"); // U+00C9 LATIN CAPITAL LETTER E WITH ACUTE eq_e_regex.should_search_match(L"e\u00e9Eb", L"e\u00e9Eb"); // U+00E9 LATIN SMALL LETTER E WITH ACUTE eq_e_regex.should_search_match(L"e\u00caEb", L"e\u00caEb"); // U+00CA LATIN CAPITAL LETTER E WITH CIRCUMFLEX - eq_e_regex.should_search_match(L"e\u00eaEb", L"e\u00eaEb"); // U+00EA LATIN SMALL LETTER A WITH CIRCUMFLEX + eq_e_regex.should_search_match(L"e\u00eaEb", L"e\u00eaEb"); // U+00EA LATIN SMALL LETTER E WITH CIRCUMFLEX eq_e_regex.should_search_match(L"e\u00cbEb", L"e\u00cbEb"); // U+00CB LATIN CAPITAL LETTER E WITH DIAERESIS - eq_e_regex.should_search_match(L"e\u00ebEb", L"e\u00ebEb"); // U+00EB LATIN SMALL LETTER A WITH DIAERESIS + eq_e_regex.should_search_match(L"e\u00ebEb", L"e\u00ebEb"); // U+00EB LATIN SMALL LETTER E WITH DIAERESIS eq_e_regex.should_search_fail(L"e\u00ccEb"); // U+00CC LATIN CAPITAL LETTER I WITH GRAVE eq_e_regex.should_search_fail(L"e\u00ecEb"); // U+00EC LATIN SMALL LETTER I WITH GRAVE } From 7b0ceedfc3249cfc25d8cd0e7d6f82e4d35d5456 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Fri, 9 May 2025 12:16:33 -0700 Subject: [PATCH 13/13] Fix /clr:pure. --- stl/inc/regex | 14 +++++++------- .../GH_005204_regex_collating_ranges/test.cpp | 4 ++++ tests/std/tests/VSO_0000000_regex_use/test.cpp | 2 ++ tests/tr1/tests/regex1/test.cpp | 2 ++ tests/tr1/tests/regex2/test.cpp | 2 ++ 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index d38c3d7c8ca..189666153f5 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -53,7 +53,7 @@ _STL_DISABLE_CLANG_WARNINGS #endif // ^^^ !defined(_DEBUG) ^^^ #endif // !defined(_ENHANCED_REGEX_VISUALIZER) -#ifdef _CPPRTTI +#if defined(_CPPRTTI) && !defined(_M_CEE_PURE) extern "C" { _STD size_t __stdcall __std_regex_transform_primary_char( _Out_writes_(_Last1 - _First1) _Post_readable_size_(return) char* _First1, char* _Last1, @@ -62,7 +62,7 @@ _STD size_t __stdcall __std_regex_transform_primary_wchar_t( _Out_writes_(_Last1 - _First1) _Post_readable_size_(return) wchar_t* _First1, wchar_t* _Last1, _In_reads_(_Last2 - _First2) const wchar_t* _First2, const wchar_t* _Last2, _In_opt_ const _Collvec*) noexcept; } // extern "C" -#endif // ^^^ defined(_CPPRTTI) ^^^ +#endif // ^^^ defined(_CPPRTTI) && !defined(_M_CEE_PURE) ^^^ _STD_BEGIN @@ -278,7 +278,7 @@ struct _Regex_traits_base { // base of all regular expression traits using char_class_type = ctype_base::mask; }; -#ifdef _CPPRTTI +#if defined(_CPPRTTI) && !defined(_M_CEE_PURE) inline size_t _Regex_transform_primary(_Out_writes_(_Last1 - _First1) _Post_readable_size_(return) char* _First1, char* _Last1, _In_reads_(_Last2 - _First2) const char* _First2, const char* _Last2, _In_opt_ const _Locinfo::_Collvec* _Vector) noexcept { @@ -290,7 +290,7 @@ inline size_t _Regex_transform_primary(_Out_writes_(_Last1 - _First1) _Post_read _In_opt_ const _Locinfo::_Collvec* _Vector) noexcept { return __std_regex_transform_primary_wchar_t(_First1, _Last1, _First2, _Last2, _Vector); } -#endif // ^^^ defined(_CPPRTTI) ^^^ +#endif // ^^^ defined(_CPPRTTI) && !defined(_M_CEE_PURE) ^^^ template class _Regex_traits : public _Regex_traits_base { // base class for regular expression traits @@ -337,7 +337,7 @@ public: string_type transform_primary(_FwdIt _First, _FwdIt _Last) const { // apply locale-specific case-insensitive transformation string_type _Res; -#ifdef _CPPRTTI +#if defined(_CPPRTTI) && !defined(_M_CEE_PURE) if (_First != _Last) { const collate<_Elem>* _Coll = _Getcoll(); const auto& _Coll_type = typeid(*_Coll); @@ -365,10 +365,10 @@ public: _Res.resize(_Count); } } -#else // ^^^ defined(_CPPRTTI) / !defined(_CPPRTTI) vvv +#else // ^^^ defined(_CPPRTTI) && !defined(_M_CEE_PURE) / !defined(_CPPRTTI) || defined(_M_CEE_PURE) vvv (void) _First; (void) _Last; -#endif // ^^^ !defined(_CPPRTTI) ^^^ +#endif // ^^^ !defined(_CPPRTTI) || defined(_M_CEE_PURE) ^^^ return _Res; } diff --git a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp index 3d9986bb252..352ed835e1c 100644 --- a/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp +++ b/tests/std/tests/GH_005204_regex_collating_ranges/test.cpp @@ -535,6 +535,7 @@ void test_gh_994() { gh_994_should_throw("[a-[.cs.]]", error_range); gh_994_should_throw("[[.cs.]-[.dzs.]]", error_range); +#ifndef _M_CEE_PURE g_regexTester.should_throw("[[=a=]-c]", error_range); g_regexTester.should_throw("[c-[=z=]]", error_range); g_regexTester.should_throw("[[=a=]-[=z=]]", error_range); @@ -589,10 +590,12 @@ void test_gh_994() { g_regexTester.should_match("AaAaaAaab", "[[=a=]]*b", icase); g_regexTester.should_not_match("AaAaaAaab", "[[=a=]]*c", icase); g_regexTester.should_match("AaAabcaAaad", "[[=a=]bc]*d", icase); +#endif // ^^^ !defined(_M_CEE_PURE) ^^^ } void test_gh_5435() { // GH-5435: : Equivalence classes have unexpected behavior with std::wregex +#ifndef _M_CEE_PURE { test_wregex_locale eq_a_regex(&g_regexTester, L"^[[=a=]]*b$", "en-US"); eq_a_regex.should_search_match(L"A\u00c0ab", L"A\u00c0ab"); // U+00C0 LATIN CAPITAL LETTER A WITH GRAVE @@ -623,6 +626,7 @@ void test_gh_5435() { eq_e_regex.should_search_fail(L"e\u00ccEb"); // U+00CC LATIN CAPITAL LETTER I WITH GRAVE eq_e_regex.should_search_fail(L"e\u00ecEb"); // U+00EC LATIN SMALL LETTER I WITH GRAVE } +#endif // ^^^ !defined(_M_CEE_PURE) ^^^ } int main() { diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index 4f7f3b1e855..266d6c0988b 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -764,9 +764,11 @@ void test_gh_4995() { g_regexTester.should_throw("[[:digit:]-e]", error_range); g_regexTester.should_throw("[e-[:digit:]]", error_range); g_regexTester.should_throw("[[:alpha:]-[:digit:]]", error_range); +#ifndef _M_CEE_PURE g_regexTester.should_throw("[[=a=]-e]", error_range, ECMAScript | regex::collate); g_regexTester.should_throw("[e-[=a=]]", error_range, ECMAScript | regex::collate); g_regexTester.should_throw("[[=a=]-[=b=]]", error_range, ECMAScript | regex::collate); +#endif // ^^^ !defined(_M_CEE_PURE) ^^^ // Test valid cases: g_regexTester.should_not_match("b", R"([\d-])"); diff --git a/tests/tr1/tests/regex1/test.cpp b/tests/tr1/tests/regex1/test.cpp index 37d24c61fd8..2aaf0ea2e7e 100644 --- a/tests/tr1/tests/regex1/test.cpp +++ b/tests/tr1/tests/regex1/test.cpp @@ -173,9 +173,11 @@ static void test_traits() { // test template regex_traits CHECKSTRING(v0.transform(carr, carr + xlen(carr)), v0.transform(carr, carr + xlen(carr))); CHECK(v0.transform(carr, carr + xlen(carr)) != v0.transform(carr0, carr0 + xlen(carr0))); CHECK(v0.transform(carr, carr + xlen(carr)) < v0.transform(carr1, carr1 + xlen(carr1))); +#ifndef _M_CEE_PURE CHECK(v0.transform_primary(carr, carr + xlen(carr)) == v0.transform_primary(carr, carr + xlen(carr))); CHECK(v0.transform_primary(carr, carr + xlen(carr)) != v0.transform_primary(carr0, carr0 + xlen(carr0))); CHECK(v0.transform_primary(carr0, carr0 + xlen(carr0)) < v0.transform_primary(carr1, carr1 + xlen(carr1))); +#endif // ^^^ !defined(_M_CEE_PURE) ^^^ for (size_t i = 0; i < sizeof(class_names) / sizeof(*class_names); ++i) { CHECK(v0.lookup_classname(class_names[i], class_names[i] + xlen(class_names[i])) != 0); diff --git a/tests/tr1/tests/regex2/test.cpp b/tests/tr1/tests/regex2/test.cpp index 994b33b2717..d52371192de 100644 --- a/tests/tr1/tests/regex2/test.cpp +++ b/tests/tr1/tests/regex2/test.cpp @@ -588,8 +588,10 @@ static const regex_test tests[] = { {__LINE__, T("[[:xdigit:]]"), T("g"), "0", ALL}, {__LINE__, T("[[:xdigit:]]"), T("1"), "1 0 1", ALL}, {__LINE__, T("[[:xdigit:]]"), T(" "), "0", ALL}, +#ifndef _M_CEE_PURE {__LINE__, T("[[=x=]]"), T("X"), "0", ALL}, {__LINE__, T("[[=x=]]"), T("x"), "1 0 1", ALL}, +#endif // ^^^ !defined(_M_CEE_PURE) ^^^ // character class ranges {__LINE__, T("[-]"), T("-"), "1 0 1", ALL},