diff --git a/stl/inc/regex b/stl/inc/regex index 5e622565c3d..dab2e1fb321 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -228,34 +228,54 @@ struct _Lex_compare_memcmp_classify_pred<_Elem, _Elem, _Std_char_traits_lt<_Elem : _Lex_compare_memcmp_classify_pred_for_std_char_traits_lt<_Elem> {}; template -struct _Cmp_cs { // functor to compare two character values for equality +struct _Cmp_icase { // functor to compare for equality following case-insensitive translation of both characters using _Elem = typename _RxTraits::char_type; - _STATIC_CALL_OPERATOR bool operator()(_Elem _Ex1, _Elem _Ex2) _CONST_CALL_OPERATOR { - return _Ex1 == _Ex2; + + explicit _Cmp_icase(const _RxTraits& _Tr) noexcept : _Traits(_Tr) {} + + bool operator()(_Elem _Ex1, _Elem _Ex2) const { + return _Traits.translate_nocase(_Ex1) == _Traits.translate_nocase(_Ex2); } + + const _RxTraits& _Traits; }; template -struct _Cmp_icase { // functor to compare for case-insensitive equality +struct _Cmp_collate { // functor to compare for equality following collating translation of both characters using _Elem = typename _RxTraits::char_type; - explicit _Cmp_icase(const _RxTraits& _Tr) noexcept : _Traits(_Tr) {} + explicit _Cmp_collate(const _RxTraits& _Tr) noexcept : _Traits(_Tr) {} bool operator()(_Elem _Ex1, _Elem _Ex2) const { - return _Traits.translate_nocase(_Ex1) == _Traits.translate_nocase(_Ex2); + return _Traits.translate(_Ex1) == _Traits.translate(_Ex2); } const _RxTraits& _Traits; }; template -struct _Cmp_collate { // functor to compare for locale-specific equality +struct _Cmp_icase_translateleft { + // functor to compare for equality following collating translation of the left character using _Elem = typename _RxTraits::char_type; - explicit _Cmp_collate(const _RxTraits& _Tr) noexcept : _Traits(_Tr) {} + explicit _Cmp_icase_translateleft(const _RxTraits& _Tr) noexcept : _Traits(_Tr) {} bool operator()(_Elem _Ex1, _Elem _Ex2) const { - return _Traits.translate(_Ex1) == _Traits.translate(_Ex2); + return _Traits.translate_nocase(_Ex1) == _Ex2; + } + + const _RxTraits& _Traits; +}; + +template +struct _Cmp_collate_translateleft { + // functor to compare for equality following collating translation of the left character + using _Elem = typename _RxTraits::char_type; + + explicit _Cmp_collate_translateleft(const _RxTraits& _Tr) noexcept : _Traits(_Tr) {} + + bool operator()(_Elem _Ex1, _Elem _Ex2) const { + return _Traits.translate(_Ex1) == _Ex2; } const _RxTraits& _Traits; @@ -2933,10 +2953,6 @@ void _Builder2<_FwdIt, _Elem, _RxTraits>::_Add_class() { // add bracket expressi template void _Builder2<_FwdIt, _Elem, _RxTraits>::_Add_char_to_bitmap(_Elem _Ch) { // add character to accelerator table - if (_Flags & regex_constants::icase) { - _Ch = _Traits.translate_nocase(_Ch); - } - _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Current); if (!_Node->_Small) { @@ -2948,10 +2964,6 @@ void _Builder2<_FwdIt, _Elem, _RxTraits>::_Add_char_to_bitmap(_Elem _Ch) { // ad template void _Builder2<_FwdIt, _Elem, _RxTraits>::_Add_char_to_array(_Elem _Ch) { // append character to character array - if (_Flags & regex_constants::icase) { - _Ch = _Traits.translate_nocase(_Ch); - } - _Node_class<_Elem, _RxTraits>* _Node = static_cast<_Node_class<_Elem, _RxTraits>*>(_Current); if (!_Node->_Large) { _Node->_Large = new _Buf<_Elem>; @@ -2962,6 +2974,12 @@ void _Builder2<_FwdIt, _Elem, _RxTraits>::_Add_char_to_array(_Elem _Ch) { // app template void _Builder2<_FwdIt, _Elem, _RxTraits>::_Add_char_to_class(_Elem _Ch) { // add character to bracket expression + if (_Flags & regex_constants::icase) { + _Ch = _Traits.translate_nocase(_Ch); + } else if (_Flags & regex_constants::collate) { + _Ch = _Traits.translate(_Ch); + } + if (static_cast(_Ch) < _Bmp_max) { _Add_char_to_bitmap(_Ch); } else { @@ -3607,23 +3625,33 @@ _BidIt1 _Cmp_chrange(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _E return _Res; } } - return _Begin2 == _End2 ? _Begin1 : _Res; } template -_BidIt1 _Compare(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, const _RxTraits& _Traits, - regex_constants::syntax_option_type _Sflags) { // compare character ranges - _BidIt1 _Res = _End1; +_BidIt1 _Compare_translate_both(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, + const _RxTraits& _Traits, regex_constants::syntax_option_type _Sflags) { + // compare character ranges, translating characters in both ranges according to syntax options if (_Sflags & regex_constants::icase) { - _Res = _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_icase<_RxTraits>{_Traits}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_icase<_RxTraits>{_Traits}); } else if (_Sflags & regex_constants::collate) { - _Res = _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_collate<_RxTraits>{_Traits}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_collate<_RxTraits>{_Traits}); } else { - _Res = _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_cs<_RxTraits>{}); + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, equal_to{}); } +} - return _Res; +template +_BidIt1 _Compare_translate_left(_BidIt1 _Begin1, _BidIt1 _End1, _BidIt2 _Begin2, _BidIt2 _End2, + const _RxTraits& _Traits, regex_constants::syntax_option_type _Sflags) { + // compare character ranges, translating characters in the left range according to syntax options + if (_Sflags & regex_constants::icase) { + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_icase_translateleft<_RxTraits>{_Traits}); + } else if (_Sflags & regex_constants::collate) { + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, _Cmp_collate_translateleft<_RxTraits>{_Traits}); + } else { + return _STD _Cmp_chrange(_Begin1, _End1, _Begin2, _End2, equal_to{}); + } } template @@ -3896,7 +3924,7 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N { // check for string match _Node_str<_Elem>* _Node = static_cast<_Node_str<_Elem>*>(_Nx); _It _Res0; - if ((_Res0 = _Compare(_Tgt_state._Cur, _End, _Node->_Data._Str(), + if ((_Res0 = _STD _Compare_translate_left(_Tgt_state._Cur, _End, _Node->_Data._Str(), _Node->_Data._Str() + _Node->_Data._Size(), _Traits, _Sflags)) != _Tgt_state._Cur) { _Tgt_state._Cur = _Res0; @@ -3971,7 +3999,8 @@ bool _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _It _Bx = _Tgt_state._Grps[_Node->_Idx]._Begin; _It _Ex = _Tgt_state._Grps[_Node->_Idx]._End; if (_Bx != _Ex // _Bx == _Ex for zero-length match - && (_Res0 = _Compare(_Tgt_state._Cur, _End, _Bx, _Ex, _Traits, _Sflags)) == _Tgt_state._Cur) { + && (_Res0 = _STD _Compare_translate_both(_Tgt_state._Cur, _End, _Bx, _Ex, _Traits, _Sflags)) + == _Tgt_state._Cur) { _Failed = true; } else { _Tgt_state._Cur = _Res0; @@ -4084,7 +4113,8 @@ _BidIt _Matcher2<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Skip(_BidIt _First_arg _Node_str<_Elem>* _Node = static_cast<_Node_str<_Elem>*>(_Nx); for (; _First_arg != _Last; ++_First_arg) { // look for starting match _BidIt _Next = _First_arg; - if (_Compare(_First_arg, ++_Next, _Node->_Data._Str(), _Node->_Data._Str() + 1, _Traits, _Sflags) + if (_STD _Compare_translate_left( + _First_arg, ++_Next, _Node->_Data._Str(), _Node->_Data._Str() + 1, _Traits, _Sflags) != _First_arg) { break; } diff --git a/tests/std/test.lst b/tests/std/test.lst index 4c7c40603d1..ea6fe9911cb 100644 --- a/tests/std/test.lst +++ b/tests/std/test.lst @@ -265,6 +265,7 @@ tests\GH_005315_destructor_tombstones tests\GH_005402_string_with_volatile_range tests\GH_005421_vector_algorithms_integer_class_type_iterator tests\GH_005472_do_not_overlap +tests\GH_005553_regex_character_translation tests\LWG2381_num_get_floating_point tests\LWG2597_complex_branch_cut tests\LWG3018_shared_ptr_function diff --git a/tests/std/tests/GH_005553_regex_character_translation/env.lst b/tests/std/tests/GH_005553_regex_character_translation/env.lst new file mode 100644 index 00000000000..19f025bd0e6 --- /dev/null +++ b/tests/std/tests/GH_005553_regex_character_translation/env.lst @@ -0,0 +1,4 @@ +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +RUNALL_INCLUDE ..\usual_matrix.lst diff --git a/tests/std/tests/GH_005553_regex_character_translation/test.cpp b/tests/std/tests/GH_005553_regex_character_translation/test.cpp new file mode 100644 index 00000000000..33c5521ad49 --- /dev/null +++ b/tests/std/tests/GH_005553_regex_character_translation/test.cpp @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include +#include +#include + +#include + +using namespace std; +using namespace std::regex_constants; + +regex_fixture g_regexTester; + +template +class nonidempotent_translate_regex_traits : private regex_traits { +private: + using rx_traits = regex_traits; + +public: + using char_type = typename rx_traits::char_type; + using string_type = typename rx_traits::string_type; + using locale_type = typename rx_traits::locale_type; + using char_class_type = typename rx_traits::char_class_type; + using uchar_type = make_unsigned_t; + + // TRANSITION, GH-995 + using _Uelem = typename rx_traits::_Uelem; + + nonidempotent_translate_regex_traits() = default; + + using rx_traits::length; + + charT translate(const charT c) const { + return static_cast(static_cast(rx_traits::translate(c)) / 2U); + } + + + charT translate_nocase(const charT c) const { + return static_cast(static_cast(rx_traits::translate_nocase(c)) / 2U); + } + + using rx_traits::getloc; + using rx_traits::imbue; + using rx_traits::isctype; + using rx_traits::lookup_classname; + using rx_traits::lookup_collatename; + using rx_traits::transform; + using rx_traits::transform_primary; + using rx_traits::value; +}; + +template +void check_match(const wstring& subject, const wstring& pattern, const Rx& re, match_flag_type flags = match_default, + bool matches = true) { + if (regex_match(subject, re, flags) != matches) { + wprintf(LR"(Expected regex_match("%s", regex("%s", 0x%X)) to be %s.)", subject.c_str(), pattern.c_str(), + static_cast(re.flags()), matches ? L"true" : L"false"); + g_regexTester.fail_regex(); + } +} + +template +void check_no_match( + const wstring& subject, const wstring& pattern, const Rx& re, match_flag_type flags = match_default) { + check_match(subject, pattern, re, flags, false); +} + +template +void check_search_match(const wstring& subject, const wstring& expected, const wstring& pattern, const Rx& re, + match_flag_type flags = match_default) { + wsmatch match; + const bool search_result = regex_search(subject, match, re, flags); + if (!search_result || match[0] != expected) { + wprintf(LR"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to find "%s", )", subject.c_str(), + pattern.c_str(), static_cast(re.flags()), static_cast(flags), expected.c_str()); + if (search_result) { + wprintf(LR"(but it matched "%s")" + L"\n", + match.str().c_str()); + } else { + puts("but it failed to match"); + } + g_regexTester.fail_regex(); + } +} + +template +void check_search_fail( + const wstring& subject, const wstring& pattern, const Rx& re, match_flag_type flags = match_default) { + wsmatch match; + const bool search_result = regex_search(subject, match, re, flags); + if (search_result) { + wprintf(LR"(Expected regex_search("%s", regex("%s", 0x%X), 0x%X) to not match, but it found "%s")" + L"\n", + subject.c_str(), pattern.c_str(), static_cast(re.flags()), static_cast(flags), + match.str().c_str()); + g_regexTester.fail_regex(); + } +} + +void test_gh_5553() { + // GH-5553 ``: Correct character translation in `icase` and `collate` mode + { + wstring pattern = L"g"; + basic_regex> charcompare_icase_pattern{pattern, icase}; + check_match(L"f", pattern, charcompare_icase_pattern); + check_match(L"F", pattern, charcompare_icase_pattern); + check_match(L"g", pattern, charcompare_icase_pattern); + check_match(L"G", pattern, charcompare_icase_pattern); + check_no_match(L"e", pattern, charcompare_icase_pattern); + check_no_match(L"E", pattern, charcompare_icase_pattern); + check_no_match(L"h", pattern, charcompare_icase_pattern); + check_no_match(L"H", pattern, charcompare_icase_pattern); + + check_search_match(L"abcdefghijklmnopqrstuvwxyz", L"f", pattern, charcompare_icase_pattern); + check_search_match(L"ABCDEFGHIJKLMNOPQRSTUVWXYZ", L"F", pattern, charcompare_icase_pattern); + check_search_match(L"zyxwvutsrqponmlkjihgfedcba", L"g", pattern, charcompare_icase_pattern); + check_search_match(L"ZYXWVUTSRQPONMLKJIHGFEDCBA", L"G", pattern, charcompare_icase_pattern); + check_search_fail(L"zyxwvutsrqponmlkjihedcba", pattern, charcompare_icase_pattern); + check_search_fail(L"ABCDEHIJKLMNOPQRSTUVWXYZ", pattern, charcompare_icase_pattern); + } + + { + wstring pattern = L"g"; + basic_regex> charcompare_collate_pattern{ + pattern, regex_constants::collate}; + check_match(L"f", pattern, charcompare_collate_pattern); + check_no_match(L"F", pattern, charcompare_collate_pattern); + check_match(L"g", pattern, charcompare_collate_pattern); + check_no_match(L"G", pattern, charcompare_collate_pattern); + check_no_match(L"e", pattern, charcompare_collate_pattern); + check_no_match(L"E", pattern, charcompare_collate_pattern); + check_no_match(L"h", pattern, charcompare_collate_pattern); + check_no_match(L"H", pattern, charcompare_collate_pattern); + + check_search_match(L"abcdefghijklmnopqrstuvwxyz", L"f", pattern, charcompare_collate_pattern); + check_search_fail(L"ABCDEFGHIJKLMNOPQRSTUVWXYZ", pattern, charcompare_collate_pattern); + check_search_match(L"zyxwvutsrqponmlkjihgfedcba", L"g", pattern, charcompare_collate_pattern); + check_search_fail(L"ZYXWVUTSRQPONMLKJIHGFEDCBA", pattern, charcompare_collate_pattern); + check_search_fail(L"zyxwvutsrqponmlkjihedcba", pattern, charcompare_collate_pattern); + } + + { + wstring pattern = L"[g]"; + basic_regex> charclasscompare_icase_pattern{ + pattern, icase}; + check_match(L"f", pattern, charclasscompare_icase_pattern); + check_match(L"F", pattern, charclasscompare_icase_pattern); + check_match(L"g", pattern, charclasscompare_icase_pattern); + check_match(L"G", pattern, charclasscompare_icase_pattern); + check_no_match(L"e", pattern, charclasscompare_icase_pattern); + check_no_match(L"E", pattern, charclasscompare_icase_pattern); + check_no_match(L"h", pattern, charclasscompare_icase_pattern); + check_no_match(L"H", pattern, charclasscompare_icase_pattern); + } + + { + wstring pattern = L"[g]"; + basic_regex> charclasscompare_collate_pattern{ + pattern, regex_constants::collate}; + check_match(L"f", pattern, charclasscompare_collate_pattern); + check_no_match(L"F", pattern, charclasscompare_collate_pattern); + check_match(L"g", pattern, charclasscompare_collate_pattern); + check_no_match(L"G", pattern, charclasscompare_collate_pattern); + check_no_match(L"e", pattern, charclasscompare_collate_pattern); + check_no_match(L"E", pattern, charclasscompare_collate_pattern); + check_no_match(L"h", pattern, charclasscompare_collate_pattern); + check_no_match(L"H", pattern, charclasscompare_collate_pattern); + } + + { + wstring pattern = L"[g-i]"; + basic_regex> charrangecompare_icase_pattern{ + pattern, icase}; + check_match(L"f", pattern, charrangecompare_icase_pattern); + check_match(L"F", pattern, charrangecompare_icase_pattern); + check_match(L"g", pattern, charrangecompare_icase_pattern); + check_match(L"G", pattern, charrangecompare_icase_pattern); + check_match(L"i", pattern, charrangecompare_icase_pattern); + check_match(L"I", pattern, charrangecompare_icase_pattern); + check_no_match(L"e", pattern, charrangecompare_icase_pattern); + check_no_match(L"E", pattern, charrangecompare_icase_pattern); + check_no_match(L"j", pattern, charrangecompare_icase_pattern); + check_no_match(L"J", pattern, charrangecompare_icase_pattern); + } + + { + wstring pattern = L"[g-i]"; + basic_regex> charrangecompare_collate_pattern{ + pattern, regex_constants::collate}; + check_match(L"f", pattern, charrangecompare_collate_pattern); + check_no_match(L"F", pattern, charrangecompare_collate_pattern); + check_match(L"g", pattern, charrangecompare_collate_pattern); + check_no_match(L"G", pattern, charrangecompare_collate_pattern); + check_match(L"i", pattern, charrangecompare_collate_pattern); + check_no_match(L"I", pattern, charrangecompare_collate_pattern); + check_no_match(L"e", pattern, charrangecompare_collate_pattern); + check_no_match(L"E", pattern, charrangecompare_collate_pattern); + check_no_match(L"j", pattern, charrangecompare_collate_pattern); + check_no_match(L"J", pattern, charrangecompare_collate_pattern); + } + + { + wstring pattern = L"[\u022d-\u022f]"; // U+022D LATIN SMALL LETTER O WITH TILDE AND MACRON + // U+022F LATIN SMALL LETTER O WITH DOT ABOVE + + basic_regex> small_unicode_charrange_pattern{ + pattern, regex_constants::icase}; + // U+022C LATIN CAPITAL LETTER O WITH TILDE AND MACRON + check_match(L"\u022c", pattern, small_unicode_charrange_pattern); + // U+022D LATIN SMALL LETTER O WITH TILDE AND MACRON + check_match(L"\u022d", pattern, small_unicode_charrange_pattern); + // U+022E LATIN CAPITAL LETTER O WITH DOT ABOVE + check_match(L"\u022e", pattern, small_unicode_charrange_pattern); + // U+022F LATIN SMALL LETTER O WITH DOT ABOVE + check_match(L"\u022f", pattern, small_unicode_charrange_pattern); + // U+022B LATIN SMALL LETTER O WITH DIAERESIS AND MACRON + check_no_match(L"\u022b", pattern, small_unicode_charrange_pattern); + // U+0230 LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + check_no_match(L"\u0230", pattern, small_unicode_charrange_pattern); + } + + { + wstring pattern = L"[\u022b-\u0230]"; // U+022B LATIN SMALL LETTER O WITH DIAERESIS AND MACRON + // U+0230 LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + + basic_regex> big_unicode_charrange_pattern{ + pattern, regex_constants::icase}; + // U+022A LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON + check_match(L"\u022a", pattern, big_unicode_charrange_pattern); + // U+022B LATIN SMALL LETTER O WITH DIAERESIS AND MACRON + check_match(L"\u022b", pattern, big_unicode_charrange_pattern); + // U+0230 LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON + check_match(L"\u0230", pattern, big_unicode_charrange_pattern); + // U+0231 LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON + check_match(L"\u0231", pattern, big_unicode_charrange_pattern); + // U+0229 LATIN SMALL LETTER E WITH CEDILLA + check_no_match(L"\u0229", pattern, big_unicode_charrange_pattern); + // U+0232 LATIN CAPITAL LETTER Y WITH MACRON + check_no_match(L"\u0232", pattern, big_unicode_charrange_pattern); + } +} + +int main() { + test_gh_5553(); + + return g_regexTester.result(); +}