From 36044b492a304a6c7c8c3ecb89916da797c0b180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Thu, 13 Nov 2025 21:12:18 +0100 Subject: [PATCH 1/8] ``: Remove capture extent vectors from stack frames --- benchmarks/CMakeLists.txt | 1 + benchmarks/src/regex_match.cpp | 35 ++++++++++++++++++++++++++++++++++ stl/inc/regex | 30 +++++++++++++++++++++++++---- 3 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 benchmarks/src/regex_match.cpp diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt index 9aa8a8c0450..c5aa8ba6773 100644 --- a/benchmarks/CMakeLists.txt +++ b/benchmarks/CMakeLists.txt @@ -125,6 +125,7 @@ add_benchmark(path_lexically_normal src/path_lexically_normal.cpp) add_benchmark(priority_queue_push_range src/priority_queue_push_range.cpp) add_benchmark(random_integer_generation src/random_integer_generation.cpp) add_benchmark(ranges_div_ceil src/ranges_div_ceil.cpp) +add_benchmark(regex_match src/regex_match.cpp) add_benchmark(regex_search src/regex_search.cpp) add_benchmark(remove src/remove.cpp) add_benchmark(replace src/replace.cpp) diff --git a/benchmarks/src/regex_match.cpp b/benchmarks/src/regex_match.cpp new file mode 100644 index 00000000000..4e5496e862a --- /dev/null +++ b/benchmarks/src/regex_match.cpp @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include +#include +#include + + +using namespace std; +using namespace regex_constants; + +void bm_match_sequence_of_as(benchmark::State& state, const char* pattern, syntax_option_type syntax = ECMAScript) { + string input(static_cast(state.range()), 'a'); + regex re{pattern, syntax}; + + for (auto _ : state) { + benchmark::DoNotOptimize(input); + const char* pos = input.data(); + const char* end = input.data() + input.size(); + cmatch match; + regex_match(pos, end, match, re); + } +} + +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "a*", "a*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "a*?", "a*?")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(?:a)*", "(?:a)*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)*", "(a)*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(?:b|a)*", "(?:b|a)*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(b|a)*", "(b|a)*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(?:b|a)*", "(a)(?:b|a)*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(b|a)*", "(a)(b|a)*")->Arg(100)->Arg(200)->Arg(400); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(?:b|a)*c", "(a)(?:b|a)*c")->Arg(100)->Arg(200)->Arg(400); + +BENCHMARK_MAIN(); diff --git a/stl/inc/regex b/stl/inc/regex index bb16a898136..98d8d3ba210 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1681,6 +1681,8 @@ enum class _Rx_unwind_ops { _Loop_nongreedy, _Loop_greedy, _Loop_restore_vals, + _Capture_restore_begin, + _Capture_restore_end }; template @@ -1689,7 +1691,7 @@ public: _Rx_unwind_ops _Code; int _Loop_idx_sav; _Node_base* _Node; - _Tgt_state_t<_BidIt> _Match_state; + _Bt_state_t<_BidIt> _Match_state; size_t _Loop_frame_idx_sav; }; @@ -3946,7 +3948,10 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N { // record current position _Node_capture* _Node = static_cast<_Node_capture*>(_Nx); if (_Node->_Idx != 0U) { - _Tgt_state._Grps[_Node->_Idx]._Begin = _Tgt_state._Cur; + auto& _Group = _Tgt_state._Grps[_Node->_Idx]; + auto _Frame_idx = _Push_frame(_Rx_unwind_ops::_Capture_restore_begin, _Node); + _Frames[_Frame_idx]._Match_state._Cur = _Group._Begin; + _Group._Begin = _Tgt_state._Cur; } break; } @@ -3956,8 +3961,11 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _Node_end_group* _Node = static_cast<_Node_end_group*>(_Nx); _Node_capture* _Node0 = static_cast<_Node_capture*>(_Node->_Back); if (_Node0->_Idx != 0U) { // update capture data - _Tgt_state._Grp_valid[_Node0->_Idx] = true; - _Tgt_state._Grps[_Node0->_Idx]._End = _Tgt_state._Cur; + auto& _Group = _Tgt_state._Grps[_Node0->_Idx]; + auto _Frame_idx = _Push_frame(_Rx_unwind_ops::_Capture_restore_end, _Node0); + _Frames[_Frame_idx]._Match_state._Cur = _Group._End; + _Tgt_state._Grp_valid[_Node0->_Idx] = true; + _Group._End = _Tgt_state._Cur; } break; } @@ -4325,6 +4333,20 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N } break; + case _Rx_unwind_ops::_Capture_restore_begin: + { // restore begin of capturing group + auto _Node = static_cast<_Node_capture*>(_Frame._Node); + _Tgt_state._Grps[_Node->_Idx]._Begin = _Frame._Match_state._Cur; + } + break; + + case _Rx_unwind_ops::_Capture_restore_end: + { // restore end of capturing group + auto _Node = static_cast<_Node_capture*>(_Frame._Node); + _Tgt_state._Grps[_Node->_Idx]._End = _Frame._Match_state._Cur; + } + break; + default: #if _ITERATOR_DEBUG_LEVEL != 0 _STL_REPORT_ERROR("internal stack of regex matcher corrupted"); From 3397e3b9388f1d97343da85fb22c06a5bbe28d54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julian=20M=C3=BCller?= Date: Sat, 15 Nov 2025 17:21:05 +0100 Subject: [PATCH 2/8] Fix capture group restoration for positive lookahead assertions --- stl/inc/regex | 51 ++++++++++++------- .../std/tests/VSO_0000000_regex_use/test.cpp | 10 ++++ 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index 98d8d3ba210..66f52bfe509 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -3921,25 +3921,40 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N } case _N_end_assert: - for (;;) { - --_Frames_count; - const auto& _Frame = _Frames[_Frames_count]; - const auto _Code = _Frame._Code; - if (_Code == _Rx_unwind_ops::_After_assert || _Code == _Rx_unwind_ops::_After_neg_assert) { - _Tgt_state._Cur = _Frame._Match_state._Cur; - _Decrease_stack_usage_count(); - if (_Code == _Rx_unwind_ops::_After_assert) { - _Next = _Frame._Node->_Next; - } else { - _Failed = true; + { + size_t _Last_capture_restore_frame = 0U; + for (;;) { + --_Frames_count; + const auto& _Frame = _Frames[_Frames_count]; + const auto _Code = _Frame._Code; + if (_Code == _Rx_unwind_ops::_After_assert || _Code == _Rx_unwind_ops::_After_neg_assert) { + _Tgt_state._Cur = _Frame._Match_state._Cur; + _Decrease_stack_usage_count(); + if (_Code == _Rx_unwind_ops::_After_assert) { + _Next = _Frame._Node->_Next; + if (_Last_capture_restore_frame != 0U) { + _Frames_count = static_cast( + _STD remove_if(_Frames.begin() + static_cast(_Frames_count), + _Frames.begin() + static_cast(_Last_capture_restore_frame) + 1, + [](const auto& _Other_frame) { + return _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_begin + && _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_end; + }) + - _Frames.begin()); + } + } else { + _Failed = true; + } + break; + } else if (_Code == _Rx_unwind_ops::_Disjunction_eval_alt_on_failure + || _Code == _Rx_unwind_ops::_Disjunction_eval_alt_always + || _Code == _Rx_unwind_ops::_Loop_greedy // + || _Code == _Rx_unwind_ops::_Loop_nongreedy + || _Code == _Rx_unwind_ops::_Loop_restore_vals) { + _Decrease_stack_usage_count(); + } else if (_Code == _Rx_unwind_ops::_Capture_restore_end && _Last_capture_restore_frame == 0U) { + _Last_capture_restore_frame = _Frames_count; } - break; - } else if (_Code == _Rx_unwind_ops::_Disjunction_eval_alt_on_failure - || _Code == _Rx_unwind_ops::_Disjunction_eval_alt_always - || _Code == _Rx_unwind_ops::_Loop_greedy // - || _Code == _Rx_unwind_ops::_Loop_nongreedy - || _Code == _Rx_unwind_ops::_Loop_restore_vals) { - _Decrease_stack_usage_count(); } } break; diff --git a/tests/std/tests/VSO_0000000_regex_use/test.cpp b/tests/std/tests/VSO_0000000_regex_use/test.cpp index 096507a3945..73fd12c692a 100644 --- a/tests/std/tests/VSO_0000000_regex_use/test.cpp +++ b/tests/std/tests/VSO_0000000_regex_use/test.cpp @@ -2350,6 +2350,15 @@ void test_gh_5798() { } } +void test_gh_5865() { + // GH-5865: : Remove capture extent vectors from stack frames + // These tests check correct restoration of capturing groups + // when backtracking over positive lookahead assertions that matched successfully. + g_regexTester.should_capture("ab", "(?:(?=(.*))ab)*", "ab"); + g_regexTester.should_capture("abcd", "(?:(?=(.*))ab)*cd", "abcd"); + g_regexTester.should_capture("abab", "(?:(?=(.*))ab)*ab", "abab"); +} + int main() { test_dev10_449367_case_insensitivity_should_work(); test_dev11_462743_regex_collate_should_not_disable_regex_icase(); @@ -2407,6 +2416,7 @@ int main() { test_gh_5792(); test_gh_5797(); test_gh_5798(); + test_gh_5865(); return g_regexTester.result(); } From 65a5124e7be9492ecfd33ec8e03195ed7d8b26ee Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 18 Nov 2025 11:25:30 -0800 Subject: [PATCH 3/8] Preserve trailing commas, --- stl/inc/regex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/regex b/stl/inc/regex index 66f52bfe509..ad308b2b763 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -1682,7 +1682,7 @@ enum class _Rx_unwind_ops { _Loop_greedy, _Loop_restore_vals, _Capture_restore_begin, - _Capture_restore_end + _Capture_restore_end, }; template From fa4cb4047eb61e9c699bbda0ad421501398578a2 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 18 Nov 2025 11:37:59 -0800 Subject: [PATCH 4/8] Use `_STATIC_LAMBDA`. --- stl/inc/regex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stl/inc/regex b/stl/inc/regex index ad308b2b763..f0a6ce94472 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -3936,7 +3936,7 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N _Frames_count = static_cast( _STD remove_if(_Frames.begin() + static_cast(_Frames_count), _Frames.begin() + static_cast(_Last_capture_restore_frame) + 1, - [](const auto& _Other_frame) { + [](const auto& _Other_frame) _STATIC_LAMBDA { return _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_begin && _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_end; }) From 7436282f3378c0952f08319871a597f7ee87bcc9 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 18 Nov 2025 11:49:40 -0800 Subject: [PATCH 5/8] Extract lambda, named `_Not_capture_restore`. --- stl/inc/regex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index f0a6ce94472..a3d9145849b 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -3933,13 +3933,14 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N if (_Code == _Rx_unwind_ops::_After_assert) { _Next = _Frame._Node->_Next; if (_Last_capture_restore_frame != 0U) { + auto _Not_capture_restore = [](const auto& _Other_frame) _STATIC_LAMBDA { + return _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_begin + && _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_end; + }; _Frames_count = static_cast( _STD remove_if(_Frames.begin() + static_cast(_Frames_count), _Frames.begin() + static_cast(_Last_capture_restore_frame) + 1, - [](const auto& _Other_frame) _STATIC_LAMBDA { - return _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_begin - && _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_end; - }) + _Not_capture_restore) - _Frames.begin()); } } else { From 2632dd1e791c5b7151c64232cdfa342ed67e86ac Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 18 Nov 2025 11:55:58 -0800 Subject: [PATCH 6/8] Extract `remove_if()`, store `_Effective_frames_end`. --- stl/inc/regex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stl/inc/regex b/stl/inc/regex index a3d9145849b..4725efa347e 100644 --- a/stl/inc/regex +++ b/stl/inc/regex @@ -3937,11 +3937,11 @@ bool _Matcher3<_BidIt, _Elem, _RxTraits, _It, _Alloc>::_Match_pat(_Node_base* _N return _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_begin && _Other_frame._Code != _Rx_unwind_ops::_Capture_restore_end; }; - _Frames_count = static_cast( + const auto _Effective_frames_end = _STD remove_if(_Frames.begin() + static_cast(_Frames_count), _Frames.begin() + static_cast(_Last_capture_restore_frame) + 1, - _Not_capture_restore) - - _Frames.begin()); + _Not_capture_restore); + _Frames_count = static_cast(_Effective_frames_end - _Frames.begin()); } } else { _Failed = true; From da8659e9f95580f859502b3281de22dd9d53caec Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 18 Nov 2025 12:14:24 -0800 Subject: [PATCH 7/8] Extract `common_args`. --- benchmarks/src/regex_match.cpp | 22 +++++++++++++--------- benchmarks/src/regex_search.cpp | 26 +++++++++++++++----------- 2 files changed, 28 insertions(+), 20 deletions(-) diff --git a/benchmarks/src/regex_match.cpp b/benchmarks/src/regex_match.cpp index 4e5496e862a..60db2b54780 100644 --- a/benchmarks/src/regex_match.cpp +++ b/benchmarks/src/regex_match.cpp @@ -22,14 +22,18 @@ void bm_match_sequence_of_as(benchmark::State& state, const char* pattern, synta } } -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "a*", "a*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "a*?", "a*?")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(?:a)*", "(?:a)*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)*", "(a)*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(?:b|a)*", "(?:b|a)*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(b|a)*", "(b|a)*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(?:b|a)*", "(a)(?:b|a)*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(b|a)*", "(a)(b|a)*")->Arg(100)->Arg(200)->Arg(400); -BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(?:b|a)*c", "(a)(?:b|a)*c")->Arg(100)->Arg(200)->Arg(400); +void common_args(auto bm) { + bm->Arg(100)->Arg(200)->Arg(400); +} + +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "a*", "a*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "a*?", "a*?")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(?:a)*", "(?:a)*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)*", "(a)*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(?:b|a)*", "(?:b|a)*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(b|a)*", "(b|a)*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(?:b|a)*", "(a)(?:b|a)*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(b|a)*", "(a)(b|a)*")->Apply(common_args); +BENCHMARK_CAPTURE(bm_match_sequence_of_as, "(a)(?:b|a)*c", "(a)(?:b|a)*c")->Apply(common_args); BENCHMARK_MAIN(); diff --git a/benchmarks/src/regex_search.cpp b/benchmarks/src/regex_search.cpp index 19018fc80f2..aeb46d88b26 100644 --- a/benchmarks/src/regex_search.cpp +++ b/benchmarks/src/regex_search.cpp @@ -32,16 +32,20 @@ void bm_lorem_search(benchmark::State& state, const char* pattern, syntax_option } } -BENCHMARK_CAPTURE(bm_lorem_search, "^bibe", "^bibe")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, "bibe", "bibe")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, "bibe".collate, "bibe", regex_constants::collate)->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, "(bibe)", "(bibe)")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, "(bibe)+", "(bibe)+")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, "(?:bibe)+", "(?:bibe)+")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, R"(\bbibe)", R"(\bbibe)")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, R"(\Bibe)", R"(\Bibe)")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, R"((?=....)bibe)", R"((?=....)bibe)")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, R"((?=bibe)....)", R"((?=bibe)....)")->Arg(2)->Arg(3)->Arg(4); -BENCHMARK_CAPTURE(bm_lorem_search, R"((?!lorem)bibe)", R"((?!lorem)bibe)")->Arg(2)->Arg(3)->Arg(4); +void common_args(auto bm) { + bm->Arg(2)->Arg(3)->Arg(4); +} + +BENCHMARK_CAPTURE(bm_lorem_search, "^bibe", "^bibe")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, "bibe", "bibe")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, "bibe".collate, "bibe", regex_constants::collate)->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, "(bibe)", "(bibe)")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, "(bibe)+", "(bibe)+")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, "(?:bibe)+", "(?:bibe)+")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, R"(\bbibe)", R"(\bbibe)")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, R"(\Bibe)", R"(\Bibe)")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, R"((?=....)bibe)", R"((?=....)bibe)")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, R"((?=bibe)....)", R"((?=bibe)....)")->Apply(common_args); +BENCHMARK_CAPTURE(bm_lorem_search, R"((?!lorem)bibe)", R"((?!lorem)bibe)")->Apply(common_args); BENCHMARK_MAIN(); From 5ecd5813bcc0da49d5b27a26831d6a8ac8d4b0b5 Mon Sep 17 00:00:00 2001 From: "Stephan T. Lavavej" Date: Tue, 18 Nov 2025 12:15:13 -0800 Subject: [PATCH 8/8] Include ``, drop extra newline. --- benchmarks/src/regex_match.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/src/regex_match.cpp b/benchmarks/src/regex_match.cpp index 60db2b54780..194bb0a8b0c 100644 --- a/benchmarks/src/regex_match.cpp +++ b/benchmarks/src/regex_match.cpp @@ -2,10 +2,10 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #include +#include #include #include - using namespace std; using namespace regex_constants;