diff --git a/stl/inc/xstring b/stl/inc/xstring index b5ab9511a73..234a533ad76 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2221,8 +2221,10 @@ public: : sizeof(value_type) <= 4 ? 3 : sizeof(value_type) <= 8 ? 1 : 0; + // capacity in SSO mode + static constexpr size_type _SMALL_STRING_CAPACITY = _BUF_SIZE - 1; - _CONSTEXPR20 value_type* _Myptr() noexcept { + _NODISCARD _CONSTEXPR20 value_type* _Myptr() noexcept { value_type* _Result = _Bx._Buf; if (_Large_string_engaged()) { _Result = _Unfancy(_Bx._Ptr); @@ -2231,7 +2233,7 @@ public: return _Result; } - _CONSTEXPR20 const value_type* _Myptr() const noexcept { + _NODISCARD _CONSTEXPR20 const value_type* _Myptr() const noexcept { const value_type* _Result = _Bx._Buf; if (_Large_string_engaged()) { _Result = _Unfancy(_Bx._Ptr); @@ -2240,11 +2242,13 @@ public: return _Result; } - _CONSTEXPR20 bool _Large_string_engaged() const noexcept { - return _BUF_SIZE <= _Myres; + _NODISCARD _CONSTEXPR20 bool _Large_string_engaged() const noexcept { + // Determines whether large mode is used for the string's capacity. + // Doesn't imply anything about the largeness of the string's size. + return _Myres > _SMALL_STRING_CAPACITY; } - constexpr void _Activate_SSO_buffer() noexcept { + _CONSTEXPR20 void _Activate_SSO_buffer() noexcept { // start the lifetime of the array elements #if _HAS_CXX20 if (_STD is_constant_evaluated()) { @@ -2273,7 +2277,7 @@ public: _Xout_of_range("invalid string position"); } - _CONSTEXPR20 size_type _Clamp_suffix_size(const size_type _Off, const size_type _Size) const noexcept { + _NODISCARD _CONSTEXPR20 size_type _Clamp_suffix_size(const size_type _Off, const size_type _Size) const noexcept { // trims _Size to the longest it can be assuming a string at/after _Off return (_STD min)(_Size, _Mysize - _Off); } @@ -2290,8 +2294,9 @@ public: }; _Bxty _Bx; - size_type _Mysize = 0; // current length of string - size_type _Myres = 0; // current storage reserved for string + // invariant: _Myres >= _Mysize, and _Myres >= _SMALL_STRING_CAPACITY + size_type _Mysize = 0; // current length of string (size) + size_type _Myres = _SMALL_STRING_CAPACITY; // current storage reserved for string (capacity) }; // get _Ty's size after being EBCO'd @@ -2361,8 +2366,18 @@ public: using const_reverse_iterator = _STD reverse_iterator; private: - static constexpr auto _BUF_SIZE = _Scary_val::_BUF_SIZE; - static constexpr auto _ALLOC_MASK = _Scary_val::_ALLOC_MASK; + static constexpr size_type _BUF_SIZE = _Scary_val::_BUF_SIZE; + static constexpr size_type _ALLOC_MASK = _Scary_val::_ALLOC_MASK; + static constexpr size_type _SMALL_STRING_CAPACITY = _Scary_val::_SMALL_STRING_CAPACITY; + + _NODISCARD static _CONSTEXPR20 bool _Entails_large_string(size_type _Size_or_capacity) noexcept { + // decides whether planned size/capacity entails large mode + return _Size_or_capacity > _SMALL_STRING_CAPACITY; + } + + // least real allocation size, including space reserved for terminating null + // used exclusively by _Move_assign_from_buffer and _Release_to_buffer + static constexpr size_type _LEAST_ALLOCATION_SIZE = _SMALL_STRING_CAPACITY + 1 + 1; // When doing _String_val operations by memcpy, we are touching: // _String_val::_Bx::_Buf (type is array of _Elem) @@ -2422,7 +2437,7 @@ private: } #endif // _HAS_CXX20 // Don't annotate small strings; only annotate on the heap. - if (_Capacity == _BUF_SIZE - 1 || !_Asan_string_should_annotate) { + if (!_Entails_large_string(_Capacity) || !_Asan_string_should_annotate) { return; } @@ -2540,10 +2555,6 @@ public: _Construct<_Construct_strategy::_From_ptr>(_Ptr, _Convert_size(_Traits::length(_Ptr))); } -#if _HAS_CXX23 - basic_string(nullptr_t) = delete; -#endif // _HAS_CXX23 - #if _HAS_CXX17 template ::value, int> = 0> #endif // _HAS_CXX17 @@ -2552,6 +2563,10 @@ public: _Construct<_Construct_strategy::_From_ptr>(_Ptr, _Convert_size(_Traits::length(_Ptr))); } +#if _HAS_CXX23 + basic_string(nullptr_t) = delete; +#endif // _HAS_CXX23 + _CONSTEXPR20 basic_string(_CRT_GUARDOVERFLOW const size_type _Count, const _Elem _Ch) : _Mypair(_Zero_then_variadic_args_t{}) { // construct from _Count * _Ch _Construct<_Construct_strategy::_From_char>(_Ch, _Count); @@ -2589,19 +2604,26 @@ public: } private: - static constexpr void _Start_element_lifetimes(_Elem* const _Ptr, const size_type _Size) { + _NODISCARD static _CONSTEXPR20 pointer _Allocate_for_capacity(_Alty& _Al, const size_type _Capacity) { + _STL_INTERNAL_CHECK(_Entails_large_string(_Capacity)); + const pointer _Fancy_ptr = _Al.allocate(_Capacity + 1); // +1 for null terminator +#if _HAS_CXX20 // Start element lifetimes to avoid UB. This is a more general mechanism than _String_val::_Activate_SSO_buffer, // but likely more impactful to throughput. -#if _HAS_CXX20 if (_STD is_constant_evaluated()) { - for (size_type _Idx = 0; _Idx < _Size; ++_Idx) { + _Elem* const _Ptr = _Unfancy(_Fancy_ptr); + for (size_type _Idx = 0; _Idx < _Capacity + 1; ++_Idx) { _STD construct_at(_Ptr + _Idx); } } -#else // ^^^ C++20-or-later / pre-C++20 vvv - (void) _Ptr; - (void) _Size; #endif // _HAS_CXX20 + return _Fancy_ptr; + } + + static _CONSTEXPR20 void _Deallocate_for_capacity( + _Alty& _Al, const pointer& _Old_ptr, const size_type _Capacity) noexcept { + _STL_INTERNAL_CHECK(_Entails_large_string(_Capacity)); + _Al.deallocate(_Old_ptr, _Capacity + 1); } enum class _Construct_strategy : uint8_t { _From_char, _From_ptr, _From_string }; @@ -2625,9 +2647,9 @@ private: auto&& _Alproxy = _GET_PROXY_ALLOCATOR(_Alty, _Al); _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); - if (_Count < _BUF_SIZE) { + if (!_Entails_large_string(_Count)) { _My_data._Mysize = _Count; - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _SMALL_STRING_CAPACITY; if constexpr (_Strat == _Construct_strategy::_From_char) { _Traits::assign(_My_data._Bx._Buf, _Count, _Arg); @@ -2647,13 +2669,11 @@ private: return; } - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _SMALL_STRING_CAPACITY; const size_type _New_capacity = _Calculate_growth(_Count); - const pointer _New_ptr = _Al.allocate(_New_capacity + 1); // throws + const pointer _New_ptr = _Allocate_for_capacity(_Al, _New_capacity); // throws _Construct_in_place(_My_data._Bx._Ptr, _New_ptr); - _Start_element_lifetimes(_Unfancy(_New_ptr), _New_capacity + 1); - _My_data._Mysize = _Count; _My_data._Myres = _New_capacity; if constexpr (_Strat == _Construct_strategy::_From_char) { @@ -2683,20 +2703,18 @@ private: _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); _My_data._Mysize = 0; - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _SMALL_STRING_CAPACITY; if constexpr (is_same_v<_Size, size_type>) { if (_Count > max_size()) { _Xlen_string(); // result too long } - if (_Count >= _BUF_SIZE) { + if (_Entails_large_string(_Count)) { const size_type _New_capacity = _Calculate_growth(_Count); - const pointer _New_ptr = _Al.allocate(_New_capacity + 1); // throws + const pointer _New_ptr = _Allocate_for_capacity(_Al, _New_capacity); // throws _Construct_in_place(_My_data._Bx._Ptr, _New_ptr); _My_data._Myres = _New_capacity; - - _Start_element_lifetimes(_Unfancy(_New_ptr), _New_capacity + 1); } } @@ -2710,12 +2728,11 @@ private: const auto _Old_ptr = _My_data._Myptr(); const size_type _New_capacity = _Calculate_growth(_My_data._Mysize); - const pointer _New_ptr = _Al.allocate(_New_capacity + 1); // throws + const pointer _New_ptr = _Allocate_for_capacity(_Al, _New_capacity); // throws - _Start_element_lifetimes(_Unfancy(_New_ptr), _New_capacity + 1); _Traits::copy(_Unfancy(_New_ptr), _Old_ptr, _My_data._Mysize); - if (_My_data._Myres >= _BUF_SIZE) { // Need to deallocate old storage - _Al.deallocate(_My_data._Bx._Ptr, _My_data._Myres + 1); + if (_My_data._Large_string_engaged()) { // Need to deallocate old storage + _Deallocate_for_capacity(_Al, _My_data._Bx._Ptr, _My_data._Myres); _My_data._Bx._Ptr = _New_ptr; } else { _Construct_in_place(_My_data._Bx._Ptr, _New_ptr); @@ -2785,19 +2802,17 @@ public: _STL_INTERNAL_CHECK(_Right_size <= max_size()); _STL_INTERNAL_CHECK(_Right_size <= max_size() - _Left_size); const auto _New_size = static_cast(_Left_size + _Right_size); - size_type _New_capacity = _BUF_SIZE - 1; + size_type _New_capacity = _SMALL_STRING_CAPACITY; auto& _My_data = _Mypair._Myval2; _Elem* _Ptr = _My_data._Bx._Buf; auto&& _Alproxy = _GET_PROXY_ALLOCATOR(_Alty, _Getal()); _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); // throws if (_New_capacity < _New_size) { - _New_capacity = _Calculate_growth(_New_size, _BUF_SIZE - 1, max_size()); - const pointer _Fancyptr = _Getal().allocate(_New_capacity + 1); // throws + _New_capacity = _Calculate_growth(_New_size, _SMALL_STRING_CAPACITY, max_size()); + const pointer _Fancyptr = _Allocate_for_capacity(_Getal(), _New_capacity); // throws _Ptr = _Unfancy(_Fancyptr); _Construct_in_place(_My_data._Bx._Ptr, _Fancyptr); - - _Start_element_lifetimes(_Ptr, _New_capacity + 1); } _My_data._Mysize = _New_size; @@ -2863,12 +2878,11 @@ public: _Xlen_string(); } - const auto _New_capacity = _Calculate_growth(_New_size, _BUF_SIZE - 1, _Max); + const auto _New_capacity = _Calculate_growth(_New_size, _SMALL_STRING_CAPACITY, _Max); auto&& _Alproxy = _GET_PROXY_ALLOCATOR(_Alty, _Getal()); _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); // throws - const pointer _Fancyptr = _Getal().allocate(_New_capacity + 1); // throws + const pointer _Fancyptr = _Allocate_for_capacity(_Getal(), _New_capacity); // throws // nothrow hereafter - _Start_element_lifetimes(_Unfancy(_Fancyptr), _New_capacity + 1); _Construct_in_place(_My_data._Bx._Ptr, _Fancyptr); _My_data._Mysize = _New_size; _My_data._Myres = _New_capacity; @@ -2905,14 +2919,15 @@ public: _Tidy_init(); } - _NODISCARD bool _Move_assign_from_buffer(_Elem* const _Right, const size_type _Size, const size_type _Res) { + _NODISCARD bool _Move_assign_from_buffer( + _Elem* const _Right, const size_type _Size, const size_type _Res /* real allocation size */) { // Move assign from a buffer, used exclusively by basic_stringbuf; returns _Large_string_engaged() auto& _My_data = _Mypair._Myval2; _STL_INTERNAL_CHECK(!_My_data._Large_string_engaged() && _My_data._Mysize == 0); _STL_INTERNAL_CHECK(_Size < _Res); // So there is room for null terminator _Traits::assign(_Right[_Size], _Elem()); - const bool _Is_large = _Res > _BUF_SIZE; // Note: _BUF_SIZE because _Res now includes the null terminator + const bool _Is_large = _Res >= _LEAST_ALLOCATION_SIZE; if (_Is_large) { _ASAN_STRING_REMOVE(*this); _Construct_in_place(_My_data._Bx._Ptr, _Refancy(_Right)); @@ -2922,7 +2937,7 @@ public: } else { _Traits::copy(_My_data._Bx._Buf, _Right, _Res); _My_data._Mysize = _Size; - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _SMALL_STRING_CAPACITY; } return _Is_large; @@ -2932,7 +2947,7 @@ public: struct _Released_buffer { pointer _Ptr; size_type _Size; - size_type _Res; + size_type _Res; // real allocation size }; _NODISCARD _Released_buffer _Release_to_buffer(_Alloc& _Al) { @@ -2945,10 +2960,10 @@ public: _Result._Ptr = _My_data._Bx._Ptr; _Result._Res = _My_data._Myres + 1; } else { - // use _BUF_SIZE + 1 to avoid SSO, if the buffer is assigned back - _Result._Ptr = _Al.allocate(_BUF_SIZE + 1); + // use _LEAST_ALLOCATION_SIZE to avoid SSO, if the buffer is assigned back + _Result._Ptr = _Al.allocate(_LEAST_ALLOCATION_SIZE); _Traits::copy(_Unfancy(_Result._Ptr), _My_data._Bx._Buf, _BUF_SIZE); - _Result._Res = _BUF_SIZE + 1; + _Result._Res = _LEAST_ALLOCATION_SIZE; } _My_data._Orphan_all(); _Tidy_init(); @@ -3054,7 +3069,7 @@ private: const auto _Result_size = _Right_data._Clamp_suffix_size(_Roff, _Size_max); const auto _Right_ptr = _Right_data._Myptr(); auto& _Al = _Getal(); - if (_Allocators_equal(_Al, _Right._Getal()) && _Result_size >= _BUF_SIZE) { + if (_Allocators_equal(_Al, _Right._Getal()) && _Entails_large_string(_Result_size)) { if (_Roff != 0) { _Traits::move(_Right_ptr, _Right_ptr + _Roff, _Result_size); } @@ -3127,30 +3142,6 @@ public: static constexpr auto npos{static_cast(-1)}; -private: - _CONSTEXPR20 void _Copy_assign_val_from_small(const basic_string& _Right) { - // TRANSITION, VSO-761321; inline into only caller when that's fixed - _Tidy_deallocate(); -#if !defined(_INSERT_STRING_ANNOTATION) - if constexpr (_Can_memcpy_val) { -#if _HAS_CXX20 - if (!_STD is_constant_evaluated()) -#endif // _HAS_CXX20 - { - _Memcpy_val_from(_Right); - return; - } - } -#endif // !defined(_INSERT_STRING_ANNOTATION) - - auto& _My_data = _Mypair._Myval2; - auto& _Right_data = _Right._Mypair._Myval2; - - _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); - _My_data._Mysize = _Right_data._Mysize; - _My_data._Myres = _Right_data._Myres; - } - public: _CONSTEXPR20 basic_string& operator=(const basic_string& _Right) { if (this == _STD addressof(_Right)) { @@ -3165,21 +3156,24 @@ public: auto&& _Right_alproxy = _GET_PROXY_ALLOCATOR(_Alty, _Right_al); _Container_proxy_ptr<_Alty> _New_proxy(_Right_alproxy, _Leave_proxy_unbound{}); // throws - if (_Right._Mypair._Myval2._Large_string_engaged()) { - const auto _New_size = _Right._Mypair._Myval2._Mysize; - const auto _New_capacity = _Calculate_growth(_New_size, 0, _Right.max_size()); + const auto _Right_size = _Right._Mypair._Myval2._Mysize; + const _Elem* const _Right_ptr = _Right._Mypair._Myval2._Myptr(); + if (_Entails_large_string(_Right_size)) { + const auto _New_capacity = + _Calculate_growth(_Right_size, _SMALL_STRING_CAPACITY, _Right.max_size()); auto _Right_al_non_const = _Right_al; - const auto _New_ptr = _Right_al_non_const.allocate(_New_capacity + 1); // throws - - _Start_element_lifetimes(_Unfancy(_New_ptr), _New_capacity + 1); + const auto _New_ptr = _Allocate_for_capacity(_Right_al_non_const, _New_capacity); // throws - _Traits::copy(_Unfancy(_New_ptr), _Unfancy(_Right._Mypair._Myval2._Bx._Ptr), _New_size + 1); + _Traits::copy(_Unfancy(_New_ptr), _Right_ptr, _Right_size + 1); _Tidy_deallocate(); _Mypair._Myval2._Bx._Ptr = _New_ptr; - _Mypair._Myval2._Mysize = _New_size; + _Mypair._Myval2._Mysize = _Right_size; _Mypair._Myval2._Myres = _New_capacity; } else { - _Copy_assign_val_from_small(_Right); + _Tidy_deallocate(); + _Traits::copy(_Mypair._Myval2._Bx._Buf, _Right_ptr, _Right_size + 1); + _Mypair._Myval2._Mysize = _Right_size; + _Mypair._Myval2._Myres = _SMALL_STRING_CAPACITY; } _Pocca(_Al, _Right_al); @@ -3988,7 +3982,7 @@ public: return; } - if (_My_data._Mysize < _BUF_SIZE) { + if (!_Entails_large_string(_My_data._Mysize)) { _Become_small(); return; } @@ -3996,14 +3990,12 @@ public: const size_type _Target_capacity = (_STD min)(_My_data._Mysize | _ALLOC_MASK, max_size()); if (_Target_capacity < _My_data._Myres) { // worth shrinking, do it auto& _Al = _Getal(); - const pointer _New_ptr = _Al.allocate(_Target_capacity + 1); // throws + const pointer _New_ptr = _Allocate_for_capacity(_Al, _Target_capacity); // throws _ASAN_STRING_REMOVE(*this); - _Start_element_lifetimes(_Unfancy(_New_ptr), _Target_capacity + 1); - _My_data._Orphan_all(); _Traits::copy(_Unfancy(_New_ptr), _Unfancy(_My_data._Bx._Ptr), _My_data._Mysize + 1); - _Al.deallocate(_My_data._Bx._Ptr, _My_data._Myres + 1); + _Deallocate_for_capacity(_Al, _My_data._Bx._Ptr, _My_data._Myres); _My_data._Bx._Ptr = _New_ptr; _My_data._Myres = _Target_capacity; _ASAN_STRING_CREATE(*this); @@ -4214,7 +4206,7 @@ public: return; } - if (_BUF_SIZE > _Newcap && _Mypair._Myval2._Large_string_engaged()) { + if (!_Entails_large_string(_Newcap) && _Mypair._Myval2._Large_string_engaged()) { // deallocate everything; switch back to "small" mode _Become_small(); return; @@ -4738,16 +4730,15 @@ private: const size_type _Old_capacity = _Mypair._Myval2._Myres; const size_type _New_capacity = _Calculate_growth(_New_size); auto& _Al = _Getal(); - const pointer _New_ptr = _Al.allocate(_New_capacity + 1); // throws + const pointer _New_ptr = _Allocate_for_capacity(_Al, _New_capacity); // throws - _Start_element_lifetimes(_Unfancy(_New_ptr), _New_capacity + 1); _Mypair._Myval2._Orphan_all(); _ASAN_STRING_REMOVE(*this); _Mypair._Myval2._Mysize = _New_size; _Mypair._Myval2._Myres = _New_capacity; _Fn(_Unfancy(_New_ptr), _New_size, _Args...); - if (_BUF_SIZE <= _Old_capacity) { - _Al.deallocate(_Mypair._Myval2._Bx._Ptr, _Old_capacity + 1); + if (_Entails_large_string(_Old_capacity)) { + _Deallocate_for_capacity(_Al, _Mypair._Myval2._Bx._Ptr, _Old_capacity); _Mypair._Myval2._Bx._Ptr = _New_ptr; } else { _Construct_in_place(_Mypair._Myval2._Bx._Ptr, _New_ptr); @@ -4771,18 +4762,17 @@ private: const size_type _Old_capacity = _My_data._Myres; const size_type _New_capacity = _Calculate_growth(_New_size); auto& _Al = _Getal(); - const pointer _New_ptr = _Al.allocate(_New_capacity + 1); // throws + const pointer _New_ptr = _Allocate_for_capacity(_Al, _New_capacity); // throws - _Start_element_lifetimes(_Unfancy(_New_ptr), _New_capacity + 1); _My_data._Orphan_all(); _ASAN_STRING_REMOVE(*this); _My_data._Mysize = _New_size; _My_data._Myres = _New_capacity; _Elem* const _Raw_new = _Unfancy(_New_ptr); - if (_BUF_SIZE <= _Old_capacity) { + if (_Entails_large_string(_Old_capacity)) { const pointer _Old_ptr = _My_data._Bx._Ptr; _Fn(_Raw_new, _Unfancy(_Old_ptr), _Old_size, _Args...); - _Al.deallocate(_Old_ptr, _Old_capacity + 1); + _Deallocate_for_capacity(_Al, _Old_ptr, _Old_capacity); _My_data._Bx._Ptr = _New_ptr; } else { _Fn(_Raw_new, _My_data._Bx._Buf, _Old_size, _Args...); @@ -4797,7 +4787,7 @@ private: // release any held storage and return to small string mode auto& _My_data = _Mypair._Myval2; _STL_INTERNAL_CHECK(_My_data._Large_string_engaged()); - _STL_INTERNAL_CHECK(_My_data._Mysize < _BUF_SIZE); + _STL_INTERNAL_CHECK(!_Entails_large_string(_My_data._Mysize)); _My_data._Orphan_all(); _ASAN_STRING_REMOVE(*this); @@ -4806,8 +4796,8 @@ private: _Destroy_in_place(_My_data._Bx._Ptr); _My_data._Activate_SSO_buffer(); _Traits::copy(_My_data._Bx._Buf, _Unfancy(_Ptr), _My_data._Mysize + 1); - _Al.deallocate(_Ptr, _My_data._Myres + 1); - _My_data._Myres = _BUF_SIZE - 1; + _Deallocate_for_capacity(_Al, _Ptr, _My_data._Myres); + _My_data._Myres = _SMALL_STRING_CAPACITY; } _CONSTEXPR20 void _Eos(const size_type _New_size) noexcept { // set new length and null terminator @@ -4819,7 +4809,7 @@ private: // initialize basic_string data members auto& _My_data = _Mypair._Myval2; _My_data._Mysize = 0; - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _SMALL_STRING_CAPACITY; _My_data._Activate_SSO_buffer(); // the _Traits::assign is last so the codegen doesn't think the char write can alias this @@ -4835,11 +4825,11 @@ private: auto& _Al = _Getal(); _Destroy_in_place(_My_data._Bx._Ptr); _My_data._Activate_SSO_buffer(); - _Al.deallocate(_Ptr, _My_data._Myres + 1); + _Deallocate_for_capacity(_Al, _Ptr, _My_data._Myres); } _My_data._Mysize = 0; - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _SMALL_STRING_CAPACITY; // the _Traits::assign is last so the codegen doesn't think the char write can alias this _Traits::assign(_My_data._Bx._Buf[0], _Elem()); }