From b1cfcde6d52cdf13e066779cbd00c5d54f4b2cfa Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 19 Oct 2022 12:10:33 -0700 Subject: [PATCH 01/30] initial attempt --- llvm-project | 2 +- stl/inc/vector | 25 +------------------ stl/inc/xmemory | 15 +++++++++++ stl/inc/xstring | 66 +++++++++++++++++++------------------------------ 4 files changed, 42 insertions(+), 66 deletions(-) diff --git a/llvm-project b/llvm-project index 725cdc451a6..b8d38e8b4fc 160000 --- a/llvm-project +++ b/llvm-project @@ -1 +1 @@ -Subproject commit 725cdc451a6bb681e5d6f397a16c2d663d7d9137 +Subproject commit b8d38e8b4fcab071c5c4cb698e154023d06de69e diff --git a/stl/inc/vector b/stl/inc/vector index 7428af2d838..3691fd2ef36 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -539,29 +539,6 @@ private: _Apply_annotation(_My_data._Myfirst, _My_data._Myend, _My_data._Mylast, _My_data._Mylast + _Count); } - _NODISCARD static const void* _Get_aligned_first(const void* _First, const void* _End) noexcept { - const auto _CFirst = reinterpret_cast(_First); - const auto _CEnd = reinterpret_cast(_End); - const size_t _Capacity = static_cast(_CEnd - _CFirst); - - if (_Capacity >= _Asan_granularity) { - // We are guaranteed to have sufficient space to find an aligned address. - return reinterpret_cast( - (reinterpret_cast(_CFirst) + (_Asan_granularity - 1)) & ~(_Asan_granularity - 1)); - } - - uintptr_t _Alignment_offset = reinterpret_cast(_CFirst) & (_Asan_granularity - 1); - if (_Alignment_offset != 0) { - _Alignment_offset = _Asan_granularity - _Alignment_offset; - } - - if (_Capacity > _Alignment_offset) { - return _CFirst + _Alignment_offset; - } - - return nullptr; - } - static _CONSTEXPR20 void _Apply_annotation( pointer _First_, pointer _End_, pointer _Old_last_, pointer _New_last_) noexcept { _STL_INTERNAL_CHECK(_First_ != nullptr); @@ -586,7 +563,7 @@ private: if constexpr (_Has_minimum_allocation_alignment) { __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const void* const _Aligned_first = _Get_aligned_first(_First, _End); + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); if (!_Aligned_first) { // There is no aligned address within the underlying buffer; nothing to do. return; diff --git a/stl/inc/xmemory b/stl/inc/xmemory index bc54938885b..8d5cda1c9fa 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -776,6 +776,21 @@ _NODISCARD constexpr allocation_result::pointe // The number of user bytes a single byte of ASAN shadow memory can track. _INLINE_VAR constexpr size_t _Asan_granularity = 8; +_NODISCARD inline const void* _Get_asan_aligned_first(const void* _First, const void* _Last) noexcept { + const auto _First_address = reinterpret_cast(_First); + const auto _Last_address = reinterpret_cast(_Last); + + const auto _Aligned_address = (_First_address + _Asan_granularity - 1) & ~(_Asan_granularity - 1); + + if (_Aligned_address > _Last_address) { + return nullptr; + } else { + // strict provenance requires that we reconstruct `_Aligned_address` via `_First` + return static_cast(_First) + (_Aligned_address - _First_address); + } +} + + _EXPORT_STD template class allocator { public: diff --git a/stl/inc/xstring b/stl/inc/xstring index f3d77c4cdaf..8c960d44ec2 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2366,7 +2366,6 @@ struct _String_constructor_rvalue_allocator_tag { _Xlength_error("string too long"); } -#if 0 // TRANSITION, VSO-1586016: String annotations disabled temporarily. #if !defined(_M_CEE_PURE) && !defined(_DISABLE_STRING_ANNOTATION) #if defined(__SANITIZE_ADDRESS__) #define _ACTIVATE_STRING_ANNOTATION @@ -2386,7 +2385,6 @@ struct _String_constructor_rvalue_allocator_tag { #pragma comment(lib, "stl_asan") #pragma detect_mismatch("annotate_string", "1") #endif // _ACTIVATE_STRING_ANNOTATION -#endif // TRANSITION, VSO-1586016 #ifdef _INSERT_STRING_ANNOTATION extern "C" { @@ -2537,27 +2535,7 @@ private: _My_data._Myptr(), _My_data._Myres, _My_data._Mysize, static_cast(_My_data._Mysize + _Count)); } - _NODISCARD static const void* _Get_aligned_first(const void* _First, const size_type _Capacity) noexcept { - const char* _CFirst = reinterpret_cast(_First); - - if (_Capacity >= _Asan_granularity) { // We are guaranteed to have sufficient space to find an aligned address - return reinterpret_cast( - (reinterpret_cast(_CFirst) + (_Asan_granularity - 1)) & ~(_Asan_granularity - 1)); - } - - uintptr_t _Alignment_offset = reinterpret_cast(_CFirst) & (_Asan_granularity - 1); - if (_Alignment_offset != 0) { - _Alignment_offset = _Asan_granularity - _Alignment_offset; - } - - if (_Capacity > _Alignment_offset) { - return _CFirst + _Alignment_offset; - } - - return nullptr; - } - - static _CONSTEXPR20 void _Apply_annotation(const value_type* _Ptr, const size_type _Capacity, + static _CONSTEXPR20 void _Apply_annotation(const value_type* const _First, const size_type _Capacity, const size_type _Old_size, const size_type _New_size) noexcept { #if _HAS_CXX20 if (_STD is_constant_evaluated()) { @@ -2569,37 +2547,43 @@ private: return; } - // We need to check whether we have a misaligned SSO buffer because of the proxy in `_Container_base` (e.g. x86) + // When SSO is active, the buffer may not be aligned on an ASan shadow memory boundary; if constexpr (_Memcpy_val_offset % _Asan_granularity != 0) { - const uintptr_t _Alignment_offset = reinterpret_cast(_Ptr) & (_Asan_granularity - 1); - if (_Alignment_offset != 0 && _Capacity == _BUF_SIZE - 1) { - return; + // The alignment can be incorrect on this platform and for this type + if (_Capacity == _BUF_SIZE - 1) { + // This string is in SSO mode, so we need to check if the buffer happens to be aligned or not + const uintptr_t _Buffer_alignment = reinterpret_cast(_First) & (_Asan_granularity - 1); + if (_Buffer_alignment != 0) { + // NOTE TO REVIEWER: shouldn't this do the `_Aligned_first` thing that's done when in large string mode? + return; + } } } - // Needs to consider the null terminator - const char* _First = reinterpret_cast(_Ptr); - const char* _End = reinterpret_cast(_Ptr + _Capacity + 1); - const char* _Old_last = reinterpret_cast(_Ptr + _Old_size + 1); - const char* _New_last = reinterpret_cast(_Ptr + _New_size + 1); + // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator, + // so we need to add one back. + const void* const _Buffer_last = _First + _Capacity + 1; + const void* const _Old_last = _First + _Old_size + 1; + const void* const _New_last = _First + _New_size + 1; if constexpr (_Has_minimum_allocation_alignment_string) { - __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); } else { - const void* _Aligned_first = _Get_aligned_first(_First, _Capacity + 1); + const void* _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); if (!_Aligned_first) { - // There is no aligned address within the underlying buffer. Nothing to do + // There is no aligned address within the underlying buffer. + // This should be impossible, as the minimum allocation size for string is 16... + // NOTE TO REVIEWER: should this be a _STL_ASSERT or smth? return; } - const void* _Aligned_old_last = _Old_last < _Aligned_first ? _Aligned_first : _Old_last; - const void* _Aligned_new_last = _New_last < _Aligned_first ? _Aligned_first : _New_last; - const void* _Aligned_end = _End < _Aligned_first ? _Aligned_first : _End; - __sanitizer_annotate_contiguous_container( - _Aligned_first, _Aligned_end, _Aligned_old_last, _Aligned_new_last); + // last must be >= than first, so fix up when `_Aligned_first > _Old_last` or `_Aligned_first > _New_last` + const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; + const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; + _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _Buffer_last, _Old_last_fixed, _New_last_fixed); } } -#define _ASAN_STRING_MODIFY(n) _Modify_annotation((n)) +#define _ASAN_STRING_MODIFY(n) _Modify_annotation(n) #define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() #define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() #else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv From 34bacc1e3d3b82d82cf5fad213a03eb0a1708639 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 19 Oct 2022 15:30:09 -0700 Subject: [PATCH 02/30] moar tests --- tests/std/tests/GH_002030_asan_annotate_string/env.lst | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/std/tests/GH_002030_asan_annotate_string/env.lst b/tests/std/tests/GH_002030_asan_annotate_string/env.lst index a18ef67c74d..bb1f1577da7 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_string/env.lst @@ -56,8 +56,7 @@ PM_CL="/D_ANNOTATE_STRING /EHsc /MTd /std:c++latest /permissive- /fno-sanitize-a PM_CL="/D_ANNOTATE_STRING /Za /EHsc /MD /std:c++latest /permissive- /fno-sanitize-address-vcasan-lib" PM_CL="/D_ANNOTATE_STRING /Za /EHsc /MDd /std:c++latest /permissive- /fno-sanitize-address-vcasan-lib" # TRANSITION, clang-cl does not support /alternatename so we cannot test /D_ANNOTATE_STRING without -fsanitize=address -# TRANSITION, VSO-1586016: String annotations disabled temporarily. clang-cl fails to link with empty main. -# PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MD /std:c++14" -# PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MD /std:c++17" -# PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive-" -# PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive- /D_HAS_CXX23 /fp:strict" +PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MD /std:c++14" +PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MD /std:c++17" +PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive-" +PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive- /D_HAS_CXX23 /fp:strict" From d51e580bffa6fa9ad56a6933bede3166f63164d1 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 20 Oct 2022 11:53:30 -0700 Subject: [PATCH 03/30] CRs, plus some more minor work --- llvm-project | 2 +- stl/inc/xmemory | 19 ++++++++++++++----- stl/inc/xstring | 38 +++++++++++--------------------------- 3 files changed, 26 insertions(+), 33 deletions(-) diff --git a/llvm-project b/llvm-project index b8d38e8b4fc..725cdc451a6 160000 --- a/llvm-project +++ b/llvm-project @@ -1 +1 @@ -Subproject commit b8d38e8b4fcab071c5c4cb698e154023d06de69e +Subproject commit 725cdc451a6bb681e5d6f397a16c2d663d7d9137 diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 8d5cda1c9fa..9791fa4121a 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -776,20 +776,29 @@ _NODISCARD constexpr allocation_result::pointe // The number of user bytes a single byte of ASAN shadow memory can track. _INLINE_VAR constexpr size_t _Asan_granularity = 8; -_NODISCARD inline const void* _Get_asan_aligned_first(const void* _First, const void* _Last) noexcept { - const auto _First_address = reinterpret_cast(_First); - const auto _Last_address = reinterpret_cast(_Last); +_NODISCARD inline const void* _Get_asan_aligned_first(const void* const _First, const void* const _Last) noexcept { + const auto _First_address = reinterpret_cast(_First); + const auto _Last_address = reinterpret_cast(_Last); const auto _Aligned_address = (_First_address + _Asan_granularity - 1) & ~(_Asan_granularity - 1); if (_Aligned_address > _Last_address) { return nullptr; } else { - // strict provenance requires that we reconstruct `_Aligned_address` via `_First` - return static_cast(_First) + (_Aligned_address - _First_address); + return reinterpret_cast(_Aligned_address); } } +// Works for standard containers with the `value_type` typedef. +template +_INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment = alignof(typename _Container::value_type); + +template +_INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment<_Container, + void_t> = + alignof(typename _Container::value_type) > _Container::allocator_type::_Minimum_allocation_alignment + ? alignof(typename _Container::value_type) + : _Container::allocator_type::_Minimum_allocation_alignment; _EXPORT_STD template class allocator { diff --git a/stl/inc/xstring b/stl/inc/xstring index 8c960d44ec2..94fdadcbcdd 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2419,14 +2419,6 @@ extern const bool _Asan_string_should_annotate; #error Unknown architecture #endif // ^^^ unknown architecture ^^^ -template -_INLINE_VAR constexpr bool _Has_minimum_allocation_alignment_string = - alignof(typename _String::value_type) >= _Asan_granularity; - -template -_INLINE_VAR constexpr bool _Has_minimum_allocation_alignment_string<_String, - void_t> = - _String::allocator_type::_Minimum_allocation_alignment >= _Asan_granularity; #else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv #pragma detect_mismatch("annotate_string", "0") #endif // !_INSERT_STRING_ANNOTATION @@ -2547,25 +2539,16 @@ private: return; } - // When SSO is active, the buffer may not be aligned on an ASan shadow memory boundary; - if constexpr (_Memcpy_val_offset % _Asan_granularity != 0) { - // The alignment can be incorrect on this platform and for this type - if (_Capacity == _BUF_SIZE - 1) { - // This string is in SSO mode, so we need to check if the buffer happens to be aligned or not - const uintptr_t _Buffer_alignment = reinterpret_cast(_First) & (_Asan_granularity - 1); - if (_Buffer_alignment != 0) { - // NOTE TO REVIEWER: shouldn't this do the `_Aligned_first` thing that's done when in large string mode? - return; - } - } - } - // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator, // so we need to add one back. - const void* const _Buffer_last = _First + _Capacity + 1; - const void* const _Old_last = _First + _Old_size + 1; - const void* const _New_last = _First + _New_size + 1; - if constexpr (_Has_minimum_allocation_alignment_string) { + const void* const _Buffer_last = _First + _Capacity + 1; + const void* const _Old_last = _First + _Old_size + 1; + const void* const _New_last = _First + _New_size + 1; + + constexpr bool _Large_string_always_aligned = + (_Container_allocation_minimum_alignment) > _Asan_granularity; + constexpr bool _Small_string_always_aligned = _Memcpy_val_offset % _Asan_granularity == 0; + if constexpr (_Large_string_always_aligned && _Small_string_always_aligned) { __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); } else { const void* _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); @@ -2579,11 +2562,12 @@ private: // last must be >= than first, so fix up when `_Aligned_first > _Old_last` or `_Aligned_first > _New_last` const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; - _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _Buffer_last, _Old_last_fixed, _New_last_fixed); + _STD __sanitizer_annotate_contiguous_container( + _Aligned_first, _Buffer_last, _Old_last_fixed, _New_last_fixed); } } -#define _ASAN_STRING_MODIFY(n) _Modify_annotation(n) +#define _ASAN_STRING_MODIFY(n) _Modify_annotation(n) #define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() #define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() #else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv From c62e749f6164e31a37027d90c96ccbfbc3bca3ba Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Thu, 20 Oct 2022 12:01:20 -0700 Subject: [PATCH 04/30] minor code quality --- stl/inc/vector | 23 ++++++++++++----------- stl/inc/xstring | 5 +++-- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index 3691fd2ef36..147263d5b70 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -540,7 +540,7 @@ private: } static _CONSTEXPR20 void _Apply_annotation( - pointer _First_, pointer _End_, pointer _Old_last_, pointer _New_last_) noexcept { + pointer _First_, pointer _Buffer_last_, pointer _Old_last_, pointer _New_last_) noexcept { _STL_INTERNAL_CHECK(_First_ != nullptr); _STL_INTERNAL_CHECK(_End_ != nullptr); _STL_INTERNAL_CHECK(_Old_last_ != nullptr); @@ -556,12 +556,12 @@ private: return; } - const auto _First = reinterpret_cast(_Unfancy(_First_)); - const auto _End = reinterpret_cast(_Unfancy(_End_)); - const auto _Old_last = reinterpret_cast(_Unfancy(_Old_last_)); - const auto _New_last = reinterpret_cast(_Unfancy(_New_last_)); - if constexpr (_Has_minimum_allocation_alignment) { - __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + const void* const _First = _STD _Unfancy(_First_); + const void* const _Buffer_last = _STD _Unfancy(_Buffer_last_); + const void* const _Old_last = _STD _Unfancy(_Old_last_); + const void* const _New_last = _STD _Unfancy(_New_last_); + if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { + __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); } else { const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); if (!_Aligned_first) { @@ -569,11 +569,12 @@ private: return; } - const void* const _Aligned_old_last = _Old_last < _Aligned_first ? _Aligned_first : _Old_last; - const void* const _Aligned_new_last = _New_last < _Aligned_first ? _Aligned_first : _New_last; - const void* const _Aligned_end = _End < _Aligned_first ? _Aligned_first : _End; + // last must be >= than first, so fix up when `_Aligned_first > _*_last` + const void* const _Buffer_last_fixed = _Buffer_last >= _Aligned_first ? _Buffer_last : _Aligned_first; + const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; + const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; __sanitizer_annotate_contiguous_container( - _Aligned_first, _Aligned_end, _Aligned_old_last, _Aligned_new_last); + _Aligned_first, _Buffer_last_fixed, _Old_last_fixed, _New_last_fixed); } } diff --git a/stl/inc/xstring b/stl/inc/xstring index 94fdadcbcdd..009caa6bc9e 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2551,7 +2551,7 @@ private: if constexpr (_Large_string_always_aligned && _Small_string_always_aligned) { __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); } else { - const void* _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); if (!_Aligned_first) { // There is no aligned address within the underlying buffer. // This should be impossible, as the minimum allocation size for string is 16... @@ -2559,7 +2559,8 @@ private: return; } - // last must be >= than first, so fix up when `_Aligned_first > _Old_last` or `_Aligned_first > _New_last` + // last must be >= than first, so fix up when `_Aligned_first > _*_last` + // note that _Buffer_last must be > 16 from _First, so it certainly is past _Aligned_first (< _First + 8) const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; _STD __sanitizer_annotate_contiguous_container( From 997f9b7629eb12532061a90b95c76cacaa5bead2 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Fri, 21 Oct 2022 10:23:40 -0700 Subject: [PATCH 05/30] fix vector --- stl/inc/vector | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index 147263d5b70..18016684f2b 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -542,7 +542,7 @@ private: static _CONSTEXPR20 void _Apply_annotation( pointer _First_, pointer _Buffer_last_, pointer _Old_last_, pointer _New_last_) noexcept { _STL_INTERNAL_CHECK(_First_ != nullptr); - _STL_INTERNAL_CHECK(_End_ != nullptr); + _STL_INTERNAL_CHECK(_Buffer_last_ != nullptr); _STL_INTERNAL_CHECK(_Old_last_ != nullptr); _STL_INTERNAL_CHECK(_New_last_ != nullptr); @@ -563,7 +563,7 @@ private: if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); if (!_Aligned_first) { // There is no aligned address within the underlying buffer; nothing to do. return; From f108c244bfba002b31f843c9673c2aeba727292a Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 24 Oct 2022 11:54:55 -0700 Subject: [PATCH 06/30] fix stuff with testing --- stl/inc/xstring | 87 +++++++++++++++++++++++++++++++------------------ 1 file changed, 56 insertions(+), 31 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 009caa6bc9e..e97306e866b 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2516,15 +2516,31 @@ private: _Apply_annotation(_My_data._Myptr(), _My_data._Myres, _My_data._Mysize, _My_data._Myres); } - _CONSTEXPR20 void _Modify_annotation(const difference_type _Count) const noexcept { - // Extends/shrinks the annotated range by _Count - if (_Count == 0) { - return; - } + _CONSTEXPR20 void _Modify_annotation(const size_type _Old_size, const size_type _New_size) const noexcept { + auto& _My_data = _Mypair._Myval2; + _Apply_annotation(_My_data._Myptr(), _My_data._Myres, _Old_size, _New_size); + } + + _CONSTEXPR20 void _Annotate_switch_to_large() const noexcept { + // annotates exactly the pointer as valid + _Remove_annotation(); auto& _My_data = _Mypair._Myval2; - _Apply_annotation( - _My_data._Myptr(), _My_data._Myres, _My_data._Mysize, static_cast(_My_data._Mysize + _Count)); + const void* const _My_buf = _My_data._Bx._Buf; + const void* const _Ptr_last = &_My_data._Bx._Ptr + 1; + const void* const _Buf_last = _My_data._Bx._Buf + _BUF_SIZE; + + constexpr bool _Small_string_always_aligned = _Memcpy_val_offset % _Asan_granularity == 0; + if constexpr (_Small_string_always_aligned) { + __sanitizer_annotate_contiguous_container(_My_buf, _Buf_last, _Buf_last, _Ptr_last); + } else { + const auto _My_buf_address = reinterpret_cast(_My_buf); + if ((_My_buf_address & (_Asan_granularity - 1)) == 0) { + __sanitizer_annotate_contiguous_container(_My_buf, _Buf_last, _Buf_last, _Ptr_last); + } + + // otherwise, there's nothing to do - we can't annotate just the latter nybble of the byte + } } static _CONSTEXPR20 void _Apply_annotation(const value_type* const _First, const size_type _Capacity, @@ -2541,7 +2557,7 @@ private: // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator, // so we need to add one back. - const void* const _Buffer_last = _First + _Capacity + 1; + const void* const _Buf_last = _First + _Capacity + 1; const void* const _Old_last = _First + _Old_size + 1; const void* const _New_last = _First + _New_size + 1; @@ -2549,9 +2565,9 @@ private: (_Container_allocation_minimum_alignment) > _Asan_granularity; constexpr bool _Small_string_always_aligned = _Memcpy_val_offset % _Asan_granularity == 0; if constexpr (_Large_string_always_aligned && _Small_string_always_aligned) { - __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); + __sanitizer_annotate_contiguous_container(_First, _Buf_last, _Old_last, _New_last); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buf_last); if (!_Aligned_first) { // There is no aligned address within the underlying buffer. // This should be impossible, as the minimum allocation size for string is 16... @@ -2560,21 +2576,23 @@ private: } // last must be >= than first, so fix up when `_Aligned_first > _*_last` - // note that _Buffer_last must be > 16 from _First, so it certainly is past _Aligned_first (< _First + 8) + // note that _Buf_last must be > 16 from _First, so it certainly is past _Aligned_first (< _First + 8) const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; _STD __sanitizer_annotate_contiguous_container( - _Aligned_first, _Buffer_last, _Old_last_fixed, _New_last_fixed); + _Aligned_first, _Buf_last, _Old_last_fixed, _New_last_fixed); } } -#define _ASAN_STRING_MODIFY(n) _Modify_annotation(n) #define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() #define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() +#define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) (_Str)._Modify_annotation(_Old_size, _New_size) +#define _ASAN_STRING_SWITCH_TO_LARGE(_Str) (_Str)._Annotate_switch_to_large() #else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv -#define _ASAN_STRING_MODIFY(n) #define _ASAN_STRING_REMOVE(_Str) #define _ASAN_STRING_CREATE(_Str) +#define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) +#define _ASAN_STRING_SWITCH_TO_LARGE(_Str) #endif // !_INSERT_STRING_ANNOTATION public: @@ -2713,8 +2731,11 @@ private: _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); if (_Count < _BUF_SIZE) { - _My_data._Mysize = _Count; _My_data._Myres = _BUF_SIZE - 1; + + _My_data._Mysize = _Count; + _ASAN_STRING_CREATE(*this); + if constexpr (_Strat == _Construct_strategy::_From_char) { _Traits::assign(_My_data._Bx._Buf, _Count, _Arg); } else if constexpr (_Strat == _Construct_strategy::_From_ptr) { @@ -2727,7 +2748,6 @@ private: #endif // !_INSERT_STRING_ANNOTATION } - _ASAN_STRING_CREATE(*this); _Proxy._Release(); return; } @@ -2812,9 +2832,11 @@ private: #endif // _HAS_CXX20 _Traits::copy(_Unfancy(_New_ptr), _Old_ptr, _My_data._Mysize); if (_My_data._Myres >= _BUF_SIZE) { // Need to deallocate old storage + _ASAN_STRING_REMOVE(*this); _Al.deallocate(_My_data._Bx._Ptr, _My_data._Myres + 1); _My_data._Bx._Ptr = _New_ptr; } else { + _ASAN_STRING_SWITCH_TO_LARGE(*this); _Construct_in_place(_My_data._Bx._Ptr, _New_ptr); } _My_data._Myres = _New_capacity; @@ -2930,7 +2952,7 @@ public: _My_data._Alloc_proxy(_GET_PROXY_ALLOCATOR(_Alty, _Getal())); // throws, hereafter nothrow in this block _Take_contents(_Left); const auto _Ptr = _My_data._Myptr(); - _ASAN_STRING_MODIFY(static_cast(_Right_size)); + _ASAN_STRING_MODIFY(*this, _Left_size, _New_size); _Traits::copy(_Ptr + _Left_size, _Right_data._Myptr(), _Right_size + 1); _My_data._Mysize = _New_size; return; @@ -2951,7 +2973,7 @@ public: _My_data._Alloc_proxy(_GET_PROXY_ALLOCATOR(_Alty, _Getal())); // throws, hereafter nothrow in this block _Take_contents(_Right); const auto _Ptr = _Unfancy(_My_data._Bx._Ptr); - _ASAN_STRING_MODIFY(static_cast(_Left_size)); + _ASAN_STRING_MODIFY(*this, _Right_size, _New_size); _Traits::move(_Ptr + _Left_size, _Ptr, _Right_size + 1); _Traits::copy(_Ptr, _Left_data._Myptr(), _Left_size); _My_data._Mysize = _New_size; @@ -3025,7 +3047,7 @@ public: _My_data._Myres = _Res - 1; _ASAN_STRING_CREATE(*this); } else { - _ASAN_STRING_MODIFY(static_cast(_Size)); + _ASAN_STRING_MODIFY(*this, _My_data._Mysize, _Size); _Traits::copy(_My_data._Bx._Buf, _Right, _Res); _My_data._Mysize = _Size; _My_data._Myres = _BUF_SIZE - 1; @@ -3138,7 +3160,6 @@ private: #ifdef _INSERT_STRING_ANNOTATION if (!_Right_data._Large_string_engaged()) { - _ASAN_STRING_REMOVE(_Right); _ASAN_STRING_CREATE(*this); } #endif // _INSERT_STRING_ANNOTATION @@ -3160,6 +3181,7 @@ private: _My_data._Mysize = _Right_data._Mysize; _My_data._Myres = _Right_data._Myres; + _ASAN_STRING_CREATE(*this); _Right._Tidy_init(); } @@ -3264,6 +3286,7 @@ private: _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); _My_data._Mysize = _Right_data._Mysize; _My_data._Myres = _Right_data._Myres; + _ASAN_STRING_CREATE(*this); } public: @@ -3328,7 +3351,7 @@ public: #endif // _HAS_CXX23 _CONSTEXPR20 basic_string& operator=(const _Elem _Ch) { // assign {_Ch, _Elem()} - _ASAN_STRING_MODIFY(static_cast(1 - _Mypair._Myval2._Mysize)); + _ASAN_STRING_MODIFY(*this, _Mypair._Myval2._Mysize, 1); _Mypair._Myval2._Mysize = 1; _Elem* const _Ptr = _Mypair._Myval2._Myptr(); _Traits::assign(_Ptr[0], _Ch); @@ -3388,7 +3411,7 @@ public: // append [_Ptr, _Ptr + _Count) const size_type _Old_size = _Mypair._Myval2._Mysize; if (_Count <= _Mypair._Myval2._Myres - _Old_size) { - _ASAN_STRING_MODIFY(static_cast(_Count)); + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Count); _Mypair._Myval2._Mysize = _Old_size + _Count; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Traits::move(_Old_ptr + _Old_size, _Ptr, _Count); @@ -3415,7 +3438,7 @@ public: // append _Count * _Ch const size_type _Old_size = _Mypair._Myval2._Mysize; if (_Count <= _Mypair._Myval2._Myres - _Old_size) { - _ASAN_STRING_MODIFY(static_cast(_Count)); + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Count); _Mypair._Myval2._Mysize = _Old_size + _Count; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Traits::assign(_Old_ptr + _Old_size, _Count, _Ch); @@ -3493,7 +3516,7 @@ public: _In_reads_(_Count) const _Elem* const _Ptr, _CRT_GUARDOVERFLOW const size_type _Count) { // assign [_Ptr, _Ptr + _Count) if (_Count <= _Mypair._Myval2._Myres) { - _ASAN_STRING_MODIFY(static_cast(_Count - _Mypair._Myval2._Mysize)); + _ASAN_STRING_MODIFY(*this, _Mypair._Myval2._Mysize, _Count); _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Mypair._Myval2._Mysize = _Count; _Traits::move(_Old_ptr, _Ptr, _Count); @@ -3517,7 +3540,7 @@ public: _CONSTEXPR20 basic_string& assign(_CRT_GUARDOVERFLOW const size_type _Count, const _Elem _Ch) { // assign _Count * _Ch if (_Count <= _Mypair._Myval2._Myres) { - _ASAN_STRING_MODIFY(static_cast(_Count - _Mypair._Myval2._Mysize)); + _ASAN_STRING_MODIFY(*this, _Mypair._Myval2._Mysize, _Count); _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Mypair._Myval2._Mysize = _Count; _Traits::assign(_Old_ptr, _Count, _Ch); @@ -3617,7 +3640,7 @@ public: #endif // _HAS_CXX20 if (_Check_overlap) { - _ASAN_STRING_MODIFY(static_cast(_Count)); + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Count); _Mypair._Myval2._Mysize = _Old_size + _Count; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Elem* const _Insert_at = _Old_ptr + _Off; @@ -3662,7 +3685,7 @@ public: _Mypair._Myval2._Check_offset(_Off); const size_type _Old_size = _Mypair._Myval2._Mysize; if (_Count <= _Mypair._Myval2._Myres - _Old_size) { - _ASAN_STRING_MODIFY(static_cast(_Count)); + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Count); _Mypair._Myval2._Mysize = _Old_size + _Count; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Elem* const _Insert_at = _Old_ptr + _Off; @@ -3756,7 +3779,7 @@ private: _Elem* const _Erase_at = _My_ptr + _Off; const size_type _New_size = _Old_size - _Count; _Traits::move(_Erase_at, _Erase_at + _Count, _New_size - _Off + 1); // move suffix + null up - _ASAN_STRING_MODIFY(-static_cast(_Count)); + _ASAN_STRING_MODIFY(*this, _Old_size, _New_size); _Mypair._Myval2._Mysize = _New_size; return *this; } @@ -3835,11 +3858,12 @@ public: const size_type _Old_size = _Mypair._Myval2._Mysize; const size_type _Suffix_size = _Old_size - _Nx - _Off + 1; if (_Count < _Nx) { // suffix shifts backwards; we don't have to move anything out of the way - _Mypair._Myval2._Mysize = _Old_size - (_Nx - _Count); _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Elem* const _Insert_at = _Old_ptr + _Off; _Traits::move(_Insert_at, _Ptr, _Count); _Traits::move(_Insert_at + _Count, _Insert_at + _Nx, _Suffix_size); + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size - (_Nx - _Count)); + _Mypair._Myval2._Mysize = _Old_size - (_Nx - _Count); return *this; } @@ -3852,6 +3876,7 @@ public: #endif // _HAS_CXX20 { if (_Growth <= _Mypair._Myval2._Myres - _Old_size) { // growth fits + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Growth); _Mypair._Myval2._Mysize = _Old_size + _Growth; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Elem* const _Insert_at = _Old_ptr + _Off; @@ -4162,7 +4187,7 @@ public: _CONSTEXPR20 void push_back(const _Elem _Ch) { // insert element at end const size_type _Old_size = _Mypair._Myval2._Mysize; if (_Old_size < _Mypair._Myval2._Myres) { - _ASAN_STRING_MODIFY(1); + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + 1); _Mypair._Myval2._Mysize = _Old_size + 1; _Elem* const _Ptr = _Mypair._Myval2._Myptr(); _Traits::assign(_Ptr[_Old_size], _Ch); @@ -4958,7 +4983,7 @@ private: } _CONSTEXPR20 void _Eos(const size_type _New_size) { // set new length and null terminator - _ASAN_STRING_MODIFY(static_cast(_New_size - _Mypair._Myval2._Mysize)); + _ASAN_STRING_MODIFY(*this, _Mypair._Myval2._Mysize, _New_size); _Traits::assign(_Mypair._Myval2._Myptr()[_Mypair._Myval2._Mysize = _New_size], _Elem()); } From 86642d029db748eac3469b826ecf393d5cce285f Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 24 Oct 2022 13:02:04 -0700 Subject: [PATCH 07/30] fixey fixey format moving code back and forth between msvc and stl is Fun --- stl/inc/xstring | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index e97306e866b..dabaacd0d0f 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2525,8 +2525,8 @@ private: // annotates exactly the pointer as valid _Remove_annotation(); - auto& _My_data = _Mypair._Myval2; - const void* const _My_buf = _My_data._Bx._Buf; + auto& _My_data = _Mypair._Myval2; + const void* const _My_buf = _My_data._Bx._Buf; const void* const _Ptr_last = &_My_data._Bx._Ptr + 1; const void* const _Buf_last = _My_data._Bx._Buf + _BUF_SIZE; @@ -2558,8 +2558,8 @@ private: // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator, // so we need to add one back. const void* const _Buf_last = _First + _Capacity + 1; - const void* const _Old_last = _First + _Old_size + 1; - const void* const _New_last = _First + _New_size + 1; + const void* const _Old_last = _First + _Old_size + 1; + const void* const _New_last = _First + _New_size + 1; constexpr bool _Large_string_always_aligned = (_Container_allocation_minimum_alignment) > _Asan_granularity; @@ -2579,15 +2579,14 @@ private: // note that _Buf_last must be > 16 from _First, so it certainly is past _Aligned_first (< _First + 8) const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; - _STD __sanitizer_annotate_contiguous_container( - _Aligned_first, _Buf_last, _Old_last_fixed, _New_last_fixed); + _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _Buf_last, _Old_last_fixed, _New_last_fixed); } } -#define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() -#define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() +#define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() +#define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() #define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) (_Str)._Modify_annotation(_Old_size, _New_size) -#define _ASAN_STRING_SWITCH_TO_LARGE(_Str) (_Str)._Annotate_switch_to_large() +#define _ASAN_STRING_SWITCH_TO_LARGE(_Str) (_Str)._Annotate_switch_to_large() #else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv #define _ASAN_STRING_REMOVE(_Str) #define _ASAN_STRING_CREATE(_Str) @@ -2731,7 +2730,7 @@ private: _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); if (_Count < _BUF_SIZE) { - _My_data._Myres = _BUF_SIZE - 1; + _My_data._Myres = _BUF_SIZE - 1; _My_data._Mysize = _Count; _ASAN_STRING_CREATE(*this); From 277e68b64f6df08e557f6ab41266ffa31bddd253 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 25 Oct 2022 15:35:06 -0700 Subject: [PATCH 08/30] casey crs --- stl/inc/vector | 21 ++++++++++----------- stl/inc/xmemory | 7 ++----- stl/inc/xstring | 19 ++++++++----------- 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index 18016684f2b..ac61f4febf4 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -540,7 +540,7 @@ private: } static _CONSTEXPR20 void _Apply_annotation( - pointer _First_, pointer _Buffer_last_, pointer _Old_last_, pointer _New_last_) noexcept { + pointer _First_, pointer _End_, pointer _Old_last_, pointer _New_last_) noexcept { _STL_INTERNAL_CHECK(_First_ != nullptr); _STL_INTERNAL_CHECK(_Buffer_last_ != nullptr); _STL_INTERNAL_CHECK(_Old_last_ != nullptr); @@ -556,12 +556,12 @@ private: return; } - const void* const _First = _STD _Unfancy(_First_); - const void* const _Buffer_last = _STD _Unfancy(_Buffer_last_); - const void* const _Old_last = _STD _Unfancy(_Old_last_); - const void* const _New_last = _STD _Unfancy(_New_last_); + const void* const _First = _STD _Unfancy(_First_); + const void* const _End = _STD _Unfancy(_End_); + const void* const _Old_last = _STD _Unfancy(_Old_last_); + const void* const _New_last = _STD _Unfancy(_New_last_); if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { - __sanitizer_annotate_contiguous_container(_First, _Buffer_last, _Old_last, _New_last); + __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); if (!_Aligned_first) { @@ -570,11 +570,10 @@ private: } // last must be >= than first, so fix up when `_Aligned_first > _*_last` - const void* const _Buffer_last_fixed = _Buffer_last >= _Aligned_first ? _Buffer_last : _Aligned_first; - const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; - const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; - __sanitizer_annotate_contiguous_container( - _Aligned_first, _Buffer_last_fixed, _Old_last_fixed, _New_last_fixed); + const void* const _End_fixed = _End >= _Aligned_first ? _End : _Aligned_first; + const void* const _Old_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; + const void* const _New_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; + __sanitizer_annotate_contiguous_container(_Aligned_first, _End_fixed, _Old_fixed, _New_fixed); } } diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 9791fa4121a..0b3c098fa16 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -785,20 +785,17 @@ _NODISCARD inline const void* _Get_asan_aligned_first(const void* const _First, if (_Aligned_address > _Last_address) { return nullptr; } else { - return reinterpret_cast(_Aligned_address); + return reinterpret_cast(_Aligned_address); } } -// Works for standard containers with the `value_type` typedef. template _INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment = alignof(typename _Container::value_type); template _INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment<_Container, void_t> = - alignof(typename _Container::value_type) > _Container::allocator_type::_Minimum_allocation_alignment - ? alignof(typename _Container::value_type) - : _Container::allocator_type::_Minimum_allocation_alignment; + (_STD max)(alignof(typename _Container::value_type), _Container::allocator_type::_Minimum_allocation_alignment); _EXPORT_STD template class allocator { diff --git a/stl/inc/xstring b/stl/inc/xstring index dabaacd0d0f..71b644cc6cc 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2504,6 +2504,9 @@ private: #endif // _HAS_CXX17 #ifdef _INSERT_STRING_ANNOTATION + constexpr bool _Small_string_always_asan_aligned = + alignof(*this) >= 8 && _Memcpy_val_offset % _Asan_granularity == 0; + _CONSTEXPR20 void _Create_annotation() const noexcept { // Annotates the valid range with shadow memory auto& _My_data = _Mypair._Myval2; @@ -2527,10 +2530,9 @@ private: auto& _My_data = _Mypair._Myval2; const void* const _My_buf = _My_data._Bx._Buf; - const void* const _Ptr_last = &_My_data._Bx._Ptr + 1; + const void* const _Ptr_last = _STD addressof(_My_data._Bx._Ptr) + 1; const void* const _Buf_last = _My_data._Bx._Buf + _BUF_SIZE; - constexpr bool _Small_string_always_aligned = _Memcpy_val_offset % _Asan_granularity == 0; if constexpr (_Small_string_always_aligned) { __sanitizer_annotate_contiguous_container(_My_buf, _Buf_last, _Buf_last, _Ptr_last); } else { @@ -2555,25 +2557,20 @@ private: return; } - // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator, - // so we need to add one back. + // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator const void* const _Buf_last = _First + _Capacity + 1; const void* const _Old_last = _First + _Old_size + 1; const void* const _New_last = _First + _New_size + 1; constexpr bool _Large_string_always_aligned = (_Container_allocation_minimum_alignment) > _Asan_granularity; - constexpr bool _Small_string_always_aligned = _Memcpy_val_offset % _Asan_granularity == 0; if constexpr (_Large_string_always_aligned && _Small_string_always_aligned) { __sanitizer_annotate_contiguous_container(_First, _Buf_last, _Old_last, _New_last); } else { const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buf_last); - if (!_Aligned_first) { - // There is no aligned address within the underlying buffer. - // This should be impossible, as the minimum allocation size for string is 16... - // NOTE TO REVIEWER: should this be a _STL_ASSERT or smth? - return; - } + // There is no aligned address within the underlying buffer. + // This is impossible, as the minimum allocation size for basic_string is at least 16. + _STL_ASSERT(_Aligned_first); // last must be >= than first, so fix up when `_Aligned_first > _*_last` // note that _Buf_last must be > 16 from _First, so it certainly is past _Aligned_first (< _First + 8) From b76170aee17440685bc6d415961bacc6dcee9178 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 25 Oct 2022 15:49:48 -0700 Subject: [PATCH 09/30] fix comment --- stl/inc/xstring | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 71b644cc6cc..bcdc4380a05 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2568,8 +2568,8 @@ private: __sanitizer_annotate_contiguous_container(_First, _Buf_last, _Old_last, _New_last); } else { const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buf_last); - // There is no aligned address within the underlying buffer. - // This is impossible, as the minimum allocation size for basic_string is at least 16. + // It is impossible for there to be no aligned address within the underlying buffer, + // as the minimum allocation size for basic_string is at least 16. _STL_ASSERT(_Aligned_first); // last must be >= than first, so fix up when `_Aligned_first > _*_last` From 99ea61aad4656990adbe3cbe39472ba7b3aad5f9 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 26 Oct 2022 12:43:40 -0700 Subject: [PATCH 10/30] moar fixin --- stl/inc/vector | 4 ++-- stl/inc/xstring | 40 +++++++++++++++++++++------------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index ac61f4febf4..381c43b05fd 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -542,7 +542,7 @@ private: static _CONSTEXPR20 void _Apply_annotation( pointer _First_, pointer _End_, pointer _Old_last_, pointer _New_last_) noexcept { _STL_INTERNAL_CHECK(_First_ != nullptr); - _STL_INTERNAL_CHECK(_Buffer_last_ != nullptr); + _STL_INTERNAL_CHECK(_End_ != nullptr); _STL_INTERNAL_CHECK(_Old_last_ != nullptr); _STL_INTERNAL_CHECK(_New_last_ != nullptr); @@ -563,7 +563,7 @@ private: if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buffer_last); + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); if (!_Aligned_first) { // There is no aligned address within the underlying buffer; nothing to do. return; diff --git a/stl/inc/xstring b/stl/inc/xstring index bcdc4380a05..8d21e5f0207 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2504,8 +2504,10 @@ private: #endif // _HAS_CXX17 #ifdef _INSERT_STRING_ANNOTATION - constexpr bool _Small_string_always_asan_aligned = - alignof(*this) >= 8 && _Memcpy_val_offset % _Asan_granularity == 0; + // this is a function so we can get the alignment of the class while defining it. + static constexpr bool _Small_string_always_asan_aligned() { + return alignof(basic_string) >= _Asan_granularity && _Memcpy_val_offset % _Asan_granularity == 0; + } _CONSTEXPR20 void _Create_annotation() const noexcept { // Annotates the valid range with shadow memory @@ -2531,14 +2533,14 @@ private: auto& _My_data = _Mypair._Myval2; const void* const _My_buf = _My_data._Bx._Buf; const void* const _Ptr_last = _STD addressof(_My_data._Bx._Ptr) + 1; - const void* const _Buf_last = _My_data._Bx._Buf + _BUF_SIZE; + const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; - if constexpr (_Small_string_always_aligned) { - __sanitizer_annotate_contiguous_container(_My_buf, _Buf_last, _Buf_last, _Ptr_last); + if constexpr (_Small_string_always_asan_aligned()) { + __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); } else { const auto _My_buf_address = reinterpret_cast(_My_buf); if ((_My_buf_address & (_Asan_granularity - 1)) == 0) { - __sanitizer_annotate_contiguous_container(_My_buf, _Buf_last, _Buf_last, _Ptr_last); + __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); } // otherwise, there's nothing to do - we can't annotate just the latter nybble of the byte @@ -2558,25 +2560,25 @@ private: } // Note that `_Capacity`, `_Old_size`, and `_New_size` do not include the null terminator - const void* const _Buf_last = _First + _Capacity + 1; + const void* const _End = _First + _Capacity + 1; const void* const _Old_last = _First + _Old_size + 1; const void* const _New_last = _First + _New_size + 1; - constexpr bool _Large_string_always_aligned = - (_Container_allocation_minimum_alignment) > _Asan_granularity; - if constexpr (_Large_string_always_aligned && _Small_string_always_aligned) { - __sanitizer_annotate_contiguous_container(_First, _Buf_last, _Old_last, _New_last); + constexpr bool _Large_string_always_asan_aligned = + (_Container_allocation_minimum_alignment) >= _Asan_granularity; + if constexpr (_Large_string_always_asan_aligned && _Small_string_always_asan_aligned()) { + __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _Buf_last); - // It is impossible for there to be no aligned address within the underlying buffer, - // as the minimum allocation size for basic_string is at least 16. - _STL_ASSERT(_Aligned_first); + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); + // It is impossible for there to be no aligned address within the underlying buffer. + // The minimum allocation size for basic_string is `_BUF_SIZE * sizeof(value_type)`, + // and `_BUF_SIZE = max(1, floor(16 / sizeof(value_type)))`. + // This implies that the smallest it can be is 9, for `sizeof(value_type) = 9`. // last must be >= than first, so fix up when `_Aligned_first > _*_last` - // note that _Buf_last must be > 16 from _First, so it certainly is past _Aligned_first (< _First + 8) - const void* const _Old_last_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; - const void* const _New_last_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; - _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _Buf_last, _Old_last_fixed, _New_last_fixed); + const void* const _Old_last_fixed = (_STD min)(_Old_last, _Aligned_first); + const void* const _New_last_fixed = (_STD min)(_New_last, _Aligned_first); + _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _End, _Old_last_fixed, _New_last_fixed); } } From 65dbe6450eb5e3658cfffbc9d11c6502f9d6a9c4 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 26 Oct 2022 22:04:59 -0700 Subject: [PATCH 11/30] [ci skip] [wip] re-enable tests getting stuck on: AddressSanitizer CHECK failed: D:\a\_work\1\s\src\vctools\asan\llvm\compiler-rt\lib\asan\asan_poisoning.cpp:405 "((*(u8*)MemToShadow(a))) == ((0))" (0xfc, 0x0) --- stl/inc/xstring | 4 +- .../GH_002030_asan_annotate_string/env.lst | 2 +- .../GH_002030_asan_annotate_string/test.cpp | 56 +++++++++---------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 8d21e5f0207..04ba6d044a6 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2576,8 +2576,8 @@ private: // This implies that the smallest it can be is 9, for `sizeof(value_type) = 9`. // last must be >= than first, so fix up when `_Aligned_first > _*_last` - const void* const _Old_last_fixed = (_STD min)(_Old_last, _Aligned_first); - const void* const _New_last_fixed = (_STD min)(_New_last, _Aligned_first); + const void* const _Old_last_fixed = _Old_last > _Aligned_first ? _Old_last : _Aligned_first; + const void* const _New_last_fixed = _New_last > _Aligned_first ? _New_last : _Aligned_first; _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _End, _Old_last_fixed, _New_last_fixed); } } diff --git a/tests/std/tests/GH_002030_asan_annotate_string/env.lst b/tests/std/tests/GH_002030_asan_annotate_string/env.lst index bb1f1577da7..6a7f9fec2f0 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_string/env.lst @@ -3,7 +3,7 @@ # This test matrix is the usual test matrix, with all currently unsupported options removed, crossed with the ASan flags. # TRANSITION, VSO-1350252 -# Due to a bug in the ASan libs using ASan with /MD or /MT requires IDL==0 and using /MDd or /MTd requires IDL==2. +# Due to a bug in vcasan.lib, using vcasan.lib with /MD or /MT requires IDL==0 and using /MDd or /MTd requires IDL==2. # clang-cl does not currently support targeting /MDd or /MTd. RUNALL_INCLUDE ..\prefix.lst RUNALL_CROSSLIST diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 4d2b11aa430..d4873e3ce98 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -3,7 +3,6 @@ // REQUIRES: asan, x64 || x86 -#if 0 // TRANSITION, VSO-1586016: String annotations disabled temporarily. #pragma warning(disable : 4389) // signed/unsigned mismatch in arithmetic #pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension #pragma warning(disable : 6326) // Potential comparison of a constant with another constant. @@ -17,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -170,29 +170,17 @@ class input_iterator_tester { }; template -bool verify_string(basic_string, Alloc>& str) { +bool verify_string(const basic_string, Alloc>& str) { #ifdef __SANITIZE_ADDRESS__ - constexpr auto proxy_size = _Size_after_ebco_v<_Container_base>; - if constexpr (proxy_size % _Asan_granularity != 0) { // If we have a misaligned SSO buffer we disable ASAN - constexpr size_t max_sso_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)) - 1; - if (str.capacity() == max_sso_size) { - return true; - } - } - - size_t buffer_size = (str.capacity() + 1) * sizeof(CharType); - void* buffer = const_cast(static_cast(str.data())); - void* aligned_start = align(8, 1, buffer, buffer_size); + const void* const buffer = str.data(); + const void* const end = str.data() + (str.capacity() + 1); + const void* const aligned_start = _Get_asan_aligned_first(buffer, end); + assert(aligned_start); - if (!aligned_start) { - return true; - } - - const void* end = const_cast(static_cast(str.data() + (str.capacity() + 1))); - const void* mid = const_cast(static_cast(str.data() + str.size() + 1)); - const void* aligned_mid = mid > aligned_start ? mid : aligned_start; + const void* const mid = str.data() + str.size() + 1; + const void* const fixed_mid = mid > aligned_start ? mid : aligned_start; - return __sanitizer_verify_contiguous_container(aligned_start, aligned_mid, end) != 0; + return __sanitizer_verify_contiguous_container(aligned_start, fixed_mid, end) != 0; #else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv (void) str; return true; @@ -250,7 +238,7 @@ struct explicit_allocator : public custom_test_allocator, Alloc>; { // constructors // range constructors - str literal_constructed{get_large_input()}; - assert(verify_string(literal_constructed)); - str literal_constructed_sso{get_sso_input()}; assert(verify_string(literal_constructed_sso)); + str literal_constructed{get_large_input()}; + assert(verify_string(literal_constructed)); + str initializer_list_constructed({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, CharType{'o'}, CharType{' '}, // CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, @@ -1809,17 +1797,25 @@ void run_tests() { template class Alloc> void run_custom_allocator_matrix() { + cerr << " pocma stateless\n"; run_tests>(); + cerr << " pocma !stateless\n"; run_tests>(); + cerr << " !pocma stateless\n"; run_tests>(); + cerr << " !pocma !stateless\n"; run_tests>(); } template void run_allocator_matrix() { + cerr << " allocator:\n"; run_tests>(); + cerr << " aligned_allocator:\n"; run_custom_allocator_matrix(); + cerr << " explicit_allocator:\n"; run_custom_allocator_matrix(); + cerr << " implicit_allocator:\n"; run_custom_allocator_matrix(); } @@ -1846,16 +1842,18 @@ void test_DevCom_10116361() { } int main() { + cerr << "char:\n"; run_allocator_matrix(); #ifdef __cpp_char8_t + cerr << "char8_t:\n"; run_allocator_matrix(); #endif // __cpp_char8_t + cerr << "char16_t:\n"; run_allocator_matrix(); + cerr << "char32_t:\n"; run_allocator_matrix(); + cerr << "wchar_t:\n"; run_allocator_matrix(); test_DevCom_10116361(); } -#endif // TRANSITION, VSO-1586016 - -int main() {} From d869441e913f633cc1aa057261e4ca7b6da56149 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 31 Oct 2022 18:24:06 -0700 Subject: [PATCH 12/30] ooh, tests are passing! --- stl/inc/xstring | 100 +++++++++++------- .../GH_002030_asan_annotate_string/test.cpp | 14 --- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 04ba6d044a6..3a214353f38 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2526,11 +2526,36 @@ private: _Apply_annotation(_My_data._Myptr(), _My_data._Myres, _Old_size, _New_size); } - _CONSTEXPR20 void _Annotate_switch_to_large() const noexcept { + _CONSTEXPR20 static void _Remove_sso_annotation(const _Scary_val& _My_data) noexcept { + // Removes annotation of the SSO buffer with shadow memory. + // This results in the ability to copy into the SSO buffer without worrying. +#if _HAS_CXX20 + if (_STD is_constant_evaluated()) { + return; + } +#endif // _HAS_CXX20 + const void* const _My_buf = _My_data._Bx._Buf; + const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; + + if constexpr (_Small_string_always_asan_aligned()) { + __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); + } else { + const void* const _Aligned_first = _STD _Get_asan_aligned_first(_My_buf, _End); + // see _Apply_annotation for reasoning on why _Aligned_first doesn't need to be checked. + _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _End, _Aligned_first, _End); + } + } + + _CONSTEXPR20 static void _Annotate_switch_to_large(const _Scary_val& _My_data) noexcept { // annotates exactly the pointer as valid - _Remove_annotation(); +#if _HAS_CXX20 + if (_STD is_constant_evaluated()) { + return; + } +#endif // _HAS_CXX20 + + _Remove_sso_annotation(_My_data); - auto& _My_data = _Mypair._Myval2; const void* const _My_buf = _My_data._Bx._Buf; const void* const _Ptr_last = _STD addressof(_My_data._Bx._Ptr) + 1; const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; @@ -2585,12 +2610,14 @@ private: #define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() #define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() #define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) (_Str)._Modify_annotation(_Old_size, _New_size) -#define _ASAN_STRING_SWITCH_TO_LARGE(_Str) (_Str)._Annotate_switch_to_large() +#define _ASAN_STRING_SWITCH_TO_LARGE(_Scary) _Annotate_switch_to_large(_Scary) +#define _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Scary) _Remove_sso_annotation(_Scary) #else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv #define _ASAN_STRING_REMOVE(_Str) #define _ASAN_STRING_CREATE(_Str) #define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) -#define _ASAN_STRING_SWITCH_TO_LARGE(_Str) +#define _ASAN_STRING_SWITCH_TO_LARGE(_Scary) +#define _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Scary) #endif // !_INSERT_STRING_ANNOTATION public: @@ -2633,13 +2660,13 @@ public: #if _HAS_CXX23 constexpr basic_string(basic_string&& _Right, const size_type _Roff, const _Alloc& _Al = _Alloc()) : _Mypair(_One_then_variadic_args_t{}, _Al) { // construct from _Right [_Roff, ), potentially move - _Move_construct_from_substr(_Right, _Roff, npos, _Al); + _Move_construct_from_substr(_Right, _Roff, npos); } constexpr basic_string( basic_string&& _Right, const size_type _Roff, const size_type _Count, const _Alloc& _Al = _Alloc()) : _Mypair(_One_then_variadic_args_t{}, _Al) { // construct from _Right [_Roff, _Roff + _Count), potentially move - _Move_construct_from_substr(_Right, _Roff, _Count, _Al); + _Move_construct_from_substr(_Right, _Roff, _Count); } #endif // _HAS_CXX23 @@ -2834,7 +2861,7 @@ private: _Al.deallocate(_My_data._Bx._Ptr, _My_data._Myres + 1); _My_data._Bx._Ptr = _New_ptr; } else { - _ASAN_STRING_SWITCH_TO_LARGE(*this); + _ASAN_STRING_SWITCH_TO_LARGE(_My_data); _Construct_in_place(_My_data._Bx._Ptr, _New_ptr); } _My_data._Myres = _New_capacity; @@ -3172,6 +3199,7 @@ private: _Right_data._Bx._Ptr = nullptr; _Swap_proxy_and_iterators(_Right); } else { // copy small string buffer + _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); _Right_data._Orphan_all(); @@ -3184,20 +3212,20 @@ private: } #if _HAS_CXX23 - constexpr void _Move_construct_from_substr( - basic_string& _Right, const size_type _Roff, const size_type _Size_max, const _Alloc& _Al) { + constexpr void _Move_construct_from_substr(basic_string& _Right, const size_type _Roff, const size_type _Size_max) { auto& _Right_data = _Right._Mypair._Myval2; _Right_data._Check_offset(_Roff); const auto _Result_size = _Right_data._Clamp_suffix_size(_Roff, _Size_max); const auto _Right_ptr = _Right_data._Myptr(); + auto& _Al = _Getal(); if (_Allocators_equal(_Al, _Right._Getal()) && _Result_size >= _BUF_SIZE) { if (_Roff != 0) { _Traits::move(_Right_ptr, _Right_ptr + _Roff, _Result_size); } _Right._Eos(_Result_size); - _Mypair._Myval2._Alloc_proxy(_GET_PROXY_ALLOCATOR(_Alty, _Getal())); + _Mypair._Myval2._Alloc_proxy(_GET_PROXY_ALLOCATOR(_Alty, _Al)); _Take_contents(_Right); } else { _Construct<_Construct_strategy::_From_ptr>(_Right_ptr + _Roff, _Result_size); @@ -4391,8 +4419,13 @@ public: // exchange a string in large mode with one in small mode const pointer _Ptr = _Starts_large._Bx._Ptr; _Destroy_in_place(_Starts_large._Bx._Ptr); + + _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Starts_large); _Starts_large._Activate_SSO_buffer(); - _Traits::copy(_Starts_large._Bx._Buf, _Starts_small._Bx._Buf, _BUF_SIZE); + _Traits::copy(_Starts_large._Bx._Buf, _Starts_small._Bx._Buf, _Starts_small._Mysize + 1); + _Starts_large._Bx._Buf[_Starts_small._Mysize] = value_type(); + + _ASAN_STRING_SWITCH_TO_LARGE(_Starts_small); _Construct_in_place(_Starts_small._Bx._Ptr, _Ptr); } @@ -4402,19 +4435,8 @@ public: const bool _My_large = _My_data._Large_string_engaged(); const bool _Right_large = _Right_data._Large_string_engaged(); -#ifdef _INSERT_STRING_ANNOTATION - if (_My_large && _Right_large) { - // nothing - } else if (_My_large) { - _ASAN_STRING_REMOVE(_Right); - } else if (_Right_large) { - _ASAN_STRING_REMOVE(*this); - } else { - _ASAN_STRING_REMOVE(_Right); - _ASAN_STRING_REMOVE(*this); - } -#endif // _INSERT_STRING_ANNOTATION +#if !defined(_INSERT_STRING_ANNOTATION) if constexpr (_Can_memcpy_val) { #if _HAS_CXX20 if (!_STD is_constant_evaluated()) @@ -4429,42 +4451,36 @@ public: _CSTD memcpy(_My_data_mem, _Right_data_mem, _Memcpy_val_size); _CSTD memcpy(_Right_data_mem, _Temp_mem, _Memcpy_val_size); -#ifdef _INSERT_STRING_ANNOTATION - if (_My_large && _Right_large) { - // nothing - } else if (_My_large) { - _ASAN_STRING_CREATE(*this); - } else if (_Right_large) { - _ASAN_STRING_CREATE(_Right); - } else { - _ASAN_STRING_CREATE(_Right); - _ASAN_STRING_CREATE(*this); - } -#endif // _INSERT_STRING_ANNOTATION - return; } } +#endif // !defined(_INSERT_STRING_ANNOTATION) if (_My_large && _Right_large) { // swap buffers, iterators preserved _Swap_adl(_My_data._Bx._Ptr, _Right_data._Bx._Ptr); } else if (_My_large) { // swap large with small _Swap_bx_large_with_small(_My_data, _Right_data); - _ASAN_STRING_CREATE(*this); } else if (_Right_large) { // swap small with large _Swap_bx_large_with_small(_Right_data, _My_data); - _ASAN_STRING_CREATE(_Right); } else { + _ASAN_STRING_REMOVE(*this); + _ASAN_STRING_REMOVE(_Right); _Elem _Temp_buf[_BUF_SIZE]; _Traits::copy(_Temp_buf, _My_data._Bx._Buf, _My_data._Mysize + 1); _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); _Traits::copy(_Right_data._Bx._Buf, _Temp_buf, _My_data._Mysize + 1); - _ASAN_STRING_CREATE(_Right); - _ASAN_STRING_CREATE(*this); } _STD swap(_My_data._Mysize, _Right_data._Mysize); _STD swap(_My_data._Myres, _Right_data._Myres); + + // intentionally swapped, since if `_Right_large` before, now `*this` is large (and vice versa). + if (!_Right_large) { + _ASAN_STRING_CREATE(*this); + } + if (!_My_large) { + _ASAN_STRING_CREATE(_Right); + } } _CONSTEXPR20 void swap(basic_string& _Right) noexcept /* strengthened */ { @@ -4973,6 +4989,7 @@ private: const pointer _Ptr = _My_data._Bx._Ptr; auto& _Al = _Getal(); _Destroy_in_place(_My_data._Bx._Ptr); + _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); _Traits::copy(_My_data._Bx._Buf, _Unfancy(_Ptr), _My_data._Mysize + 1); _Al.deallocate(_Ptr, _My_data._Myres + 1); @@ -5004,6 +5021,7 @@ private: const pointer _Ptr = _My_data._Bx._Ptr; auto& _Al = _Getal(); _Destroy_in_place(_My_data._Bx._Ptr); + _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); _Al.deallocate(_Ptr, _My_data._Myres + 1); } diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index d4873e3ce98..7afa296120c 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -1797,25 +1796,17 @@ void run_tests() { template class Alloc> void run_custom_allocator_matrix() { - cerr << " pocma stateless\n"; run_tests>(); - cerr << " pocma !stateless\n"; run_tests>(); - cerr << " !pocma stateless\n"; run_tests>(); - cerr << " !pocma !stateless\n"; run_tests>(); } template void run_allocator_matrix() { - cerr << " allocator:\n"; run_tests>(); - cerr << " aligned_allocator:\n"; run_custom_allocator_matrix(); - cerr << " explicit_allocator:\n"; run_custom_allocator_matrix(); - cerr << " implicit_allocator:\n"; run_custom_allocator_matrix(); } @@ -1842,17 +1833,12 @@ void test_DevCom_10116361() { } int main() { - cerr << "char:\n"; run_allocator_matrix(); #ifdef __cpp_char8_t - cerr << "char8_t:\n"; run_allocator_matrix(); #endif // __cpp_char8_t - cerr << "char16_t:\n"; run_allocator_matrix(); - cerr << "char32_t:\n"; run_allocator_matrix(); - cerr << "wchar_t:\n"; run_allocator_matrix(); test_DevCom_10116361(); From 90ed742cf411a5d17d6e02bc6b1abbfb5a037c95 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 31 Oct 2022 21:08:31 -0700 Subject: [PATCH 13/30] oh okay, so you need to align both first and end comment explaining what's going on at the top of _Get_asan_aligned_first_last. --- stl/inc/vector | 14 +++--- stl/inc/xmemory | 46 +++++++++++++++++-- stl/inc/xstring | 37 ++++++++------- .../GH_002030_asan_annotate_string/test.cpp | 13 +++--- 4 files changed, 74 insertions(+), 36 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index 381c43b05fd..be57d73fd36 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -563,17 +563,15 @@ private: if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); - if (!_Aligned_first) { - // There is no aligned address within the underlying buffer; nothing to do. + const auto _Aligned = _STD _Get_asan_aligned_first_last(_First, _End); + if (!_Aligned._First) { + // The buffer does not cover at least a full shadow memory section; nothing to do. return; } - // last must be >= than first, so fix up when `_Aligned_first > _*_last` - const void* const _End_fixed = _End >= _Aligned_first ? _End : _Aligned_first; - const void* const _Old_fixed = _Old_last >= _Aligned_first ? _Old_last : _Aligned_first; - const void* const _New_fixed = _New_last >= _Aligned_first ? _New_last : _Aligned_first; - __sanitizer_annotate_contiguous_container(_Aligned_first, _End_fixed, _Old_fixed, _New_fixed); + const void* const _Old_fixed = _Aligned._Clamp(_Old_last); + const void* const _New_fixed = _Aligned._Clamp(_New_last); + __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } } diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 0b3c098fa16..024fc1acacf 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -776,16 +776,52 @@ _NODISCARD constexpr allocation_result::pointe // The number of user bytes a single byte of ASAN shadow memory can track. _INLINE_VAR constexpr size_t _Asan_granularity = 8; -_NODISCARD inline const void* _Get_asan_aligned_first(const void* const _First, const void* const _Last) noexcept { +struct _AsanAlignedPointers { + const void* _First; + const void* _End; + + constexpr const void* _Clamp(const void* _Mid) const noexcept { + if (_Mid < _First) { + return _First; + } else if (_Mid > _End) { + return _End; + } else { + return _Mid; + } + } +}; +// Returns {nullptr, nullptr} if [_First, _Last) does not cover a full ASAN shadow memory section. +// `__sanitizer_annotate_contiguous_container` _requires_ +// its first pointer to be aligned to the beginning of an ASAN shadow memory section (a single 8-byte). +// When the `end` pointer is not on a boundary, it will also mark the remainder of the section. +// This means that if you have, for example: +// struct alignas(8) cat { +// int meow; // bytes [0, 4) +// char buffer[16]; // bytes [4, 20) +// int purr; // bytes [20, 24) +// }; +// you cannot mark `buffer + [0, 4)`, since `buffer + 0` doesn't start a shadow memory section +// (i.e., it's not on an 8-byte boundary). +// Additionally, if you attempt to mark up to `buffer + 16` (the real end of buffer), it'll also mark `purr`. +// Thus, the only part you can _actually_ mark is the complete shadow memory section of `buffer + [4, 12)` +// (i.e., bytes [8, 16)). +_NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_last( + const void* const _First, const void* const _Last) noexcept { + constexpr static uintptr_t _Mask = ~(_Asan_granularity - 1); + const auto _First_address = reinterpret_cast(_First); const auto _Last_address = reinterpret_cast(_Last); - const auto _Aligned_address = (_First_address + _Asan_granularity - 1) & ~(_Asan_granularity - 1); + const uintptr_t _Aligned_first_address = (_First_address + _Asan_granularity - 1) & _Mask; + const uintptr_t _Aligned_last_address = _Last_address & _Mask; - if (_Aligned_address > _Last_address) { - return nullptr; + if (_Aligned_first_address == _Aligned_last_address) { + return {nullptr, nullptr}; } else { - return reinterpret_cast(_Aligned_address); + return { + reinterpret_cast(_Aligned_first_address), + reinterpret_cast(_Aligned_last_address), + }; } } diff --git a/stl/inc/xstring b/stl/inc/xstring index 3a214353f38..5fc21f36086 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2538,11 +2538,15 @@ private: const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; if constexpr (_Small_string_always_asan_aligned()) { - __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); + _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_My_buf, _End); - // see _Apply_annotation for reasoning on why _Aligned_first doesn't need to be checked. - _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _End, _Aligned_first, _End); + const auto _Aligned = _STD _Get_asan_aligned_first_last(_My_buf, _End); + if (!_Aligned._First) { + // The buffer does not cover at least a full shadow memory section; nothing to do. + return; + } + _STD __sanitizer_annotate_contiguous_container( + _Aligned._First, _Aligned._End, _Aligned._First, _Aligned._End); } } @@ -2561,11 +2565,11 @@ private: const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; if constexpr (_Small_string_always_asan_aligned()) { - __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); + _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); } else { const auto _My_buf_address = reinterpret_cast(_My_buf); if ((_My_buf_address & (_Asan_granularity - 1)) == 0) { - __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); + _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); } // otherwise, there's nothing to do - we can't annotate just the latter nybble of the byte @@ -2592,18 +2596,18 @@ private: constexpr bool _Large_string_always_asan_aligned = (_Container_allocation_minimum_alignment) >= _Asan_granularity; if constexpr (_Large_string_always_asan_aligned && _Small_string_always_asan_aligned()) { - __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + _STD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const void* const _Aligned_first = _STD _Get_asan_aligned_first(_First, _End); - // It is impossible for there to be no aligned address within the underlying buffer. - // The minimum allocation size for basic_string is `_BUF_SIZE * sizeof(value_type)`, - // and `_BUF_SIZE = max(1, floor(16 / sizeof(value_type)))`. - // This implies that the smallest it can be is 9, for `sizeof(value_type) = 9`. + const auto _Aligned = _STD _Get_asan_aligned_first_last(_First, _End); + if (!_Aligned._First) { + // The buffer does not cover at least a full shadow memory section; nothing to do. + return; + } - // last must be >= than first, so fix up when `_Aligned_first > _*_last` - const void* const _Old_last_fixed = _Old_last > _Aligned_first ? _Old_last : _Aligned_first; - const void* const _New_last_fixed = _New_last > _Aligned_first ? _New_last : _Aligned_first; - _STD __sanitizer_annotate_contiguous_container(_Aligned_first, _End, _Old_last_fixed, _New_last_fixed); + // last must be >= than first, so fix up when `_Aligned._First > _*_last` + const void* const _Old_fixed = _Aligned._Clamp(_Old_last); + const void* const _New_fixed = _Aligned._Clamp(_New_last); + _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } } @@ -4423,7 +4427,6 @@ public: _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Starts_large); _Starts_large._Activate_SSO_buffer(); _Traits::copy(_Starts_large._Bx._Buf, _Starts_small._Bx._Buf, _Starts_small._Mysize + 1); - _Starts_large._Bx._Buf[_Starts_small._Mysize] = value_type(); _ASAN_STRING_SWITCH_TO_LARGE(_Starts_small); _Construct_in_place(_Starts_small._Bx._Ptr, _Ptr); diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 7afa296120c..65048cf9898 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -171,15 +171,16 @@ class input_iterator_tester { template bool verify_string(const basic_string, Alloc>& str) { #ifdef __SANITIZE_ADDRESS__ - const void* const buffer = str.data(); - const void* const end = str.data() + (str.capacity() + 1); - const void* const aligned_start = _Get_asan_aligned_first(buffer, end); - assert(aligned_start); + const void* const buffer = str.data(); + const void* const end = str.data() + (str.capacity() + 1); + const auto aligned = _Get_asan_aligned_first_last(buffer, end); + assert(aligned._First); + assert(aligned._End); const void* const mid = str.data() + str.size() + 1; - const void* const fixed_mid = mid > aligned_start ? mid : aligned_start; + const void* const fixed_mid = aligned._Clamp(mid); - return __sanitizer_verify_contiguous_container(aligned_start, fixed_mid, end) != 0; + return __sanitizer_verify_contiguous_container(aligned._First, fixed_mid, aligned._End) != 0; #else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv (void) str; return true; From 7f52caa4aea48ff7a4912a32df58d3239fe8de74 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 1 Nov 2022 07:19:10 -0700 Subject: [PATCH 14/30] CRs, plus fix vector test additionally, in tests, assume std::allocator is Good --- stl/inc/vector | 2 +- stl/inc/xmemory | 8 ++--- stl/inc/xstring | 13 +++++--- .../GH_002030_asan_annotate_string/test.cpp | 12 +++++-- .../GH_002030_asan_annotate_vector/test.cpp | 32 ++++++++++++------- 5 files changed, 43 insertions(+), 24 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index be57d73fd36..5ec7da99d02 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -563,7 +563,7 @@ private: if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const auto _Aligned = _STD _Get_asan_aligned_first_last(_First, _End); + const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); if (!_Aligned._First) { // The buffer does not cover at least a full shadow memory section; nothing to do. return; diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 024fc1acacf..6b368210508 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -780,7 +780,7 @@ struct _AsanAlignedPointers { const void* _First; const void* _End; - constexpr const void* _Clamp(const void* _Mid) const noexcept { + _NODISCARD constexpr const void* _Clamp(const void* _Mid) const noexcept { if (_Mid < _First) { return _First; } else if (_Mid > _End) { @@ -790,7 +790,7 @@ struct _AsanAlignedPointers { } } }; -// Returns {nullptr, nullptr} if [_First, _Last) does not cover a full ASAN shadow memory section. +// Returns {nullptr, nullptr} if [_First, _End) does not cover a full ASAN shadow memory section. // `__sanitizer_annotate_contiguous_container` _requires_ // its first pointer to be aligned to the beginning of an ASAN shadow memory section (a single 8-byte). // When the `end` pointer is not on a boundary, it will also mark the remainder of the section. @@ -805,7 +805,7 @@ struct _AsanAlignedPointers { // Additionally, if you attempt to mark up to `buffer + 16` (the real end of buffer), it'll also mark `purr`. // Thus, the only part you can _actually_ mark is the complete shadow memory section of `buffer + [4, 12)` // (i.e., bytes [8, 16)). -_NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_last( +_NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_end( const void* const _First, const void* const _Last) noexcept { constexpr static uintptr_t _Mask = ~(_Asan_granularity - 1); @@ -815,7 +815,7 @@ _NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_last( const uintptr_t _Aligned_first_address = (_First_address + _Asan_granularity - 1) & _Mask; const uintptr_t _Aligned_last_address = _Last_address & _Mask; - if (_Aligned_first_address == _Aligned_last_address) { + if (_Aligned_first_address >= _Aligned_last_address) { return {nullptr, nullptr}; } else { return { diff --git a/stl/inc/xstring b/stl/inc/xstring index 5fc21f36086..4d708c4e625 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2505,7 +2505,7 @@ private: #ifdef _INSERT_STRING_ANNOTATION // this is a function so we can get the alignment of the class while defining it. - static constexpr bool _Small_string_always_asan_aligned() { + _NODISCARD static constexpr bool _Small_string_always_asan_aligned() noexcept { return alignof(basic_string) >= _Asan_granularity && _Memcpy_val_offset % _Asan_granularity == 0; } @@ -2522,6 +2522,10 @@ private: } _CONSTEXPR20 void _Modify_annotation(const size_type _Old_size, const size_type _New_size) const noexcept { + if (_Old_size == _New_size) { + return; + } + auto& _My_data = _Mypair._Myval2; _Apply_annotation(_My_data._Myptr(), _My_data._Myres, _Old_size, _New_size); } @@ -2540,7 +2544,7 @@ private: if constexpr (_Small_string_always_asan_aligned()) { _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); } else { - const auto _Aligned = _STD _Get_asan_aligned_first_last(_My_buf, _End); + const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); if (!_Aligned._First) { // The buffer does not cover at least a full shadow memory section; nothing to do. return; @@ -2598,7 +2602,7 @@ private: if constexpr (_Large_string_always_asan_aligned && _Small_string_always_asan_aligned()) { _STD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { - const auto _Aligned = _STD _Get_asan_aligned_first_last(_First, _End); + const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); if (!_Aligned._First) { // The buffer does not cover at least a full shadow memory section; nothing to do. return; @@ -2842,6 +2846,8 @@ private: } } + _ASAN_STRING_REMOVE(*this); + _Tidy_deallocate_guard _Guard{this}; for (; _First != _Last; ++_First) { if constexpr (!is_same_v<_Size, size_type>) { @@ -2861,7 +2867,6 @@ private: #endif // _HAS_CXX20 _Traits::copy(_Unfancy(_New_ptr), _Old_ptr, _My_data._Mysize); if (_My_data._Myres >= _BUF_SIZE) { // Need to deallocate old storage - _ASAN_STRING_REMOVE(*this); _Al.deallocate(_My_data._Bx._Ptr, _My_data._Myres + 1); _My_data._Bx._Ptr = _New_ptr; } else { diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 65048cf9898..03226b7a6d5 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -173,9 +173,15 @@ bool verify_string(const basic_string, Alloc>& s #ifdef __SANITIZE_ADDRESS__ const void* const buffer = str.data(); const void* const end = str.data() + (str.capacity() + 1); - const auto aligned = _Get_asan_aligned_first_last(buffer, end); - assert(aligned._First); - assert(aligned._End); + + _AsanAlignedPointers aligned; + if constexpr ((_Container_allocation_minimum_alignment, Alloc>>) > 8) { + aligned = {buffer, buf_end}; + } else { + aligned = _Get_asan_aligned_first_end(buffer, end); + assert(aligned._First); + assert(aligned._End); + } const void* const mid = str.data() + str.size() + 1; const void* const fixed_mid = aligned._Clamp(mid); diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp index ff60de47e53..025e9570ae6 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp @@ -3,6 +3,8 @@ // REQUIRES: x64 || x86 +#pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension + #include #include #include @@ -148,19 +150,23 @@ class input_iterator_tester { template bool verify_vector(vector& vec) { #ifdef __SANITIZE_ADDRESS__ - size_t buffer_bytes = vec.capacity() * sizeof(T); - void* buffer = vec.data(); - void* aligned_start = align(8, 1, buffer, buffer_bytes); + const void* buffer = vec.data(); + const void* buf_end = vec.data() + vec.capacity(); + _AsanAlignedPointers aligned; - if (!aligned_start) { - return true; + if constexpr ((_Container_allocation_minimum_alignment>) > 8) { + aligned = {buffer, buf_end}; + } else { + aligned = _Get_asan_aligned_first_end(buffer, buf_end); + if (!aligned._First) { + return true; + } } - void* mid = vec.data() + vec.size(); - mid = mid > aligned_start ? mid : aligned_start; + const void* const mid = vec.data() + vec.size(); + const void* const fixed_mid = aligned._Clamp(mid); - void* bad_address = - __sanitizer_contiguous_container_find_bad_address(aligned_start, mid, vec.data() + vec.capacity()); + void* bad_address = __sanitizer_contiguous_container_find_bad_address(aligned._First, fixed_mid, aligned._End); if (bad_address == nullptr) { return true; } @@ -172,9 +178,11 @@ bool verify_vector(vector& vec) { } cout << "Vector State:" << endl; cout << " begin: " << buffer << endl; - cout << " aligned begin: " << aligned_start << endl; - cout << " last: " << reinterpret_cast(vec.data() + vec.size()) << endl; - cout << " end: " << reinterpret_cast(vec.data() + vec.capacity()) << endl; + cout << " aligned begin: " << aligned._First << endl; + cout << " last: " << mid << endl; + cout << " aligned_last: " << fixed_mid << endl; + cout << " end: " << buf_end << endl; + cout << " aligned_end: " << aligned._End << endl; __asan_describe_address(bad_address); return false; From 8afead500dd7dd06e31db159ab84bc5a50f7c1fe Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 1 Nov 2022 07:50:08 -0700 Subject: [PATCH 15/30] add test for DevCom-10109507 --- tests/std/tests/GH_002030_asan_annotate_string/test.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 03226b7a6d5..d46dc6e20a8 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -1839,6 +1839,13 @@ void test_DevCom_10116361() { s1.~string(); } +void test_DevCom_10109507() { + // replace failed to correctly munge asan annotations while working + string s("abcd"); + s.replace(0, 1, "ef", 2); + s.replace(0, 0, "xy", 2); +} + int main() { run_allocator_matrix(); #ifdef __cpp_char8_t @@ -1849,4 +1856,5 @@ int main() { run_allocator_matrix(); test_DevCom_10116361(); + test_DevCom_10109507(); } From 888cc8bc5143191802696719859aa7c2e684159b Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 1 Nov 2022 08:26:42 -0700 Subject: [PATCH 16/30] please tell me i actually fixed the tests this time tested all of them since I'm now on a powerful enough computer that that doesn't take ages --- tests/std/tests/GH_002030_asan_annotate_string/env.lst | 6 +++--- tests/std/tests/GH_002030_asan_annotate_string/test.cpp | 9 +++------ tests/std/tests/GH_002030_asan_annotate_vector/env.lst | 6 +++--- tests/std/tests/GH_002030_asan_annotate_vector/test.cpp | 8 ++++++-- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tests/std/tests/GH_002030_asan_annotate_string/env.lst b/tests/std/tests/GH_002030_asan_annotate_string/env.lst index 6a7f9fec2f0..30aec2f679c 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_string/env.lst @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # This test matrix is the usual test matrix, with all currently unsupported options removed, crossed with the ASan flags. -# TRANSITION, VSO-1350252 -# Due to a bug in vcasan.lib, using vcasan.lib with /MD or /MT requires IDL==0 and using /MDd or /MTd requires IDL==2. -# clang-cl does not currently support targeting /MDd or /MTd. + +# TRANSITION, VSO-1350252 - due to vcasan.lib including the standard library, we can't use it (pending new release). +# TRANSITION, google/sanitizers#328 - clang-cl does not currently support targeting /MDd or /MTd. RUNALL_INCLUDE ..\prefix.lst RUNALL_CROSSLIST PM_CL="/Zi /wd4611 /w14640 /Zc:threadSafeInit-" PM_LINK="/debug" diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index d46dc6e20a8..20ac76cad86 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -3,12 +3,9 @@ // REQUIRES: asan, x64 || x86 -#pragma warning(disable : 4389) // signed/unsigned mismatch in arithmetic #pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension -#pragma warning(disable : 6326) // Potential comparison of a constant with another constant. #ifdef __clang__ -#pragma clang diagnostic ignored "-Wsign-compare" #pragma clang diagnostic ignored "-Wc++17-extensions" // constexpr if is a C++17 extension #endif // __clang__ @@ -171,14 +168,14 @@ class input_iterator_tester { template bool verify_string(const basic_string, Alloc>& str) { #ifdef __SANITIZE_ADDRESS__ - const void* const buffer = str.data(); - const void* const end = str.data() + (str.capacity() + 1); + const void* const buffer = str.data(); + const void* const buf_end = str.data() + (str.capacity() + 1); _AsanAlignedPointers aligned; if constexpr ((_Container_allocation_minimum_alignment, Alloc>>) > 8) { aligned = {buffer, buf_end}; } else { - aligned = _Get_asan_aligned_first_end(buffer, end); + aligned = _Get_asan_aligned_first_end(buffer, buf_end); assert(aligned._First); assert(aligned._End); } diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/env.lst b/tests/std/tests/GH_002030_asan_annotate_vector/env.lst index b6dd36fe057..9e11db811c8 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_vector/env.lst @@ -2,9 +2,9 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # This test matrix is the usual test matrix, with all currently unsupported options removed, crossed with the ASan flags. -# TRANSITION, VSO-1350252 -# Due to a bug in the ASan libs using ASan with /MD or /MT requires IDL==0 and using /MDd or /MTd requires IDL==2. -# clang-cl does not currently support targeting /MDd or /MTd. + +# TRANSITION, VSO-1350252 - due to vcasan.lib including the standard library, we can't use it (pending new release). +# TRANSITION, google/sanitizers#328 - clang-cl does not currently support targeting /MDd or /MTd. RUNALL_INCLUDE ..\prefix.lst RUNALL_CROSSLIST PM_CL="/Zi /wd4611 /w14640 /Zc:threadSafeInit-" PM_LINK="/debug" diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp index 025e9570ae6..6bc9af7f29a 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp @@ -3,8 +3,6 @@ // REQUIRES: x64 || x86 -#pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension - #include #include #include @@ -14,6 +12,12 @@ #include #include +#pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension + +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wc++17-extensions" // constexpr if is a C++17 extension +#endif // __clang__ + using namespace std; #ifndef __SANITIZE_ADDRESS__ From dfaa6b1312818997a1636aab71f300c36d34929a Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 1 Nov 2022 15:12:05 -0700 Subject: [PATCH 17/30] ooh we can actually DO MORE --- stl/inc/vector | 23 ++++- stl/inc/xmemory | 95 ++++++++++++------- stl/inc/xstring | 67 +++++++++---- .../GH_002030_asan_annotate_string/env.lst | 4 +- .../GH_002030_asan_annotate_string/test.cpp | 7 +- .../GH_002030_asan_annotate_vector/test.cpp | 4 +- 6 files changed, 135 insertions(+), 65 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index 5ec7da99d02..e71dc28812f 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -561,16 +561,31 @@ private: const void* const _Old_last = _STD _Unfancy(_Old_last_); const void* const _New_last = _STD _Unfancy(_New_last_); if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { + // old state: + // [_First, _Old_last) valid + // [_Old_last, _End) poison + // new state: + // [_First, _New_last) valid + // [_New_last, _End) poison __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); - if (!_Aligned._First) { - // The buffer does not cover at least a full shadow memory section; nothing to do. + if (_Aligned._First == _Aligned._End) { + // The buffer does not end at least one shadow memory section; nothing to do. return; } - const void* const _Old_fixed = _Aligned._Clamp(_Old_last); - const void* const _New_fixed = _Aligned._Clamp(_New_last); + const void* const _Old_fixed = _Aligned._Clamp_to_end(_Old_last); + const void* const _New_fixed = _Aligned._Clamp_to_end(_New_last); + + // old state: + // [_Aligned._First, _Old_fixed) valid + // [_Old_fixed, _Aligned._End) poison + // [_Aligned._End, _End) valid + // new state: + // [_Aligned._First, _New_fixed) valid + // [_New_fixed, _Aligned._End) poison + // [_Aligned._End, _End) valid __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } } diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 6b368210508..f0fd32a1582 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -780,49 +780,80 @@ struct _AsanAlignedPointers { const void* _First; const void* _End; - _NODISCARD constexpr const void* _Clamp(const void* _Mid) const noexcept { - if (_Mid < _First) { - return _First; - } else if (_Mid > _End) { + _NODISCARD constexpr const void* _Clamp_to_end(const void* _Mid) const noexcept { + _STL_ASSERT(_Mid >= _First, ""); + if (_Mid > _End) { return _End; } else { return _Mid; } } }; -// Returns {nullptr, nullptr} if [_First, _End) does not cover a full ASAN shadow memory section. -// `__sanitizer_annotate_contiguous_container` _requires_ -// its first pointer to be aligned to the beginning of an ASAN shadow memory section (a single 8-byte). -// When the `end` pointer is not on a boundary, it will also mark the remainder of the section. -// This means that if you have, for example: + +// The way that ASan shadow memory works, each eight byte block of memory ("shadow memory section") +// has a single byte to mark it as either poison or valid. +// Each section has 0 to 8 "valid" bytes followed by poison bytes, so: +// [ v v v p p p p p ] +// or +// [ v v v v v v v v ] +// are okay, but +// [ p p p p v v v v ] +// is not. +// +// This function exists to fix up `first` and `end` pointers so that one can call +// `__sanitizer_annotate_contiguous_container`: +// +// - `__sanitizer_annotate_contiguous_container` checks that `first` is aligned to an 8-byte boundary +// - if `end` is not aligned to an 8-byte boundary, `__sanitizer_annotate_contiguous_container` still poisons the +// remaining bytes in the shadow memory section. +// +// Because of the second property, we can only mark poison up to the final aligned address before the true `last`. +// Otherwise, we'd poison the memory _after_ `last` as well. +// For the first property, we can assume that everything before `first` in the shadow memory section is valid +// (since otherwise we couldn't mark `first` valid), and so we just return back the first address in +// `first`'s shadow memory section. +// +// ### Example +// +// ```cpp // struct alignas(8) cat { -// int meow; // bytes [0, 4) -// char buffer[16]; // bytes [4, 20) -// int purr; // bytes [20, 24) +// int meow; // bytes [0, 4) +// char buffer[16]; // bytes [4, 20) +// int purr; // bytes [20, 24) +// }; +// ``` +// +// First, `meow` and `purr` are just regular data members, not container buffers, so they _must_ be valid. +// Then, assume we want to poison all of `buffer`. +// We call `aligned = _Get_asan_aligned_first_end(cat.buffer, cat.buffer + 16);`, and we get back +// +// ```cpp +// aligned = { +// ._First = &cat.meow, +// ._End = cat.buffer + 12, // }; -// you cannot mark `buffer + [0, 4)`, since `buffer + 0` doesn't start a shadow memory section -// (i.e., it's not on an 8-byte boundary). -// Additionally, if you attempt to mark up to `buffer + 16` (the real end of buffer), it'll also mark `purr`. -// Thus, the only part you can _actually_ mark is the complete shadow memory section of `buffer + [4, 12)` -// (i.e., bytes [8, 16)). +// ``` +// +// Then, we poison as much of buffer as we can via +// +// ```cpp +// __sanitizer_annotate_contiguous_container( +// aligned._First, +// aligned._End, +// cat.buffer, +// aligned._Clamp_to_end(cat.buffer + 16)); +// ``` +// +// We are allowed to assume that `aligned._First` is valid, since otherwise `cat.buffer + [0, 4)` could not be valid. +// We cannot poison up to `cat.buffer + 16`, since then `purr` could not be valid. _NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_end( const void* const _First, const void* const _Last) noexcept { - constexpr static uintptr_t _Mask = ~(_Asan_granularity - 1); - - const auto _First_address = reinterpret_cast(_First); - const auto _Last_address = reinterpret_cast(_Last); - - const uintptr_t _Aligned_first_address = (_First_address + _Asan_granularity - 1) & _Mask; - const uintptr_t _Aligned_last_address = _Last_address & _Mask; + static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); - if (_Aligned_first_address >= _Aligned_last_address) { - return {nullptr, nullptr}; - } else { - return { - reinterpret_cast(_Aligned_first_address), - reinterpret_cast(_Aligned_last_address), - }; - } + return { + reinterpret_cast(reinterpret_cast(_First) & _Mask), + reinterpret_cast(reinterpret_cast(_Last) & _Mask), + }; } template diff --git a/stl/inc/xstring b/stl/inc/xstring index 4d708c4e625..5fe1cd37135 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2506,7 +2506,8 @@ private: #ifdef _INSERT_STRING_ANNOTATION // this is a function so we can get the alignment of the class while defining it. _NODISCARD static constexpr bool _Small_string_always_asan_aligned() noexcept { - return alignof(basic_string) >= _Asan_granularity && _Memcpy_val_offset % _Asan_granularity == 0; + return alignof(basic_string) >= _Asan_granularity && _Memcpy_val_offset % _Asan_granularity == 0 + && (sizeof(value_type[_BUF_SIZE]) % _Asan_granularity == 0 || alignof(size_type) >= _Asan_granularity); } _CONSTEXPR20 void _Create_annotation() const noexcept { @@ -2542,15 +2543,20 @@ private: const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; if constexpr (_Small_string_always_asan_aligned()) { + // old state: + // [_My_buf, _End) unknown + // new state: + // [_My_buf, _End) valid _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); - if (!_Aligned._First) { - // The buffer does not cover at least a full shadow memory section; nothing to do. - return; - } - _STD __sanitizer_annotate_contiguous_container( - _Aligned._First, _Aligned._End, _Aligned._First, _Aligned._End); + // old state: + // [_Aligned._First, _My_buf) valid + // [_My_buf, _Aligned._End) unknown + // [_Aligned._End, _End) valid + // new state: + // [_Aligned._First, _End) valid + _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _My_buf, _Aligned._End); } } @@ -2569,14 +2575,21 @@ private: const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; if constexpr (_Small_string_always_asan_aligned()) { + // old state: + // [_My_buf, _End) valid (from _Remove_sso_annotation) + // new state: + // [_My_buf, _Ptr_last) valid + // [_Ptr_last, _End) poison _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); } else { - const auto _My_buf_address = reinterpret_cast(_My_buf); - if ((_My_buf_address & (_Asan_granularity - 1)) == 0) { - _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); - } - - // otherwise, there's nothing to do - we can't annotate just the latter nybble of the byte + const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); + // old state: + // [_Aligned._First, _End) valid (from _Remove_sso_annotation) + // new state: + // [_Aligned._First, _Ptr_last) valid + // [_Ptr_last, _Aligned._End) poison + // [_Aligned._End, _End) valid + _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Aligned._End, _Ptr_last); } } @@ -2600,17 +2613,29 @@ private: constexpr bool _Large_string_always_asan_aligned = (_Container_allocation_minimum_alignment) >= _Asan_granularity; if constexpr (_Large_string_always_asan_aligned && _Small_string_always_asan_aligned()) { + // old state: + // [_First, _Old_last) valid + // [_Old_last, _End) poison + // new state: + // [_First, _New_last) valid + // [_New_last, _End) poison _STD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); - if (!_Aligned._First) { - // The buffer does not cover at least a full shadow memory section; nothing to do. - return; - } - - // last must be >= than first, so fix up when `_Aligned._First > _*_last` - const void* const _Old_fixed = _Aligned._Clamp(_Old_last); - const void* const _New_fixed = _Aligned._Clamp(_New_last); + // The buffer must always have size >= 9 bytes, so it will always end at least one shadow memory section. + + // last must be <= than end, so fix up when `_Aligned._End < _*_last` + const void* const _Old_fixed = _Aligned._Clamp_to_end(_Old_last); + const void* const _New_fixed = _Aligned._Clamp_to_end(_New_last); + + // old state: + // [_Aligned._First, _Old_fixed) valid + // [_Old_fixed, _Aligned._End) poison + // [_Aligned._End, _End) valid + // new state: + // [_Aligned._First, _New_fixed) valid + // [_New_fixed, _Aligned._End) poison + // [_Aligned._End, _End) valid _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } } diff --git a/tests/std/tests/GH_002030_asan_annotate_string/env.lst b/tests/std/tests/GH_002030_asan_annotate_string/env.lst index 30aec2f679c..60ca4d911b9 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_string/env.lst @@ -58,5 +58,5 @@ PM_CL="/D_ANNOTATE_STRING /Za /EHsc /MDd /std:c++latest /permissive- /fno-saniti # TRANSITION, clang-cl does not support /alternatename so we cannot test /D_ANNOTATE_STRING without -fsanitize=address PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MD /std:c++14" PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MD /std:c++17" -PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive-" -PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive- /D_HAS_CXX23 /fp:strict" +PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++20 /permissive-" +PM_COMPILER="clang-cl" PM_CL="-fsanitize=address -fno-ms-compatibility -fno-delayed-template-parsing -Wno-unqualified-std-cast-call /EHsc /MT /std:c++latest /permissive- /fp:strict" diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 20ac76cad86..789f7dbf4e0 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// REQUIRES: asan, x64 || x86 +// REQUIRES: x64 || x86 #pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension @@ -176,12 +176,11 @@ bool verify_string(const basic_string, Alloc>& s aligned = {buffer, buf_end}; } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); - assert(aligned._First); - assert(aligned._End); + assert(aligned._First != aligned._End); } const void* const mid = str.data() + str.size() + 1; - const void* const fixed_mid = aligned._Clamp(mid); + const void* const fixed_mid = aligned._Clamp_to_end(mid); return __sanitizer_verify_contiguous_container(aligned._First, fixed_mid, aligned._End) != 0; #else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp index 6bc9af7f29a..f46e271647b 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp @@ -162,13 +162,13 @@ bool verify_vector(vector& vec) { aligned = {buffer, buf_end}; } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); - if (!aligned._First) { + if (aligned._First == aligned._End) { return true; } } const void* const mid = vec.data() + vec.size(); - const void* const fixed_mid = aligned._Clamp(mid); + const void* const fixed_mid = aligned._Clamp_to_end(mid); void* bad_address = __sanitizer_contiguous_container_find_bad_address(aligned._First, fixed_mid, aligned._End); if (bad_address == nullptr) { From 42ef1a5d5c4599de72554cb2aeacd22737a4e381 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 2 Nov 2022 09:35:28 -0700 Subject: [PATCH 18/30] blampley blample more example --- stl/inc/xmemory | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/stl/inc/xmemory b/stl/inc/xmemory index f0fd32a1582..6f998fc98a7 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -793,11 +793,17 @@ struct _AsanAlignedPointers { // The way that ASan shadow memory works, each eight byte block of memory ("shadow memory section") // has a single byte to mark it as either poison or valid. // Each section has 0 to 8 "valid" bytes followed by poison bytes, so: +// ``` // [ v v v p p p p p ] +// ``` // or +// ``` // [ v v v v v v v v ] +// ``` // are okay, but +// ``` // [ p p p p v v v v ] +// ``` // is not. // // This function exists to fix up `first` and `end` pointers so that one can call @@ -825,6 +831,23 @@ struct _AsanAlignedPointers { // // First, `meow` and `purr` are just regular data members, not container buffers, so they _must_ be valid. // Then, assume we want to poison all of `buffer`. +// This would mean that, in a perfect world, we want something like: +// +// ``` +// | meow | buffer | purr | +// [ v v v v p p p p ][ p p p p p p p p ][ p p p p v v v v ] +// sm1 sm2 sm3 +// ``` +// +// However, note that by the rules above, `sm3` is not a valid shadow memory section; we always need +// the valid bytes to come before the poison bytes. Thus, the closest we can actually get to it is: +// +// ``` +// | meow | buffer | purr | +// [ v v v v p p p p ][ p p p p p p p p ][ v v v v v v v v ] +// sm1 sm2 sm3 +// ``` +// // We call `aligned = _Get_asan_aligned_first_end(cat.buffer, cat.buffer + 16);`, and we get back // // ```cpp @@ -844,8 +867,9 @@ struct _AsanAlignedPointers { // aligned._Clamp_to_end(cat.buffer + 16)); // ``` // -// We are allowed to assume that `aligned._First` is valid, since otherwise `cat.buffer + [0, 4)` could not be valid. -// We cannot poison up to `cat.buffer + 16`, since then `purr` could not be valid. +// We are allowed to assume that `&cat.meow` is valid, since otherwise `cat.buffer + [0, 4)` could not be valid. +// We cannot poison up to `cat.buffer + 16`, since then `&purr` could not be valid. +// Thus, this results in the shadow memory state from the second example. _NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_end( const void* const _First, const void* const _Last) noexcept { static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); From 9f503ca22bcc32cd885184ea754ec4223759c431 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 2 Nov 2022 13:53:59 -0700 Subject: [PATCH 19/30] remove noexcept from asan function additionally, split it out into its own header; hopefully the rest of the STL team is okay with that :P --- stl/CMakeLists.txt | 1 + .../__msvc_sanitizer_annotate_container.hpp | 117 ++++++++++++++++++ stl/inc/header-units.json | 1 + stl/inc/vector | 70 +---------- stl/inc/xstring | 71 ++--------- 5 files changed, 131 insertions(+), 129 deletions(-) create mode 100644 stl/inc/__msvc_sanitizer_annotate_container.hpp diff --git a/stl/CMakeLists.txt b/stl/CMakeLists.txt index ce5a5d44cec..616b52ae1d7 100644 --- a/stl/CMakeLists.txt +++ b/stl/CMakeLists.txt @@ -7,6 +7,7 @@ set(HEADERS ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_format_ucd_tables.hpp ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_int128.hpp ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_iter_core.hpp + ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_sanitizer_annotate_container.hpp ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_system_error_abi.hpp ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_tzdb.hpp ${CMAKE_CURRENT_LIST_DIR}/inc/__msvc_xlocinfo_types.hpp diff --git a/stl/inc/__msvc_sanitizer_annotate_container.hpp b/stl/inc/__msvc_sanitizer_annotate_container.hpp new file mode 100644 index 00000000000..13083751c13 --- /dev/null +++ b/stl/inc/__msvc_sanitizer_annotate_container.hpp @@ -0,0 +1,117 @@ +// __msvc_sanitizer_annotate_container.hpp internal header + +// Copyright (c) Microsoft Corporation. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#pragma once + +#if !defined(_M_CEE_PURE) && !(defined(_DISABLE_VECTOR_ANNOTATION) && defined(_DISABLE_STRING_ANNOTATION)) + +#if defined(__SANITIZE_ADDRESS__) + +#define _ACTIVATE_VECTOR_ANNOTATION +#define _INSERT_VECTOR_ANNOTATION +#define _ACTIVATE_STRING_ANNOTATION +#define _INSERT_STRING_ANNOTATION + +#elif defined(__clang__) && defined(__has_feature) // ^^^ __SANITIZE_ADDRESS__ ^^^ // vvv __clang__ vvv + +#if __has_feature(address_sanitizer) +#define _ACTIVATE_VECTOR_ANNOTATION +#define _INSERT_VECTOR_ANNOTATION +#define _ACTIVATE_STRING_ANNOTATION +#define _INSERT_STRING_ANNOTATION +#pragma comment(linker, "/INFERASANLIBS") +#endif // __has_feature(address_sanitizer) + +#else // ^^^ __clang__ ^^^ // vvv !__clang__ && !__SANITIZE_ADDRESS__ vvv + +#ifdef _ANNOTATE_VECTOR +#define _INSERT_VECTOR_ANNOTATION +#endif +#ifdef _ANNOTATE_STRING +#define _INSERT_STRING_ANNOTATION +#endif // _ANNOTATE_VECTOR + +#endif // __SANITIZE_ADDRESS__ + +#if defined(_DISABLE_VECTOR_ANNOTATION) +#undef _ACTIVATE_VECTOR_ANNOTATION +#undef _INSERT_VECTOR_ANNOTATION +#endif +#if defined(_DISABLE_STRING_ANNOTATION) +#undef _ACTIVATE_STRING_ANNOTATION +#undef _INSERT_STRING_ANNOTATION +#endif + +#if !defined(_INSERT_VECTOR_ANNOTATION) +#pragma detect_mismatch("annotate_vector", "0") +#endif // !_INSERT_VECTOR_ANNOTATION +#if !defined(_INSERT_STRING_ANNOTATION) +#pragma detect_mismatch("annotate_string", "0") +#endif // !_INSERT_STRING_ANNOTATION + +#ifdef _ACTIVATE_VECTOR_ANNOTATION +#pragma comment(lib, "stl_asan") +#pragma detect_mismatch("annotate_vector", "1") +#endif // _ACTIVATE_VECTOR_ANNOTATION +#ifdef _ACTIVATE_STRING_ANNOTATION +#pragma comment(lib, "stl_asan") +#pragma detect_mismatch("annotate_string", "1") +#endif // _ACTIVATE_STRING_ANNOTATION + +// only _INSERT_*_ANNOTATION should be used from here on +#undef _ACTIVATE_STRING_ANNOTATION +#undef _ACTIVATE_VECTOR_ANNOTATION + +extern "C" { +#ifdef _INSERT_VECTOR_ANNOTATION +extern const bool _Asan_vector_should_annotate; +#endif + +#ifdef _INSERT_STRING_ANNOTATION +extern const bool _Asan_string_should_annotate; +#endif +} + +#if defined(_INSERT_VECTOR_ANNOTATION) || defined(_INSERT_STRING_ANNOTATION) +extern "C" { +void __cdecl __sanitizer_annotate_contiguous_container( + const void* _First, const void* _End, const void* _Old_last, const void* _New_last); +} + +#if defined(_M_ARM64EC) +#pragma comment(linker, \ + "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") +#pragma comment(linker, \ + "/alternatename:__sanitizer_annotate_contiguous_container=__sanitizer_annotate_contiguous_container_default") +#pragma comment(linker, "/alternatename:#_Asan_vector_should_annotate=#_Asan_vector_should_annotate_default") +#pragma comment(linker, "/alternatename:_Asan_vector_should_annotate=_Asan_vector_should_annotate_default") +#pragma comment(linker, "/alternatename:#_Asan_string_should_annotate=#_Asan_string_should_annotate_default") +#pragma comment(linker, "/alternatename:_Asan_string_should_annotate=_Asan_string_should_annotate_default") +#elif defined(_M_HYBRID) +#pragma comment(linker, \ + "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") +#pragma comment(linker, \ + "/alternatename:___sanitizer_annotate_contiguous_container=___sanitizer_annotate_contiguous_container_default") +#pragma comment(linker, "/alternatename:#_Asan_vector_should_annotate=#_Asan_vector_should_annotate_default") +#pragma comment(linker, "/alternatename:__Asan_vector_should_annotate=__Asan_vector_should_annotate_default") +#pragma comment(linker, "/alternatename:#_Asan_string_should_annotate=#_Asan_string_should_annotate_default") +#pragma comment(linker, "/alternatename:__Asan_string_should_annotate=__Asan_string_should_annotate_default") +#elif defined(_M_IX86) +#pragma comment(linker, \ + "/alternatename:___sanitizer_annotate_contiguous_container=___sanitizer_annotate_contiguous_container_default") +#pragma comment(linker, "/alternatename:__Asan_vector_should_annotate=__Asan_vector_should_annotate_default") +#pragma comment(linker, "/alternatename:__Asan_string_should_annotate=__Asan_string_should_annotate_default") +#elif defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64) +#pragma comment(linker, \ + "/alternatename:__sanitizer_annotate_contiguous_container=__sanitizer_annotate_contiguous_container_default") +#pragma comment(linker, "/alternatename:_Asan_vector_should_annotate=_Asan_vector_should_annotate_default") +#pragma comment(linker, "/alternatename:_Asan_string_should_annotate=_Asan_string_should_annotate_default") +#else // ^^^ known architecture / unknown architecture vvv +#error Unknown architecture +#endif // ^^^ unknown architecture ^^^ + +#endif // insert asan annotations + +#endif // !_M_CEE_PURE && asan not disabled diff --git a/stl/inc/header-units.json b/stl/inc/header-units.json index da7529b5105..fa4079f7805 100644 --- a/stl/inc/header-units.json +++ b/stl/inc/header-units.json @@ -9,6 +9,7 @@ "__msvc_format_ucd_tables.hpp", "__msvc_int128.hpp", "__msvc_iter_core.hpp", + "__msvc_sanitizer_annotate_container.hpp", "__msvc_system_error_abi.hpp", "__msvc_tzdb.hpp", "__msvc_xlocinfo_types.hpp", diff --git a/stl/inc/vector b/stl/inc/vector index e71dc28812f..f52b36a67ff 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -14,6 +14,8 @@ #include #endif // _HAS_CXX17 +#include <__msvc_sanitizer_annotate_container.hpp> + #pragma pack(push, _CRT_PACKING) #pragma warning(push, _STL_WARNING_LEVEL) #pragma warning(disable : _STL_DISABLED_WARNINGS) @@ -426,70 +428,6 @@ constexpr _Ty* _Unfancy_maybe_null(_Ty* _Ptr) noexcept { // do nothing for plain return _Ptr; } -#if !defined(_M_CEE_PURE) && !defined(_DISABLE_VECTOR_ANNOTATION) -#if defined(__SANITIZE_ADDRESS__) -#define _ACTIVATE_VECTOR_ANNOTATION -#define _INSERT_VECTOR_ANNOTATION -#elif defined(__clang__) && defined(__has_feature) // ^^^ __SANITIZE_ADDRESS__ ^^^ // vvv __clang__ vvv -#if __has_feature(address_sanitizer) -#define _ACTIVATE_VECTOR_ANNOTATION -#define _INSERT_VECTOR_ANNOTATION -#pragma comment(linker, "/INFERASANLIBS") -#endif // __has_feature(address_sanitizer) -#elif defined(_ANNOTATE_VECTOR) // ^^^ __clang__ ^^^ // vvv _ANNOTATE_VECTOR vvv -#define _INSERT_VECTOR_ANNOTATION -#endif // _ANNOTATE_VECTOR -#endif // !_M_CEE_PURE && !_DISABLE_VECTOR_ANNOTATION - -#ifdef _ACTIVATE_VECTOR_ANNOTATION -#pragma comment(lib, "stl_asan") -#pragma detect_mismatch("annotate_vector", "1") -#endif // _ACTIVATE_VECTOR_ANNOTATION - -#ifdef _INSERT_VECTOR_ANNOTATION -extern "C" { -void __cdecl __sanitizer_annotate_contiguous_container( - const void* _First, const void* _End, const void* _Old_last, const void* _New_last) noexcept; -extern const bool _Asan_vector_should_annotate; -} - -#if defined(_M_ARM64EC) -#pragma comment(linker, \ - "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, \ - "/alternatename:__sanitizer_annotate_contiguous_container=__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:#_Asan_vector_should_annotate=#_Asan_vector_should_annotate_default") -#pragma comment(linker, "/alternatename:_Asan_vector_should_annotate=_Asan_vector_should_annotate_default") -#elif defined(_M_HYBRID) -#pragma comment(linker, \ - "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, \ - "/alternatename:___sanitizer_annotate_contiguous_container=___sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:#_Asan_vector_should_annotate=#_Asan_vector_should_annotate_default") -#pragma comment(linker, "/alternatename:__Asan_vector_should_annotate=__Asan_vector_should_annotate_default") -#elif defined(_M_IX86) -#pragma comment(linker, \ - "/alternatename:___sanitizer_annotate_contiguous_container=___sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:__Asan_vector_should_annotate=__Asan_vector_should_annotate_default") -#elif defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64) -#pragma comment(linker, \ - "/alternatename:__sanitizer_annotate_contiguous_container=__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:_Asan_vector_should_annotate=_Asan_vector_should_annotate_default") -#else // ^^^ known architecture / unknown architecture vvv -#error Unknown architecture -#endif // ^^^ unknown architecture ^^^ - -template -_INLINE_VAR constexpr bool _Has_minimum_allocation_alignment = alignof(typename _Vec::value_type) >= _Asan_granularity; - -template -_INLINE_VAR constexpr bool - _Has_minimum_allocation_alignment<_Vec, void_t> = - _Vec::allocator_type::_Minimum_allocation_alignment >= _Asan_granularity; -#else // ^^^ _INSERT_VECTOR_ANNOTATION ^^^ // vvv !_INSERT_VECTOR_ANNOTATION vvv -#pragma detect_mismatch("annotate_vector", "0") -#endif // !_INSERT_VECTOR_ANNOTATION - _EXPORT_STD template > class vector { // varying size array of values private: @@ -567,7 +505,7 @@ private: // new state: // [_First, _New_last) valid // [_New_last, _End) poison - __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + _CSTD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); if (_Aligned._First == _Aligned._End) { @@ -586,7 +524,7 @@ private: // [_Aligned._First, _New_fixed) valid // [_New_fixed, _Aligned._End) poison // [_Aligned._End, _End) valid - __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); + _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } } diff --git a/stl/inc/xstring b/stl/inc/xstring index 5fe1cd37135..ff6d3558495 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -17,6 +17,8 @@ #include #endif // _HAS_CXX17 +#include <__msvc_sanitizer_annotate_container.hpp> + #pragma pack(push, _CRT_PACKING) #pragma warning(push, _STL_WARNING_LEVEL) #pragma warning(disable : _STL_DISABLED_WARNINGS) @@ -2366,63 +2368,6 @@ struct _String_constructor_rvalue_allocator_tag { _Xlength_error("string too long"); } -#if !defined(_M_CEE_PURE) && !defined(_DISABLE_STRING_ANNOTATION) -#if defined(__SANITIZE_ADDRESS__) -#define _ACTIVATE_STRING_ANNOTATION -#define _INSERT_STRING_ANNOTATION -#elif defined(__clang__) && defined(__has_feature) // ^^^ __SANITIZE_ADDRESS__ ^^^ // vvv __clang__ vvv -#if __has_feature(address_sanitizer) -#define _ACTIVATE_STRING_ANNOTATION -#define _INSERT_STRING_ANNOTATION -#pragma comment(linker, "/INFERASANLIBS") -#endif // __has_feature(address_sanitizer) -#elif defined(_ANNOTATE_STRING) // ^^^ __clang__ ^^^ // vvv _ANNOTATE_STRING vvv -#define _INSERT_STRING_ANNOTATION -#endif // _ANNOTATE_STRING -#endif // !_M_CEE_PURE && !_DISABLE_STRING_ANNOTATION - -#ifdef _ACTIVATE_STRING_ANNOTATION -#pragma comment(lib, "stl_asan") -#pragma detect_mismatch("annotate_string", "1") -#endif // _ACTIVATE_STRING_ANNOTATION - -#ifdef _INSERT_STRING_ANNOTATION -extern "C" { -void __cdecl __sanitizer_annotate_contiguous_container( - const void* _First, const void* _End, const void* _Old_last, const void* _New_last) noexcept; -extern const bool _Asan_string_should_annotate; -} - -#if defined(_M_ARM64EC) -#pragma comment(linker, \ - "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, \ - "/alternatename:__sanitizer_annotate_contiguous_container=__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:#_Asan_string_should_annotate=#_Asan_string_should_annotate_default") -#pragma comment(linker, "/alternatename:_Asan_string_should_annotate=_Asan_string_should_annotate_default") -#elif defined(_M_HYBRID) -#pragma comment(linker, \ - "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, \ - "/alternatename:___sanitizer_annotate_contiguous_container=___sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:#_Asan_string_should_annotate=#_Asan_string_should_annotate_default") -#pragma comment(linker, "/alternatename:__Asan_string_should_annotate=__Asan_string_should_annotate_default") -#elif defined(_M_IX86) -#pragma comment(linker, \ - "/alternatename:___sanitizer_annotate_contiguous_container=___sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:__Asan_string_should_annotate=__Asan_string_should_annotate_default") -#elif defined(_M_X64) || defined(_M_ARM) || defined(_M_ARM64) -#pragma comment(linker, \ - "/alternatename:__sanitizer_annotate_contiguous_container=__sanitizer_annotate_contiguous_container_default") -#pragma comment(linker, "/alternatename:_Asan_string_should_annotate=_Asan_string_should_annotate_default") -#else // ^^^ known architecture / unknown architecture vvv -#error Unknown architecture -#endif // ^^^ unknown architecture ^^^ - -#else // ^^^ _INSERT_STRING_ANNOTATION ^^^ // vvv !_INSERT_STRING_ANNOTATION vvv -#pragma detect_mismatch("annotate_string", "0") -#endif // !_INSERT_STRING_ANNOTATION - #if _HAS_CXX23 && defined(__cpp_lib_concepts) // TRANSITION, GH-395 template concept _Contiguous_range_of = @@ -2547,7 +2492,7 @@ private: // [_My_buf, _End) unknown // new state: // [_My_buf, _End) valid - _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); + _CSTD __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); // old state: @@ -2556,7 +2501,7 @@ private: // [_Aligned._End, _End) valid // new state: // [_Aligned._First, _End) valid - _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _My_buf, _Aligned._End); + _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _My_buf, _Aligned._End); } } @@ -2580,7 +2525,7 @@ private: // new state: // [_My_buf, _Ptr_last) valid // [_Ptr_last, _End) poison - _STD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); + _CSTD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); // old state: @@ -2589,7 +2534,7 @@ private: // [_Aligned._First, _Ptr_last) valid // [_Ptr_last, _Aligned._End) poison // [_Aligned._End, _End) valid - _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Aligned._End, _Ptr_last); + _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Aligned._End, _Ptr_last); } } @@ -2619,7 +2564,7 @@ private: // new state: // [_First, _New_last) valid // [_New_last, _End) poison - _STD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + _CSTD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); // The buffer must always have size >= 9 bytes, so it will always end at least one shadow memory section. @@ -2636,7 +2581,7 @@ private: // [_Aligned._First, _New_fixed) valid // [_New_fixed, _Aligned._End) poison // [_Aligned._End, _End) valid - _STD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); + _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } } From f80f1665dba6f1aea38a42ef6c2f8f68a2968848 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Fri, 4 Nov 2022 11:30:30 -0700 Subject: [PATCH 20/30] Amy CRs; force strings to be aligned on 4 --- stl/inc/vector | 5 +- stl/inc/xmemory | 13 +++- stl/inc/xstring | 19 ++++-- .../GH_002030_asan_annotate_string/test.cpp | 68 +++++++++++++++++-- 4 files changed, 87 insertions(+), 18 deletions(-) diff --git a/stl/inc/vector b/stl/inc/vector index f52b36a67ff..b8c96def688 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -504,8 +504,9 @@ private: // [_Old_last, _End) poison // new state: // [_First, _New_last) valid - // [_New_last, _End) poison - _CSTD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + // [_New_last, asan_aligned_after(_End)) poison + _CSTD __sanitizer_annotate_contiguous_container( + _First, _STD _Get_asan_aligned_after(_End), _Old_last, _New_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); if (_Aligned._First == _Aligned._End) { diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 6f998fc98a7..4aa0b6484cd 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -871,15 +871,22 @@ struct _AsanAlignedPointers { // We cannot poison up to `cat.buffer + 16`, since then `&purr` could not be valid. // Thus, this results in the shadow memory state from the second example. _NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_end( - const void* const _First, const void* const _Last) noexcept { + const void* const _First, const void* const _End) noexcept { static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); - return { reinterpret_cast(reinterpret_cast(_First) & _Mask), - reinterpret_cast(reinterpret_cast(_Last) & _Mask), + reinterpret_cast(reinterpret_cast(_End) & _Mask), }; } +// When we can assume that the allocator we are using will always align allocations to the 8-byte, +// we can simply push the `_End` pointer to the end of the shadow memory section. +// This is _not_ safe in general (see _Get_asan_aligned_first_end's comment for why). +_NODISCARD inline const void* _Get_asan_aligned_after(const void* const _End) noexcept { + static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); + return reinterpret_cast((reinterpret_cast(_End) + _Asan_granularity - 1) & _Mask); +} + template _INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment = alignof(typename _Container::value_type); diff --git a/stl/inc/xstring b/stl/inc/xstring index ff6d3558495..52e5e10edbe 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2492,7 +2492,9 @@ private: // [_My_buf, _End) unknown // new state: // [_My_buf, _End) valid - _CSTD __sanitizer_annotate_contiguous_container(_My_buf, _End, _My_buf, _End); + // [_End, asan_aligned_after(_End)) poison + _CSTD __sanitizer_annotate_contiguous_container( + _My_buf, _STD _Get_asan_aligned_after(_End), _My_buf, _End); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); // old state: @@ -2524,8 +2526,9 @@ private: // [_My_buf, _End) valid (from _Remove_sso_annotation) // new state: // [_My_buf, _Ptr_last) valid - // [_Ptr_last, _End) poison - _CSTD __sanitizer_annotate_contiguous_container(_My_buf, _End, _End, _Ptr_last); + // [_Ptr_last, asan_aligned_after(_End)) poison + _CSTD __sanitizer_annotate_contiguous_container( + _My_buf, _STD _Get_asan_aligned_after(_End), _End, _Ptr_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); // old state: @@ -2563,8 +2566,9 @@ private: // [_Old_last, _End) poison // new state: // [_First, _New_last) valid - // [_New_last, _End) poison - _CSTD __sanitizer_annotate_contiguous_container(_First, _End, _Old_last, _New_last); + // [_New_last, asan_aligned_after(_End)) poison + _CSTD __sanitizer_annotate_contiguous_container( + _First, _STD _Get_asan_aligned_after(_End), _Old_last, _New_last); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); // The buffer must always have size >= 9 bytes, so it will always end at least one shadow memory section. @@ -3937,8 +3941,9 @@ public: const size_type _Old_size = _Mypair._Myval2._Mysize; if (_Count < _Nx || _Count - _Nx <= _Mypair._Myval2._Myres - _Old_size) { // either we are shrinking, or the growth fits - _Mypair._Myval2._Mysize = _Old_size + _Count - _Nx; // may temporarily overflow; - // OK because size_type must be unsigned + // may temporarily overflow; OK because size_type must be unsigned + _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Count - _Nx); + _Mypair._Myval2._Mysize = _Old_size + _Count - _Nx; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Elem* const _Insert_at = _Old_ptr + _Off; _Traits::move(_Insert_at + _Count, _Insert_at + _Nx, _Old_size - _Nx - _Off + 1); diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 789f7dbf4e0..3d1cc7fdd69 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -4,6 +4,8 @@ // REQUIRES: x64 || x86 #pragma warning(disable : 4984) // 'if constexpr' is a C++17 language extension +#pragma warning(disable : 4324) // '%s': structure was padded due to alignment specifier +#pragma warning(disable : 4365) // '%s': conversion from '%s' to '%s', signed/unsigned mismatch #ifdef __clang__ #pragma clang diagnostic ignored "-Wc++17-extensions" // constexpr if is a C++17 extension @@ -38,6 +40,60 @@ constexpr auto literal_input_u16 = u"Hello fluffy kittens"; constexpr auto literal_input_u32 = U"Hello fluffy kittens"; constexpr auto literal_input_w = L"Hello fluffy kittens"; +struct FourByte { + int padding = 0; +}; + +#if _HAS_CXX17 +template +struct alignas(8) AlignOnFour : FourByte, T { + using T::T; + using T::operator=; + + T& base() & { + return *this; + } + const T& base() const& { + return *this; + } + T&& base() && { + return move(*this); + } + const T&& base() const&& { + return move(*this); + } + + template + friend auto operator+(T1&& t1, T2&& t2) + -> enable_if_t, AlignOnFour> || is_same_v, AlignOnFour>, AlignOnFour> { + return AlignOnFour(InternalConstructTag{}, [&] { + if constexpr (is_same_v, AlignOnFour> && is_same_v, AlignOnFour>) { + return forward(t1).base() + forward(t2).base(); + } else if constexpr (is_same_v, AlignOnFour>) { + return forward(t1).base() + forward(t2); + } else { + return forward(t1) + forward(t2).base(); + } + }); + } + + template + AlignOnFour substr(Ts&&... ts) const { + return AlignOnFour(InternalConstructTag{}, [&] { return base().substr(forward(ts)...); }); + } + +private: + struct InternalConstructTag {}; + + template + AlignOnFour(InternalConstructTag, F f) : T(f()) {} +}; +#else // ^^^ no workaround / workaround vvv +// in C++14, you can't do the `using T::T;` thing and also define your own ctor +template +using AlignOnFour = T; +#endif + template constexpr auto get_large_input() { if constexpr (is_same_v) { @@ -263,7 +319,7 @@ struct implicit_allocator : public custom_test_allocator void test_construction() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = AlignOnFour, Alloc>>; { // constructors // range constructors str literal_constructed_sso{get_sso_input()}; @@ -516,7 +572,7 @@ void test_construction() { template void test_append() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = AlignOnFour, Alloc>>; constexpr size_t large_size = 20; constexpr size_t sso_size = 1; @@ -858,7 +914,7 @@ void test_append() { template void test_assign() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = AlignOnFour, Alloc>>; constexpr size_t large_size = 20; constexpr size_t sso_size = 2; @@ -1189,7 +1245,7 @@ void test_assign() { template void test_insertion() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = AlignOnFour, Alloc>>; constexpr size_t large_size = 20; constexpr size_t sso_size = 1; @@ -1371,7 +1427,7 @@ void test_insertion() { template void test_removal() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = AlignOnFour, Alloc>>; constexpr size_t large_size = 20; constexpr size_t sso_size = 2; @@ -1507,7 +1563,7 @@ void test_removal() { template void test_misc() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = AlignOnFour, Alloc>>; constexpr size_t large_size = 20; constexpr size_t sso_size = 2; From 1641a6ef19863fe0045c3e941c554cf4e20c4d32 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Fri, 4 Nov 2022 11:59:54 -0700 Subject: [PATCH 21/30] I dunno why this wasn't formatted --- stl/inc/xstring | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 52e5e10edbe..aedd3163784 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2493,8 +2493,7 @@ private: // new state: // [_My_buf, _End) valid // [_End, asan_aligned_after(_End)) poison - _CSTD __sanitizer_annotate_contiguous_container( - _My_buf, _STD _Get_asan_aligned_after(_End), _My_buf, _End); + _CSTD __sanitizer_annotate_contiguous_container(_My_buf, _STD _Get_asan_aligned_after(_End), _My_buf, _End); } else { const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); // old state: From f12db09d658fdbf3e947d835aa32b7b551e02bd0 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Fri, 4 Nov 2022 12:45:11 -0700 Subject: [PATCH 22/30] Stephan CRs --- .../__msvc_sanitizer_annotate_container.hpp | 54 ++++++++++++------- stl/inc/vector | 2 +- stl/inc/xmemory | 13 ++--- stl/inc/xstring | 26 +++++++-- .../GH_002030_asan_annotate_string/env.lst | 2 +- .../GH_002030_asan_annotate_string/test.cpp | 26 +++++++-- .../GH_002030_asan_annotate_vector/env.lst | 2 +- .../GH_002030_asan_annotate_vector/test.cpp | 7 ++- 8 files changed, 92 insertions(+), 40 deletions(-) diff --git a/stl/inc/__msvc_sanitizer_annotate_container.hpp b/stl/inc/__msvc_sanitizer_annotate_container.hpp index 13083751c13..5f7b11d5c8e 100644 --- a/stl/inc/__msvc_sanitizer_annotate_container.hpp +++ b/stl/inc/__msvc_sanitizer_annotate_container.hpp @@ -4,61 +4,72 @@ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception #pragma once +#ifndef __MSVC_SANITIZER_ANNOTATE_CONTAINER_HPP +#define __MSVC_SANITIZER_ANNOTATE_CONTAINER_HPP +#include +#if _STL_COMPILER_PREPROCESSOR -#if !defined(_M_CEE_PURE) && !(defined(_DISABLE_VECTOR_ANNOTATION) && defined(_DISABLE_STRING_ANNOTATION)) +#pragma pack(push, _CRT_PACKING) +#pragma warning(push, _STL_WARNING_LEVEL) +#pragma warning(disable : _STL_DISABLED_WARNINGS) +_STL_DISABLE_CLANG_WARNINGS +#pragma push_macro("new") +#undef new + +#if !defined(_M_CEE_PURE) && !(defined(_DISABLE_STRING_ANNOTATION) && defined(_DISABLE_VECTOR_ANNOTATION)) #if defined(__SANITIZE_ADDRESS__) -#define _ACTIVATE_VECTOR_ANNOTATION -#define _INSERT_VECTOR_ANNOTATION #define _ACTIVATE_STRING_ANNOTATION #define _INSERT_STRING_ANNOTATION +#define _ACTIVATE_VECTOR_ANNOTATION +#define _INSERT_VECTOR_ANNOTATION #elif defined(__clang__) && defined(__has_feature) // ^^^ __SANITIZE_ADDRESS__ ^^^ // vvv __clang__ vvv #if __has_feature(address_sanitizer) -#define _ACTIVATE_VECTOR_ANNOTATION -#define _INSERT_VECTOR_ANNOTATION #define _ACTIVATE_STRING_ANNOTATION #define _INSERT_STRING_ANNOTATION +#define _ACTIVATE_VECTOR_ANNOTATION +#define _INSERT_VECTOR_ANNOTATION #pragma comment(linker, "/INFERASANLIBS") #endif // __has_feature(address_sanitizer) #else // ^^^ __clang__ ^^^ // vvv !__clang__ && !__SANITIZE_ADDRESS__ vvv -#ifdef _ANNOTATE_VECTOR -#define _INSERT_VECTOR_ANNOTATION -#endif #ifdef _ANNOTATE_STRING #define _INSERT_STRING_ANNOTATION +#endif // _ANNOTATE_STRING +#ifdef _ANNOTATE_VECTOR +#define _INSERT_VECTOR_ANNOTATION #endif // _ANNOTATE_VECTOR #endif // __SANITIZE_ADDRESS__ -#if defined(_DISABLE_VECTOR_ANNOTATION) -#undef _ACTIVATE_VECTOR_ANNOTATION -#undef _INSERT_VECTOR_ANNOTATION -#endif #if defined(_DISABLE_STRING_ANNOTATION) #undef _ACTIVATE_STRING_ANNOTATION #undef _INSERT_STRING_ANNOTATION -#endif +#endif // _DISABLE_STRING_ANNOTATION +#if defined(_DISABLE_VECTOR_ANNOTATION) +#undef _ACTIVATE_VECTOR_ANNOTATION +#undef _INSERT_VECTOR_ANNOTATION +#endif // _DISABLE_VECTOR_ANNOTATION -#if !defined(_INSERT_VECTOR_ANNOTATION) -#pragma detect_mismatch("annotate_vector", "0") -#endif // !_INSERT_VECTOR_ANNOTATION #if !defined(_INSERT_STRING_ANNOTATION) #pragma detect_mismatch("annotate_string", "0") #endif // !_INSERT_STRING_ANNOTATION +#if !defined(_INSERT_VECTOR_ANNOTATION) +#pragma detect_mismatch("annotate_vector", "0") +#endif // !_INSERT_VECTOR_ANNOTATION -#ifdef _ACTIVATE_VECTOR_ANNOTATION -#pragma comment(lib, "stl_asan") -#pragma detect_mismatch("annotate_vector", "1") -#endif // _ACTIVATE_VECTOR_ANNOTATION #ifdef _ACTIVATE_STRING_ANNOTATION #pragma comment(lib, "stl_asan") #pragma detect_mismatch("annotate_string", "1") #endif // _ACTIVATE_STRING_ANNOTATION +#ifdef _ACTIVATE_VECTOR_ANNOTATION +#pragma comment(lib, "stl_asan") +#pragma detect_mismatch("annotate_vector", "1") +#endif // _ACTIVATE_VECTOR_ANNOTATION // only _INSERT_*_ANNOTATION should be used from here on #undef _ACTIVATE_STRING_ANNOTATION @@ -115,3 +126,6 @@ void __cdecl __sanitizer_annotate_contiguous_container( #endif // insert asan annotations #endif // !_M_CEE_PURE && asan not disabled + +#endif // _STL_COMPILER_PREPROCESSOR +#endif // __MSVC_SANITIZER_ANNOTATE_CONTAINER_HPP diff --git a/stl/inc/vector b/stl/inc/vector index b8c96def688..9d7560fa6fa 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -498,7 +498,7 @@ private: const void* const _End = _STD _Unfancy(_End_); const void* const _Old_last = _STD _Unfancy(_Old_last_); const void* const _New_last = _STD _Unfancy(_New_last_); - if constexpr ((_Container_allocation_minimum_alignment) > _Asan_granularity) { + if constexpr ((_Container_allocation_minimum_alignment) >= _Asan_granularity) { // old state: // [_First, _Old_last) valid // [_Old_last, _End) poison diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 4aa0b6484cd..5c2166d3af6 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -776,12 +776,12 @@ _NODISCARD constexpr allocation_result::pointe // The number of user bytes a single byte of ASAN shadow memory can track. _INLINE_VAR constexpr size_t _Asan_granularity = 8; -struct _AsanAlignedPointers { +struct _Asan_aligned_pointers { const void* _First; const void* _End; _NODISCARD constexpr const void* _Clamp_to_end(const void* _Mid) const noexcept { - _STL_ASSERT(_Mid >= _First, ""); + _STL_INTERNAL_CHECK(_Mid >= _First); if (_Mid > _End) { return _End; } else { @@ -870,7 +870,7 @@ struct _AsanAlignedPointers { // We are allowed to assume that `&cat.meow` is valid, since otherwise `cat.buffer + [0, 4)` could not be valid. // We cannot poison up to `cat.buffer + 16`, since then `&purr` could not be valid. // Thus, this results in the shadow memory state from the second example. -_NODISCARD inline _AsanAlignedPointers _Get_asan_aligned_first_end( +_NODISCARD inline _Asan_aligned_pointers _Get_asan_aligned_first_end( const void* const _First, const void* const _End) noexcept { static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); return { @@ -892,8 +892,9 @@ _INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment = alignof(t template _INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment<_Container, - void_t> = - (_STD max)(alignof(typename _Container::value_type), _Container::allocator_type::_Minimum_allocation_alignment); + void_t> = + (_STD max)( + alignof(typename _Container::value_type), _Container::allocator_type::_Minimum_asan_allocation_alignment); _EXPORT_STD template class allocator { @@ -981,7 +982,7 @@ public: } #endif // _HAS_DEPRECATED_ALLOCATOR_MEMBERS - static constexpr size_t _Minimum_allocation_alignment = _Asan_granularity; + static constexpr size_t _Minimum_asan_allocation_alignment = _Asan_granularity; }; #if _HAS_DEPRECATED_ALLOCATOR_VOID || _HAS_DEPRECATED_ALLOCATOR_MEMBERS diff --git a/stl/inc/xstring b/stl/inc/xstring index aedd3163784..b46e934076c 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2451,8 +2451,21 @@ private: #ifdef _INSERT_STRING_ANNOTATION // this is a function so we can get the alignment of the class while defining it. _NODISCARD static constexpr bool _Small_string_always_asan_aligned() noexcept { - return alignof(basic_string) >= _Asan_granularity && _Memcpy_val_offset % _Asan_granularity == 0 - && (sizeof(value_type[_BUF_SIZE]) % _Asan_granularity == 0 || alignof(size_type) >= _Asan_granularity); + constexpr bool _Aligned_first = + // The offset of the buffer is a multiple of 8 bytes from the front of the object, and + _Memcpy_val_offset % _Asan_granularity == 0 + // the front of the object is always on an 8-byte boundary. + && alignof(basic_string) >= _Asan_granularity; + + constexpr bool _Aligned_end = + // The size of the buffer is a multiple of 8 bytes, or + sizeof(value_type[_BUF_SIZE]) % _Asan_granularity == 0 + // even though the size of the buffer is not a multiple of 8 bytes, + // _Mysize must be aligned on an 8-byte boundary, + // so there's padding between the end of _Bx._Buf and _Mysize. + || alignof(size_type) >= _Asan_granularity; + + return _Aligned_first && _Aligned_end; } _CONSTEXPR20 void _Create_annotation() const noexcept { @@ -2558,7 +2571,7 @@ private: const void* const _New_last = _First + _New_size + 1; constexpr bool _Large_string_always_asan_aligned = - (_Container_allocation_minimum_alignment) >= _Asan_granularity; + (_Container_allocation_minimum_asan_alignment) >= _Asan_granularity; if constexpr (_Large_string_always_asan_aligned && _Small_string_always_asan_aligned()) { // old state: // [_First, _Old_last) valid @@ -3870,8 +3883,10 @@ public: _Elem* const _Insert_at = _Old_ptr + _Off; _Traits::move(_Insert_at, _Ptr, _Count); _Traits::move(_Insert_at + _Count, _Insert_at + _Nx, _Suffix_size); - _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size - (_Nx - _Count)); - _Mypair._Myval2._Mysize = _Old_size - (_Nx - _Count); + + const auto _New_size = _Old_size - (_Nx - _Count); + _ASAN_STRING_MODIFY(*this, _Old_size, _New_size); + _Mypair._Myval2._Mysize = _New_size; return *this; } @@ -4460,6 +4475,7 @@ public: if (!_Right_large) { _ASAN_STRING_CREATE(*this); } + if (!_My_large) { _ASAN_STRING_CREATE(_Right); } diff --git a/tests/std/tests/GH_002030_asan_annotate_string/env.lst b/tests/std/tests/GH_002030_asan_annotate_string/env.lst index 60ca4d911b9..6a1706876f4 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_string/env.lst @@ -3,7 +3,7 @@ # This test matrix is the usual test matrix, with all currently unsupported options removed, crossed with the ASan flags. -# TRANSITION, VSO-1350252 - due to vcasan.lib including the standard library, we can't use it (pending new release). +# TRANSITION, VSO-1350252 - due to vcasan.lib including the standard library, we can't use it (pending 17.5 preview 2). # TRANSITION, google/sanitizers#328 - clang-cl does not currently support targeting /MDd or /MTd. RUNALL_INCLUDE ..\prefix.lst RUNALL_CROSSLIST diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 3d1cc7fdd69..0a26b7d3f71 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -225,10 +225,10 @@ template bool verify_string(const basic_string, Alloc>& str) { #ifdef __SANITIZE_ADDRESS__ const void* const buffer = str.data(); - const void* const buf_end = str.data() + (str.capacity() + 1); + const void* const buf_end = str.data() + str.capacity() + 1; - _AsanAlignedPointers aligned; - if constexpr ((_Container_allocation_minimum_alignment, Alloc>>) > 8) { + _Asan_aligned_pointers aligned; + if constexpr ((_Container_allocation_minimum_alignment>) >= 8) { aligned = {buffer, buf_end}; } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); @@ -267,7 +267,7 @@ constexpr bool operator!=( template struct aligned_allocator : public custom_test_allocator { - static constexpr size_t _Minimum_allocation_alignment = 8; + static constexpr size_t _Minimum_asan_allocation_alignment = 8; aligned_allocator() = default; template @@ -281,10 +281,15 @@ struct aligned_allocator : public custom_test_allocator, aligned_allocator>> == 8); +static_assert(_Container_allocation_minimum_asan_alignment< + basic_string, aligned_allocator>> + == 8); template struct explicit_allocator : public custom_test_allocator { - static constexpr size_t _Minimum_allocation_alignment = alignof(CharType); + static constexpr size_t _Minimum_asan_allocation_alignment = alignof(CharType); explicit_allocator() = default; template @@ -299,6 +304,11 @@ struct explicit_allocator : public custom_test_allocator, explicit_allocator>> == 1); +static_assert(_Container_allocation_minimum_asan_alignment< + basic_string, explicit_allocator>> + == 2); template struct implicit_allocator : public custom_test_allocator { @@ -315,6 +325,11 @@ struct implicit_allocator : public custom_test_allocator, implicit_allocator>> == 1); +static_assert(_Container_allocation_minimum_asan_alignment< + basic_string, implicit_allocator>> + == 2); template void test_construction() { @@ -1896,6 +1911,7 @@ void test_DevCom_10109507() { string s("abcd"); s.replace(0, 1, "ef", 2); s.replace(0, 0, "xy", 2); + assert(s == "xyefbcd"); } int main() { diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/env.lst b/tests/std/tests/GH_002030_asan_annotate_vector/env.lst index 9e11db811c8..ce99dfafa04 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/env.lst +++ b/tests/std/tests/GH_002030_asan_annotate_vector/env.lst @@ -3,7 +3,7 @@ # This test matrix is the usual test matrix, with all currently unsupported options removed, crossed with the ASan flags. -# TRANSITION, VSO-1350252 - due to vcasan.lib including the standard library, we can't use it (pending new release). +# TRANSITION, VSO-1350252 - due to vcasan.lib including the standard library, we can't use it (pending 17.5 preview 2). # TRANSITION, google/sanitizers#328 - clang-cl does not currently support targeting /MDd or /MTd. RUNALL_INCLUDE ..\prefix.lst RUNALL_CROSSLIST diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp index f46e271647b..b8788d444bc 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp @@ -158,7 +158,7 @@ bool verify_vector(vector& vec) { const void* buf_end = vec.data() + vec.capacity(); _AsanAlignedPointers aligned; - if constexpr ((_Container_allocation_minimum_alignment>) > 8) { + if constexpr ((_Container_allocation_minimum_alignment>) >= 8) { aligned = {buffer, buf_end}; } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); @@ -232,6 +232,7 @@ struct aligned_allocator : custom_test_allocator { delete[] p; } }; +static_assert(_Container_allocation_minimum_asan_alignment>> == 8); template struct explicit_allocator : custom_test_allocator { @@ -250,6 +251,8 @@ struct explicit_allocator : custom_test_allocator { delete[] (p - 1); } }; +static_assert(_Container_allocation_minimum_asan_alignment>> == 1); +static_assert(_Container_allocation_minimum_asan_alignment>> == 2); template struct implicit_allocator : custom_test_allocator { @@ -266,6 +269,8 @@ struct implicit_allocator : custom_test_allocator { delete[] (p - 1); } }; +static_assert(_Container_allocation_minimum_asan_alignment>> == 1); +static_assert(_Container_allocation_minimum_asan_alignment>> == 2); template void test_push_pop() { From 840cae15c9c5c2cbab02c96f93ab3689fa362c56 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Fri, 4 Nov 2022 14:30:46 -0700 Subject: [PATCH 23/30] misc cleanups --- .../__msvc_sanitizer_annotate_container.hpp | 5 ++ stl/inc/vector | 2 +- stl/inc/xmemory | 4 +- .../GH_002030_asan_annotate_string/test.cpp | 61 ++++++++++++++++--- .../GH_002030_asan_annotate_vector/test.cpp | 20 +++--- 5 files changed, 72 insertions(+), 20 deletions(-) diff --git a/stl/inc/__msvc_sanitizer_annotate_container.hpp b/stl/inc/__msvc_sanitizer_annotate_container.hpp index 5f7b11d5c8e..49b3510c7e9 100644 --- a/stl/inc/__msvc_sanitizer_annotate_container.hpp +++ b/stl/inc/__msvc_sanitizer_annotate_container.hpp @@ -127,5 +127,10 @@ void __cdecl __sanitizer_annotate_contiguous_container( #endif // !_M_CEE_PURE && asan not disabled +#pragma pop_macro("new") +_STL_RESTORE_CLANG_WARNINGS +#pragma warning(pop) +#pragma pack(pop) + #endif // _STL_COMPILER_PREPROCESSOR #endif // __MSVC_SANITIZER_ANNOTATE_CONTAINER_HPP diff --git a/stl/inc/vector b/stl/inc/vector index 9d7560fa6fa..ad2ced1cd48 100644 --- a/stl/inc/vector +++ b/stl/inc/vector @@ -498,7 +498,7 @@ private: const void* const _End = _STD _Unfancy(_End_); const void* const _Old_last = _STD _Unfancy(_Old_last_); const void* const _New_last = _STD _Unfancy(_New_last_); - if constexpr ((_Container_allocation_minimum_alignment) >= _Asan_granularity) { + if constexpr ((_Container_allocation_minimum_asan_alignment) >= _Asan_granularity) { // old state: // [_First, _Old_last) valid // [_Old_last, _End) poison diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 5c2166d3af6..91693198b77 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -888,10 +888,10 @@ _NODISCARD inline const void* _Get_asan_aligned_after(const void* const _End) no } template -_INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment = alignof(typename _Container::value_type); +_INLINE_VAR constexpr size_t _Container_allocation_minimum_asan_alignment = alignof(typename _Container::value_type); template -_INLINE_VAR constexpr size_t _Container_allocation_minimum_alignment<_Container, +_INLINE_VAR constexpr size_t _Container_allocation_minimum_asan_alignment<_Container, void_t> = (_STD max)( alignof(typename _Container::value_type), _Container::allocator_type::_Minimum_asan_allocation_alignment); diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 0a26b7d3f71..291e45c402a 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -28,6 +28,8 @@ using namespace std; +#define STATIC_ASSERT(...) static_assert(__VA_ARGS__, #__VA_ARGS__) + #ifdef __SANITIZE_ADDRESS__ extern "C" int __sanitizer_verify_contiguous_container(const void* beg, const void* mid, const void* end) noexcept; #endif // ASan instrumentation enabled @@ -160,7 +162,7 @@ template struct throw_on_conversion { throw_on_conversion() = default; throw_on_conversion(CharType) {} - operator const CharType() const { + operator CharType() const { throw 42; } }; @@ -228,7 +230,7 @@ bool verify_string(const basic_string, Alloc>& s const void* const buf_end = str.data() + str.capacity() + 1; _Asan_aligned_pointers aligned; - if constexpr ((_Container_allocation_minimum_alignment>) >= 8) { + if constexpr ((_Container_allocation_minimum_asan_alignment>) >= 8) { aligned = {buffer, buf_end}; } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); @@ -281,9 +283,9 @@ struct aligned_allocator : public custom_test_allocator, aligned_allocator>> == 8); -static_assert(_Container_allocation_minimum_asan_alignment< +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< basic_string, aligned_allocator>> == 8); @@ -304,9 +306,9 @@ struct explicit_allocator : public custom_test_allocator, explicit_allocator>> == 1); -static_assert(_Container_allocation_minimum_asan_alignment< +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< basic_string, explicit_allocator>> == 2); @@ -325,9 +327,9 @@ struct implicit_allocator : public custom_test_allocator, implicit_allocator>> == 1); -static_assert(_Container_allocation_minimum_asan_alignment< +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< basic_string, implicit_allocator>> == 2); @@ -1642,6 +1644,49 @@ void test_misc() { assert(verify_string(resize_char_sso_to_sso)); } + { // replace + const CharType mrow[] = {'m', 'r', 'o', 'w', '\0'}; + + str replace_front_bigger{input}; + replace_front_bigger.replace(0, 2, mrow); + assert(verify_string(replace_front_bigger)); + str replace_front_same{input}; + replace_front_same.replace(0, 4, mrow); + assert(verify_string(replace_front_same)); + str replace_front_smaller{input}; + replace_front_smaller.replace(0, 6, mrow); + assert(verify_string(replace_front_smaller)); + + str replace_mid_bigger{input}; + replace_mid_bigger.replace(2, 2, mrow); + assert(verify_string(replace_mid_bigger)); + str replace_mid_same{input}; + replace_mid_same.replace(2, 4, mrow); + assert(verify_string(replace_mid_same)); + str replace_mid_smaller{input}; + replace_mid_smaller.replace(2, 6, mrow); + assert(verify_string(replace_mid_smaller)); + + str replace_back_bigger{input}; + replace_back_bigger.replace(replace_back_bigger.size() - 2, 2, mrow); + assert(verify_string(replace_back_bigger)); + str replace_back_same{input}; + replace_back_same.replace(replace_back_same.size() - 4, 4, mrow); + assert(verify_string(replace_back_same)); + str replace_back_smaller{input}; + replace_back_smaller.replace(replace_back_smaller.size() - 6, 6, mrow); + assert(verify_string(replace_back_smaller)); + + const CharType hi[] = {'h', 'i', '\0'}; + str replace_large_to_sso{input}; + replace_large_to_sso.replace(0, replace_large_to_sso.size() - 1, hi); + assert(verify_string(replace_large_to_sso)); + + str replace_sso_to_large{input_sso}; + replace_sso_to_large.replace(0, 1, input); + assert(verify_string(replace_sso_to_large)); + } + if constexpr (allocator_traits::propagate_on_container_swap::value) { // swap str first_large{input}; str second_large = input + str{CharType{'c'}, CharType{'a'}, CharType{'t'}}; diff --git a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp index b8788d444bc..4890313e951 100644 --- a/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_vector/test.cpp @@ -20,6 +20,8 @@ using namespace std; +#define STATIC_ASSERT(...) static_assert(__VA_ARGS__, #__VA_ARGS__) + #ifndef __SANITIZE_ADDRESS__ #if defined(__clang__) && defined(__has_feature) #if __has_feature(address_sanitizer) @@ -156,9 +158,9 @@ bool verify_vector(vector& vec) { #ifdef __SANITIZE_ADDRESS__ const void* buffer = vec.data(); const void* buf_end = vec.data() + vec.capacity(); - _AsanAlignedPointers aligned; + _Asan_aligned_pointers aligned; - if constexpr ((_Container_allocation_minimum_alignment>) >= 8) { + if constexpr ((_Container_allocation_minimum_asan_alignment>) >= 8) { aligned = {buffer, buf_end}; } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); @@ -218,7 +220,7 @@ constexpr bool operator!=( template struct aligned_allocator : custom_test_allocator { - static constexpr size_t _Minimum_allocation_alignment = 8; + static constexpr size_t _Minimum_asan_allocation_alignment = 8; aligned_allocator() = default; template @@ -232,11 +234,11 @@ struct aligned_allocator : custom_test_allocator { delete[] p; } }; -static_assert(_Container_allocation_minimum_asan_alignment>> == 8); +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment>> == 8); template struct explicit_allocator : custom_test_allocator { - static constexpr size_t _Minimum_allocation_alignment = alignof(T); + static constexpr size_t _Minimum_asan_allocation_alignment = alignof(T); explicit_allocator() = default; template @@ -251,8 +253,8 @@ struct explicit_allocator : custom_test_allocator { delete[] (p - 1); } }; -static_assert(_Container_allocation_minimum_asan_alignment>> == 1); -static_assert(_Container_allocation_minimum_asan_alignment>> == 2); +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment>> == 1); +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment>> == 2); template struct implicit_allocator : custom_test_allocator { @@ -269,8 +271,8 @@ struct implicit_allocator : custom_test_allocator { delete[] (p - 1); } }; -static_assert(_Container_allocation_minimum_asan_alignment>> == 1); -static_assert(_Container_allocation_minimum_asan_alignment>> == 2); +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment>> == 1); +STATIC_ASSERT(_Container_allocation_minimum_asan_alignment>> == 2); template void test_push_pop() { From 6e37b5cf519f2059c1106082bd3e4f02f70c782a Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 8 Nov 2022 10:22:31 -0800 Subject: [PATCH 24/30] fix tests --- stl/inc/xstring | 71 +-- .../GH_002030_asan_annotate_string/test.cpp | 468 ++++++++++-------- 2 files changed, 303 insertions(+), 236 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index b46e934076c..f9ed44253f9 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2570,35 +2570,48 @@ private: const void* const _Old_last = _First + _Old_size + 1; const void* const _New_last = _First + _New_size + 1; + _Asan_aligned_pointers _Aligned; + constexpr bool _Large_string_always_asan_aligned = (_Container_allocation_minimum_asan_alignment) >= _Asan_granularity; - if constexpr (_Large_string_always_asan_aligned && _Small_string_always_asan_aligned()) { - // old state: - // [_First, _Old_last) valid - // [_Old_last, _End) poison - // new state: - // [_First, _New_last) valid - // [_New_last, asan_aligned_after(_End)) poison - _CSTD __sanitizer_annotate_contiguous_container( - _First, _STD _Get_asan_aligned_after(_End), _Old_last, _New_last); - } else { - const auto _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); - // The buffer must always have size >= 9 bytes, so it will always end at least one shadow memory section. - // last must be <= than end, so fix up when `_Aligned._End < _*_last` - const void* const _Old_fixed = _Aligned._Clamp_to_end(_Old_last); - const void* const _New_fixed = _Aligned._Clamp_to_end(_New_last); + // for the non-aligned buffer options, the buffer must always have size >= 9 bytes, + // so it will always end at least one shadow memory section. - // old state: - // [_Aligned._First, _Old_fixed) valid - // [_Old_fixed, _Aligned._End) poison - // [_Aligned._End, _End) valid - // new state: - // [_Aligned._First, _New_fixed) valid - // [_New_fixed, _Aligned._End) poison - // [_Aligned._End, _End) valid - _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); + if (_Capacity == _BUF_SIZE - 1) { + if constexpr (_Small_string_always_asan_aligned()) { + _Aligned = {_First, _STD _Get_asan_aligned_after(_End)}; + } else { + _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); + } + } else { + if constexpr (_Large_string_always_asan_aligned) { + _Aligned = {_First, _STD _Get_asan_aligned_after(_End)}; + } else { + _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); + } } + const void* const _Old_fixed = _Aligned._Clamp_to_end(_Old_last); + const void* const _New_fixed = _Aligned._Clamp_to_end(_New_last); + + // --- always aligned case --- + // old state: + // [_First, _Old_last) valid + // [_Old_last, asan_aligned_after(_End)) poison + // new state: + // [_First, _New_last) valid + // [_New_last, asan_aligned_after(_End)) poison + + // --- sometimes non-aligned case --- + // old state: + // [_Aligned._First, _Old_fixed) valid + // [_Old_fixed, _Aligned._End) poison + // [_Aligned._End, _End) valid + // new state: + // [_Aligned._First, _New_fixed) valid + // [_New_fixed, _Aligned._End) poison + // [_Aligned._End, _End) valid + _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Old_fixed, _New_fixed); } #define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() @@ -2615,8 +2628,8 @@ private: #endif // !_INSERT_STRING_ANNOTATION public: - _CONSTEXPR20 basic_string() noexcept(is_nothrow_default_constructible_v<_Alty>) - : _Mypair(_Zero_then_variadic_args_t{}) { + _CONSTEXPR20 + basic_string() noexcept(is_nothrow_default_constructible_v<_Alty>) : _Mypair(_Zero_then_variadic_args_t{}) { _Mypair._Myval2._Alloc_proxy(_GET_PROXY_ALLOCATOR(_Alty, _Getal())); _Tidy_init(); } @@ -4604,7 +4617,8 @@ public: } _NODISCARD _CONSTEXPR20 size_type find_first_of( - _In_z_ const _Elem* const _Ptr, const size_type _Off = 0) const noexcept /* strengthened */ { + _In_z_ const _Elem* const _Ptr, const size_type _Off = 0) const noexcept + /* strengthened */ { // look for one of [_Ptr, ) at or after _Off return static_cast(_Traits_find_first_of<_Traits>( _Mypair._Myval2._Myptr(), _Mypair._Myval2._Mysize, _Off, _Ptr, _Traits::length(_Ptr))); @@ -4679,7 +4693,8 @@ public: } _NODISCARD _CONSTEXPR20 size_type find_first_not_of( - _In_z_ const _Elem* const _Ptr, size_type _Off = 0) const noexcept /* strengthened */ { + _In_z_ const _Elem* const _Ptr, size_type _Off = 0) const noexcept + /* strengthened */ { // look for one of [_Ptr, ) at or after _Off return static_cast(_Traits_find_first_not_of<_Traits>( _Mypair._Myval2._Myptr(), _Mypair._Myval2._Mysize, _Off, _Ptr, _Traits::length(_Ptr))); diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 291e45c402a..2cff5626f0e 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -9,7 +9,7 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wc++17-extensions" // constexpr if is a C++17 extension -#endif // __clang__ +#endif // __clang__ #include #include @@ -31,7 +31,7 @@ using namespace std; #define STATIC_ASSERT(...) static_assert(__VA_ARGS__, #__VA_ARGS__) #ifdef __SANITIZE_ADDRESS__ -extern "C" int __sanitizer_verify_contiguous_container(const void* beg, const void* mid, const void* end) noexcept; +extern "C" int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, const void *end) noexcept; #endif // ASan instrumentation enabled constexpr auto literal_input = "Hello fluffy kittens"; @@ -40,303 +40,323 @@ constexpr auto literal_input_u8 = u8"Hello fluffy kittens"; #endif // __cpp_char8_t constexpr auto literal_input_u16 = u"Hello fluffy kittens"; constexpr auto literal_input_u32 = U"Hello fluffy kittens"; -constexpr auto literal_input_w = L"Hello fluffy kittens"; - -struct FourByte { - int padding = 0; -}; - -#if _HAS_CXX17 -template -struct alignas(8) AlignOnFour : FourByte, T { - using T::T; - using T::operator=; - - T& base() & { - return *this; - } - const T& base() const& { - return *this; - } - T&& base() && { - return move(*this); - } - const T&& base() const&& { - return move(*this); - } - - template - friend auto operator+(T1&& t1, T2&& t2) - -> enable_if_t, AlignOnFour> || is_same_v, AlignOnFour>, AlignOnFour> { - return AlignOnFour(InternalConstructTag{}, [&] { - if constexpr (is_same_v, AlignOnFour> && is_same_v, AlignOnFour>) { - return forward(t1).base() + forward(t2).base(); - } else if constexpr (is_same_v, AlignOnFour>) { - return forward(t1).base() + forward(t2); - } else { - return forward(t1) + forward(t2).base(); - } - }); - } - - template - AlignOnFour substr(Ts&&... ts) const { - return AlignOnFour(InternalConstructTag{}, [&] { return base().substr(forward(ts)...); }); - } - -private: - struct InternalConstructTag {}; - - template - AlignOnFour(InternalConstructTag, F f) : T(f()) {} -}; -#else // ^^^ no workaround / workaround vvv -// in C++14, you can't do the `using T::T;` thing and also define your own ctor -template -using AlignOnFour = T; -#endif +constexpr auto literal_input_w = L"Hello fluffy kittens"; template -constexpr auto get_large_input() { - if constexpr (is_same_v) { +constexpr auto get_large_input() +{ + if constexpr (is_same_v) + { return literal_input; #ifdef __cpp_char8_t - } else if constexpr (is_same_v) { + } + else if constexpr (is_same_v) + { return literal_input_u8; #endif // __cpp_char8_t - } else if constexpr (is_same_v) { + } + else if constexpr (is_same_v) + { return literal_input_u16; - } else if constexpr (is_same_v) { + } + else if constexpr (is_same_v) + { return literal_input_u32; - } else { + } + else + { return literal_input_w; } } template -constexpr auto get_sso_input() { - if constexpr (is_same_v) { +constexpr auto get_sso_input() +{ + if constexpr (is_same_v) + { return "cat"; #ifdef __cpp_char8_t - } else if constexpr (is_same_v) { + } + else if constexpr (is_same_v) + { return u8"cat"; #endif // __cpp_char8_t - } else if constexpr (is_same_v) { + } + else if constexpr (is_same_v) + { return u"cat"; - } else if constexpr (is_same_v) { + } + else if constexpr (is_same_v) + { return U"cat"; - } else { + } + else + { return L"cat"; } } +template +constexpr size_t max_sso_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)) - 1; + #if _HAS_CXX17 template -constexpr auto get_large_input_view() { +constexpr auto get_large_input_view() +{ return basic_string_view{get_large_input()}; } template -constexpr auto get_sso_input_view() { +constexpr auto get_sso_input_view() +{ return basic_string_view{get_sso_input()}; } #endif // _HAS_CXX17 #if _HAS_CXX17 template -struct string_view_convertible { - constexpr operator basic_string_view() const noexcept { +struct string_view_convertible +{ + constexpr operator basic_string_view() const noexcept + { return get_large_input_view(); } }; template -struct string_view_convertible_sso { - constexpr operator basic_string_view() const noexcept { +struct string_view_convertible_sso +{ + constexpr operator basic_string_view() const noexcept + { return get_sso_input_view(); } }; #endif // _HAS_CXX17 template -struct throw_on_conversion { +struct throw_on_conversion +{ throw_on_conversion() = default; throw_on_conversion(CharType) {} - operator CharType() const { + operator CharType() const + { throw 42; } }; template -class input_iterator_tester { +class input_iterator_tester +{ private: CharType data[N] = {}; public: - input_iterator_tester() noexcept { + input_iterator_tester() noexcept + { fill(data, data + N, CharType{'b'}); } - class iterator { + class iterator + { private: - CharType* curr; + CharType *curr; public: using iterator_category = input_iterator_tag; - using value_type = CharType; - using difference_type = ptrdiff_t; - using pointer = void; - using reference = CharType&; + using value_type = CharType; + using difference_type = ptrdiff_t; + using pointer = void; + using reference = CharType &; - explicit iterator(CharType* start) : curr(start) {} + explicit iterator(CharType *start) : curr(start) {} - reference operator*() const { + reference operator*() const + { return *curr; } - iterator& operator++() { + iterator &operator++() + { ++curr; return *this; } - iterator operator++(int) { + iterator operator++(int) + { auto tmp = *this; ++curr; return tmp; } - bool operator==(const iterator& that) const { + bool operator==(const iterator &that) const + { return curr == that.curr; } - bool operator!=(const iterator& that) const { + bool operator!=(const iterator &that) const + { return !(*this == that); } }; - iterator begin() { + iterator begin() + { return iterator(data); } - iterator end() { + iterator end() + { return iterator(data + N); } }; template -bool verify_string(const basic_string, Alloc>& str) { +bool verify_string(const basic_string, Alloc> &str) +{ #ifdef __SANITIZE_ADDRESS__ - const void* const buffer = str.data(); - const void* const buf_end = str.data() + str.capacity() + 1; + const void *const buffer = str.data(); + const void *const buf_end = str.data() + str.capacity() + 1; _Asan_aligned_pointers aligned; - if constexpr ((_Container_allocation_minimum_asan_alignment>) >= 8) { - aligned = {buffer, buf_end}; - } else { - aligned = _Get_asan_aligned_first_end(buffer, buf_end); - assert(aligned._First != aligned._End); + constexpr bool _Large_string_always_aligned = (_Container_allocation_minimum_asan_alignment>) >= 8; + constexpr bool _Small_string_always_aligned = + alignof(CharType *) >= 8 || alignof(CharType) >= 8; + + if (str.capacity() == max_sso_size) + { + // SSO + if constexpr (_Small_string_always_aligned) + { + aligned = {buffer, _Get_asan_aligned_after(buf_end)}; + } + else + { + aligned = _Get_asan_aligned_first_end(buffer, buf_end); + } } + else + { + if constexpr (_Large_string_always_aligned) + { + aligned = {buffer, _Get_asan_aligned_after(buf_end)}; + } + else + { + aligned = _Get_asan_aligned_first_end(buffer, buf_end); + } + } + assert(aligned._First != aligned._End); - const void* const mid = str.data() + str.size() + 1; - const void* const fixed_mid = aligned._Clamp_to_end(mid); + const void *const mid = str.data() + str.size() + 1; + const void *const fixed_mid = aligned._Clamp_to_end(mid); return __sanitizer_verify_contiguous_container(aligned._First, fixed_mid, aligned._End) != 0; -#else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv - (void) str; +#else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv + (void)str; return true; #endif // ASan instrumentation disabled } // Note: This class does not satisfy all the allocator requirements but is sufficient for this test. template -struct custom_test_allocator { - using value_type = CharType; +struct custom_test_allocator +{ + using value_type = CharType; using propagate_on_container_move_assignment = Pocma; - using is_always_equal = Stateless; + using is_always_equal = Stateless; }; template constexpr bool operator==( - const custom_test_allocator&, const custom_test_allocator&) noexcept { + const custom_test_allocator &, const custom_test_allocator &) noexcept +{ return Stateless::value; } template constexpr bool operator!=( - const custom_test_allocator&, const custom_test_allocator&) noexcept { + const custom_test_allocator &, const custom_test_allocator &) noexcept +{ return !Stateless::value; } template -struct aligned_allocator : public custom_test_allocator { +struct aligned_allocator : public custom_test_allocator +{ static constexpr size_t _Minimum_asan_allocation_alignment = 8; aligned_allocator() = default; template - constexpr aligned_allocator(const aligned_allocator&) noexcept {} + constexpr aligned_allocator(const aligned_allocator &) noexcept {} - CharType* allocate(size_t n) { + CharType *allocate(size_t n) + { return new CharType[n]; } - void deallocate(CharType* p, size_t) noexcept { + void deallocate(CharType *p, size_t) noexcept + { delete[] p; } }; STATIC_ASSERT( _Container_allocation_minimum_asan_alignment, aligned_allocator>> == 8); STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< - basic_string, aligned_allocator>> - == 8); + basic_string, aligned_allocator>> == 8); template -struct explicit_allocator : public custom_test_allocator { +struct explicit_allocator : public custom_test_allocator +{ static constexpr size_t _Minimum_asan_allocation_alignment = alignof(CharType); explicit_allocator() = default; template - constexpr explicit_allocator(const explicit_allocator&) noexcept {} + constexpr explicit_allocator(const explicit_allocator &) noexcept {} - CharType* allocate(size_t n) { - CharType* mem = new CharType[n + 1]; + CharType *allocate(size_t n) + { + CharType *mem = new CharType[n + 1]; return mem + 1; } - void deallocate(CharType* p, size_t) noexcept { + void deallocate(CharType *p, size_t) noexcept + { delete[] (p - 1); } }; STATIC_ASSERT( _Container_allocation_minimum_asan_alignment, explicit_allocator>> == 1); STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< - basic_string, explicit_allocator>> - == 2); + basic_string, explicit_allocator>> == 2); template -struct implicit_allocator : public custom_test_allocator { +struct implicit_allocator : public custom_test_allocator +{ implicit_allocator() = default; template - constexpr implicit_allocator(const implicit_allocator&) noexcept {} + constexpr implicit_allocator(const implicit_allocator &) noexcept {} - CharType* allocate(size_t n) { - CharType* mem = new CharType[n + 1]; + CharType *allocate(size_t n) + { + CharType *mem = new CharType[n + 1]; return mem + 1; } - void deallocate(CharType* p, size_t) noexcept { + void deallocate(CharType *p, size_t) noexcept + { delete[] (p - 1); } }; STATIC_ASSERT( _Container_allocation_minimum_asan_alignment, implicit_allocator>> == 1); STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< - basic_string, implicit_allocator>> - == 2); + basic_string, implicit_allocator>> == 2); template -void test_construction() { +void test_construction() +{ using CharType = typename Alloc::value_type; - using str = AlignOnFour, Alloc>>; + using str = basic_string, Alloc>; { // constructors // range constructors str literal_constructed_sso{get_sso_input()}; @@ -346,9 +366,9 @@ void test_construction() { assert(verify_string(literal_constructed)); str initializer_list_constructed({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, CharType{'o'}, - CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); + CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); assert(verify_string(initializer_list_constructed)); str initializer_list_constructed_sso({CharType{'c'}, CharType{'a'}, CharType{'t'}}); @@ -503,9 +523,9 @@ void test_construction() { str initializer_list_constructed( {CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, - CharType{'s'}}, + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, + CharType{'s'}}, alloc); assert(verify_string(initializer_list_constructed)); @@ -587,17 +607,17 @@ void test_construction() { } template -void test_append() { +void test_append() +{ using CharType = typename Alloc::value_type; - using str = AlignOnFour, Alloc>>; + using str = basic_string, Alloc>; - constexpr size_t large_size = 20; - constexpr size_t sso_size = 1; - constexpr size_t max_sso_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)) - 1; + constexpr size_t large_size = 20; + constexpr size_t sso_size = 1; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); - const str input_sso_growing(max_sso_size, CharType{'b'}); + const str input_sso_growing(max_sso_size, CharType{'b'}); { // push_back str push_back{input}; @@ -914,7 +934,7 @@ void test_append() { str op_rstr_char_sso = str(sso_size, CharType{'b'}) + CharType{'!'}; assert(verify_string(op_rstr_char_sso)); - str op_rstr_char_sso_growing = str(max_sso_size, CharType{'b'}) + CharType{'!'}; + str op_rstr_char_sso_growing = str(max_sso_size, CharType{'b'}) + CharType{'!'}; assert(verify_string(op_rstr_char_sso_growing)); str op_char_rstr_large = CharType{'!'} + str(large_size, CharType{'b'}); @@ -923,18 +943,19 @@ void test_append() { str op_char_rstr_sso = CharType{'!'} + str(sso_size, CharType{'b'}); assert(verify_string(op_char_rstr_sso)); - str op_char_rstr_sso_growing = CharType{'!'} + str(max_sso_size, CharType{'b'}); + str op_char_rstr_sso_growing = CharType{'!'} + str(max_sso_size, CharType{'b'}); assert(verify_string(op_char_rstr_sso_growing)); } } template -void test_assign() { +void test_assign() +{ using CharType = typename Alloc::value_type; - using str = AlignOnFour, Alloc>>; + using str = basic_string, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t sso_size = 2; const str start(large_size - 1, CharType{'a'}); const str start_sso(sso_size + 1, CharType{'a'}); @@ -1008,9 +1029,9 @@ void test_assign() { str initializer_list_assigned_large_large{start}; initializer_list_assigned_large_large = {CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; assert(verify_string(initializer_list_assigned_large_large)); str initializer_list_assigned_large_sso{start}; @@ -1019,9 +1040,9 @@ void test_assign() { str initializer_list_assigned_sso_large{start_sso}; initializer_list_assigned_sso_large = {CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; assert(verify_string(initializer_list_assigned_sso_large)); str initializer_list_assigned_sso_sso{start_sso}; @@ -1200,9 +1221,9 @@ void test_assign() { str assign_initializer_list_large_large{start}; assign_initializer_list_large_large.assign({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); assert(verify_string(assign_initializer_list_large_large)); str assign_initializer_list_large_sso{start}; @@ -1211,9 +1232,9 @@ void test_assign() { str assign_initializer_list_sso_large{start_sso}; assign_initializer_list_sso_large.assign({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); assert(verify_string(assign_initializer_list_sso_large)); str assign_initializer_list_sso_sso{start_sso}; @@ -1260,17 +1281,17 @@ void test_assign() { } template -void test_insertion() { +void test_insertion() +{ using CharType = typename Alloc::value_type; - using str = AlignOnFour, Alloc>>; + using str = basic_string, Alloc>; - constexpr size_t large_size = 20; - constexpr size_t sso_size = 1; - constexpr size_t max_sso_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)) - 1; + constexpr size_t large_size = 20; + constexpr size_t sso_size = 1; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); - const str input_sso_growing(max_sso_size, CharType{'b'}); + const str input_sso_growing(max_sso_size, CharType{'b'}); input_iterator_tester input_iter_data_sso; @@ -1442,12 +1463,13 @@ void test_insertion() { } template -void test_removal() { +void test_removal() +{ using CharType = typename Alloc::value_type; - using str = AlignOnFour, Alloc>>; + using str = basic_string, Alloc>; - constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t large_size = 20; + constexpr size_t sso_size = 2; constexpr size_t min_large_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)); const str input(large_size, CharType{'b'}); @@ -1536,11 +1558,13 @@ void test_removal() { assert(verify_string(erased_free_sso)); str erased_free_if{get_large_input()}; - erase_if(erased_free_if, [](const CharType val) { return val == CharType{'t'}; }); + erase_if(erased_free_if, [](const CharType val) + { return val == CharType{'t'}; }); assert(verify_string(erased_free_if)); str erased_free_if_sso{get_sso_input()}; - erase_if(erased_free_if_sso, [](const CharType val) { return val == CharType{'t'}; }); + erase_if(erased_free_if_sso, [](const CharType val) + { return val == CharType{'t'}; }); assert(verify_string(erased_free_if_sso)); #endif // _HAS_CXX20 } @@ -1578,12 +1602,13 @@ void test_removal() { } template -void test_misc() { +void test_misc() +{ using CharType = typename Alloc::value_type; - using str = AlignOnFour, Alloc>>; + using str = basic_string, Alloc>; - constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t large_size = 20; + constexpr size_t sso_size = 2; constexpr size_t min_large_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)); const str input(large_size, CharType{'b'}); @@ -1687,7 +1712,8 @@ void test_misc() { assert(verify_string(replace_sso_to_large)); } - if constexpr (allocator_traits::propagate_on_container_swap::value) { // swap + if constexpr (allocator_traits::propagate_on_container_swap::value) + { // swap str first_large{input}; str second_large = input + str{CharType{'c'}, CharType{'a'}, CharType{'t'}}; @@ -1713,13 +1739,14 @@ void test_misc() { } template -void test_sstream() { +void test_sstream() +{ using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; - using stream = basic_stringbuf, Alloc>; + using str = basic_string, Alloc>; + using stream = basic_stringbuf, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t sso_size = 2; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); @@ -1761,12 +1788,13 @@ void test_sstream() { } template -void test_exceptions() { +void test_exceptions() +{ using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t sso_size = 2; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); @@ -1776,11 +1804,14 @@ void test_exceptions() { { // append str append_iterator{input}; - try { + try + { assert(verify_string(append_iterator)); append_iterator.append(begin(iter_data), end(iter_data)); assert(false); - } catch (...) { + } + catch (...) + { assert(verify_string(append_iterator)); } @@ -1796,11 +1827,14 @@ void test_exceptions() { #endif str append_input_iterator{input}; - try { + try + { assert(verify_string(append_input_iterator)); append_input_iterator.append(input_iter_data.begin(), input_iter_data.end()); assert(false); - } catch (...) { + } + catch (...) + { assert(verify_string(append_input_iterator)); } @@ -1816,11 +1850,14 @@ void test_exceptions() { { // assign str assign_iterator{input}; - try { + try + { assert(verify_string(assign_iterator)); assign_iterator.assign(begin(iter_data), end(iter_data)); assert(false); - } catch (...) { + } + catch (...) + { assert(verify_string(assign_iterator)); } @@ -1835,11 +1872,14 @@ void test_exceptions() { #endif str assign_input_iterator{input}; - try { + try + { assert(verify_string(assign_input_iterator)); assign_input_iterator.assign(input_iter_data.begin(), input_iter_data.end()); assert(false); - } catch (...) { + } + catch (...) + { assert(verify_string(assign_input_iterator)); } @@ -1857,11 +1897,14 @@ void test_exceptions() { { // insert str insert_iterator{input}; - try { + try + { assert(verify_string(insert_iterator)); insert_iterator.insert(insert_iterator.begin(), begin(iter_data), end(iter_data)); assert(false); - } catch (...) { + } + catch (...) + { assert(verify_string(insert_iterator)); } @@ -1877,11 +1920,14 @@ void test_exceptions() { #endif str insert_input_iterator{input}; - try { + try + { assert(verify_string(insert_input_iterator)); insert_input_iterator.insert(insert_input_iterator.begin(), input_iter_data.begin(), input_iter_data.end()); assert(false); - } catch (...) { + } + catch (...) + { assert(verify_string(insert_input_iterator)); } @@ -1900,7 +1946,8 @@ void test_exceptions() { } template -void run_tests() { +void run_tests() +{ test_construction(); test_append(); test_assign(); @@ -1914,7 +1961,8 @@ void run_tests() { } template class Alloc> -void run_custom_allocator_matrix() { +void run_custom_allocator_matrix() +{ run_tests>(); run_tests>(); run_tests>(); @@ -1922,21 +1970,23 @@ void run_custom_allocator_matrix() { } template -void run_allocator_matrix() { +void run_allocator_matrix() +{ run_tests>(); run_custom_allocator_matrix(); run_custom_allocator_matrix(); run_custom_allocator_matrix(); } -void test_DevCom_10116361() { +void test_DevCom_10116361() +{ // We failed to null-terminate copies of SSO strings with ASAN annotations active. #ifdef _WIN64 - constexpr const char* text = "testtest"; - constexpr size_t n = 8; + constexpr const char *text = "testtest"; + constexpr size_t n = 8; #else - constexpr const char* text = "test"; - constexpr size_t n = 4; + constexpr const char *text = "test"; + constexpr size_t n = 4; #endif string s0{text}; @@ -1945,13 +1995,14 @@ void test_DevCom_10116361() { alignas(string) unsigned char space[sizeof(string)]; memset(space, 0xff, sizeof(space)); - string& s1 = *::new (&space) string{s0}; + string &s1 = *::new (&space) string{s0}; assert(s1.c_str()[n] == '\0'); s1.~string(); } -void test_DevCom_10109507() { +void test_DevCom_10109507() +{ // replace failed to correctly munge asan annotations while working string s("abcd"); s.replace(0, 1, "ef", 2); @@ -1959,7 +2010,8 @@ void test_DevCom_10109507() { assert(s == "xyefbcd"); } -int main() { +int main() +{ run_allocator_matrix(); #ifdef __cpp_char8_t run_allocator_matrix(); From 74f8320b65c8d9eb3763968fa46d77b19053d9d0 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 8 Nov 2022 10:50:30 -0800 Subject: [PATCH 25/30] forgot to format as I was copying back and forth --- .../GH_002030_asan_annotate_string/test.cpp | 392 +++++++----------- 1 file changed, 150 insertions(+), 242 deletions(-) diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 2cff5626f0e..f3678f93120 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -9,7 +9,7 @@ #ifdef __clang__ #pragma clang diagnostic ignored "-Wc++17-extensions" // constexpr if is a C++17 extension -#endif // __clang__ +#endif // __clang__ #include #include @@ -31,7 +31,7 @@ using namespace std; #define STATIC_ASSERT(...) static_assert(__VA_ARGS__, #__VA_ARGS__) #ifdef __SANITIZE_ADDRESS__ -extern "C" int __sanitizer_verify_contiguous_container(const void *beg, const void *mid, const void *end) noexcept; +extern "C" int __sanitizer_verify_contiguous_container(const void* beg, const void* mid, const void* end) noexcept; #endif // ASan instrumentation enabled constexpr auto literal_input = "Hello fluffy kittens"; @@ -40,58 +40,38 @@ constexpr auto literal_input_u8 = u8"Hello fluffy kittens"; #endif // __cpp_char8_t constexpr auto literal_input_u16 = u"Hello fluffy kittens"; constexpr auto literal_input_u32 = U"Hello fluffy kittens"; -constexpr auto literal_input_w = L"Hello fluffy kittens"; +constexpr auto literal_input_w = L"Hello fluffy kittens"; template -constexpr auto get_large_input() -{ - if constexpr (is_same_v) - { +constexpr auto get_large_input() { + if constexpr (is_same_v) { return literal_input; #ifdef __cpp_char8_t - } - else if constexpr (is_same_v) - { + } else if constexpr (is_same_v) { return literal_input_u8; #endif // __cpp_char8_t - } - else if constexpr (is_same_v) - { + } else if constexpr (is_same_v) { return literal_input_u16; - } - else if constexpr (is_same_v) - { + } else if constexpr (is_same_v) { return literal_input_u32; - } - else - { + } else { return literal_input_w; } } template -constexpr auto get_sso_input() -{ - if constexpr (is_same_v) - { +constexpr auto get_sso_input() { + if constexpr (is_same_v) { return "cat"; #ifdef __cpp_char8_t - } - else if constexpr (is_same_v) - { + } else if constexpr (is_same_v) { return u8"cat"; #endif // __cpp_char8_t - } - else if constexpr (is_same_v) - { + } else if constexpr (is_same_v) { return u"cat"; - } - else if constexpr (is_same_v) - { + } else if constexpr (is_same_v) { return U"cat"; - } - else - { + } else { return L"cat"; } } @@ -101,262 +81,224 @@ constexpr size_t max_sso_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(Cha #if _HAS_CXX17 template -constexpr auto get_large_input_view() -{ +constexpr auto get_large_input_view() { return basic_string_view{get_large_input()}; } template -constexpr auto get_sso_input_view() -{ +constexpr auto get_sso_input_view() { return basic_string_view{get_sso_input()}; } #endif // _HAS_CXX17 #if _HAS_CXX17 template -struct string_view_convertible -{ - constexpr operator basic_string_view() const noexcept - { +struct string_view_convertible { + constexpr operator basic_string_view() const noexcept { return get_large_input_view(); } }; template -struct string_view_convertible_sso -{ - constexpr operator basic_string_view() const noexcept - { +struct string_view_convertible_sso { + constexpr operator basic_string_view() const noexcept { return get_sso_input_view(); } }; #endif // _HAS_CXX17 template -struct throw_on_conversion -{ +struct throw_on_conversion { throw_on_conversion() = default; throw_on_conversion(CharType) {} - operator CharType() const - { + operator CharType() const { throw 42; } }; template -class input_iterator_tester -{ +class input_iterator_tester { private: CharType data[N] = {}; public: - input_iterator_tester() noexcept - { + input_iterator_tester() noexcept { fill(data, data + N, CharType{'b'}); } - class iterator - { + class iterator { private: - CharType *curr; + CharType* curr; public: using iterator_category = input_iterator_tag; - using value_type = CharType; - using difference_type = ptrdiff_t; - using pointer = void; - using reference = CharType &; + using value_type = CharType; + using difference_type = ptrdiff_t; + using pointer = void; + using reference = CharType&; - explicit iterator(CharType *start) : curr(start) {} + explicit iterator(CharType* start) : curr(start) {} - reference operator*() const - { + reference operator*() const { return *curr; } - iterator &operator++() - { + iterator& operator++() { ++curr; return *this; } - iterator operator++(int) - { + iterator operator++(int) { auto tmp = *this; ++curr; return tmp; } - bool operator==(const iterator &that) const - { + bool operator==(const iterator& that) const { return curr == that.curr; } - bool operator!=(const iterator &that) const - { + bool operator!=(const iterator& that) const { return !(*this == that); } }; - iterator begin() - { + iterator begin() { return iterator(data); } - iterator end() - { + iterator end() { return iterator(data + N); } }; template -bool verify_string(const basic_string, Alloc> &str) -{ +bool verify_string(const basic_string, Alloc>& str) { #ifdef __SANITIZE_ADDRESS__ - const void *const buffer = str.data(); - const void *const buf_end = str.data() + str.capacity() + 1; + const void* const buffer = str.data(); + const void* const buf_end = str.data() + str.capacity() + 1; _Asan_aligned_pointers aligned; - constexpr bool _Large_string_always_aligned = (_Container_allocation_minimum_asan_alignment>) >= 8; - constexpr bool _Small_string_always_aligned = - alignof(CharType *) >= 8 || alignof(CharType) >= 8; + constexpr bool _Large_string_always_aligned = + (_Container_allocation_minimum_asan_alignment>) >= 8; + constexpr bool _Small_string_always_aligned = alignof(CharType*) >= 8 || alignof(CharType) >= 8; - if (str.capacity() == max_sso_size) - { + if (str.capacity() == max_sso_size) { // SSO - if constexpr (_Small_string_always_aligned) - { + if constexpr (_Small_string_always_aligned) { aligned = {buffer, _Get_asan_aligned_after(buf_end)}; - } - else - { + } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); } - } - else - { - if constexpr (_Large_string_always_aligned) - { + } else { + if constexpr (_Large_string_always_aligned) { aligned = {buffer, _Get_asan_aligned_after(buf_end)}; - } - else - { + } else { aligned = _Get_asan_aligned_first_end(buffer, buf_end); } } assert(aligned._First != aligned._End); - const void *const mid = str.data() + str.size() + 1; - const void *const fixed_mid = aligned._Clamp_to_end(mid); + const void* const mid = str.data() + str.size() + 1; + const void* const fixed_mid = aligned._Clamp_to_end(mid); return __sanitizer_verify_contiguous_container(aligned._First, fixed_mid, aligned._End) != 0; -#else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv - (void)str; +#else // ^^^ ASan instrumentation enabled ^^^ // vvv ASan instrumentation disabled vvv + (void) str; return true; #endif // ASan instrumentation disabled } // Note: This class does not satisfy all the allocator requirements but is sufficient for this test. template -struct custom_test_allocator -{ - using value_type = CharType; +struct custom_test_allocator { + using value_type = CharType; using propagate_on_container_move_assignment = Pocma; - using is_always_equal = Stateless; + using is_always_equal = Stateless; }; template constexpr bool operator==( - const custom_test_allocator &, const custom_test_allocator &) noexcept -{ + const custom_test_allocator&, const custom_test_allocator&) noexcept { return Stateless::value; } template constexpr bool operator!=( - const custom_test_allocator &, const custom_test_allocator &) noexcept -{ + const custom_test_allocator&, const custom_test_allocator&) noexcept { return !Stateless::value; } template -struct aligned_allocator : public custom_test_allocator -{ +struct aligned_allocator : public custom_test_allocator { static constexpr size_t _Minimum_asan_allocation_alignment = 8; aligned_allocator() = default; template - constexpr aligned_allocator(const aligned_allocator &) noexcept {} + constexpr aligned_allocator(const aligned_allocator&) noexcept {} - CharType *allocate(size_t n) - { + CharType* allocate(size_t n) { return new CharType[n]; } - void deallocate(CharType *p, size_t) noexcept - { + void deallocate(CharType* p, size_t) noexcept { delete[] p; } }; STATIC_ASSERT( _Container_allocation_minimum_asan_alignment, aligned_allocator>> == 8); STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< - basic_string, aligned_allocator>> == 8); + basic_string, aligned_allocator>> + == 8); template -struct explicit_allocator : public custom_test_allocator -{ +struct explicit_allocator : public custom_test_allocator { static constexpr size_t _Minimum_asan_allocation_alignment = alignof(CharType); explicit_allocator() = default; template - constexpr explicit_allocator(const explicit_allocator &) noexcept {} + constexpr explicit_allocator(const explicit_allocator&) noexcept {} - CharType *allocate(size_t n) - { - CharType *mem = new CharType[n + 1]; + CharType* allocate(size_t n) { + CharType* mem = new CharType[n + 1]; return mem + 1; } - void deallocate(CharType *p, size_t) noexcept - { + void deallocate(CharType* p, size_t) noexcept { delete[] (p - 1); } }; STATIC_ASSERT( _Container_allocation_minimum_asan_alignment, explicit_allocator>> == 1); STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< - basic_string, explicit_allocator>> == 2); + basic_string, explicit_allocator>> + == 2); template -struct implicit_allocator : public custom_test_allocator -{ +struct implicit_allocator : public custom_test_allocator { implicit_allocator() = default; template - constexpr implicit_allocator(const implicit_allocator &) noexcept {} + constexpr implicit_allocator(const implicit_allocator&) noexcept {} - CharType *allocate(size_t n) - { - CharType *mem = new CharType[n + 1]; + CharType* allocate(size_t n) { + CharType* mem = new CharType[n + 1]; return mem + 1; } - void deallocate(CharType *p, size_t) noexcept - { + void deallocate(CharType* p, size_t) noexcept { delete[] (p - 1); } }; STATIC_ASSERT( _Container_allocation_minimum_asan_alignment, implicit_allocator>> == 1); STATIC_ASSERT(_Container_allocation_minimum_asan_alignment< - basic_string, implicit_allocator>> == 2); + basic_string, implicit_allocator>> + == 2); template -void test_construction() -{ +void test_construction() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; { // constructors // range constructors str literal_constructed_sso{get_sso_input()}; @@ -366,9 +308,9 @@ void test_construction() assert(verify_string(literal_constructed)); str initializer_list_constructed({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, CharType{'o'}, - CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); + CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); assert(verify_string(initializer_list_constructed)); str initializer_list_constructed_sso({CharType{'c'}, CharType{'a'}, CharType{'t'}}); @@ -523,9 +465,9 @@ void test_construction() str initializer_list_constructed( {CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, - CharType{'s'}}, + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, + CharType{'s'}}, alloc); assert(verify_string(initializer_list_constructed)); @@ -607,13 +549,12 @@ void test_construction() } template -void test_append() -{ +void test_append() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 1; + constexpr size_t sso_size = 1; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); @@ -949,13 +890,12 @@ void test_append() } template -void test_assign() -{ +void test_assign() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t sso_size = 2; const str start(large_size - 1, CharType{'a'}); const str start_sso(sso_size + 1, CharType{'a'}); @@ -1029,9 +969,9 @@ void test_assign() str initializer_list_assigned_large_large{start}; initializer_list_assigned_large_large = {CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; assert(verify_string(initializer_list_assigned_large_large)); str initializer_list_assigned_large_sso{start}; @@ -1040,9 +980,9 @@ void test_assign() str initializer_list_assigned_sso_large{start_sso}; initializer_list_assigned_sso_large = {CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}; assert(verify_string(initializer_list_assigned_sso_large)); str initializer_list_assigned_sso_sso{start_sso}; @@ -1221,9 +1161,9 @@ void test_assign() str assign_initializer_list_large_large{start}; assign_initializer_list_large_large.assign({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); assert(verify_string(assign_initializer_list_large_large)); str assign_initializer_list_large_sso{start}; @@ -1232,9 +1172,9 @@ void test_assign() str assign_initializer_list_sso_large{start_sso}; assign_initializer_list_sso_large.assign({CharType{'H'}, CharType{'e'}, CharType{'l'}, CharType{'l'}, - CharType{'o'}, CharType{' '}, // - CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, - CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); + CharType{'o'}, CharType{' '}, // + CharType{'f'}, CharType{'l'}, CharType{'u'}, CharType{'f'}, CharType{'f'}, CharType{'y'}, CharType{' '}, + CharType{'k'}, CharType{'i'}, CharType{'t'}, CharType{'t'}, CharType{'e'}, CharType{'n'}, CharType{'s'}}); assert(verify_string(assign_initializer_list_sso_large)); str assign_initializer_list_sso_sso{start_sso}; @@ -1281,13 +1221,12 @@ void test_assign() } template -void test_insertion() -{ +void test_insertion() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 1; + constexpr size_t sso_size = 1; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); @@ -1463,13 +1402,12 @@ void test_insertion() } template -void test_removal() -{ +void test_removal() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; - constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t large_size = 20; + constexpr size_t sso_size = 2; constexpr size_t min_large_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)); const str input(large_size, CharType{'b'}); @@ -1558,13 +1496,11 @@ void test_removal() assert(verify_string(erased_free_sso)); str erased_free_if{get_large_input()}; - erase_if(erased_free_if, [](const CharType val) - { return val == CharType{'t'}; }); + erase_if(erased_free_if, [](const CharType val) { return val == CharType{'t'}; }); assert(verify_string(erased_free_if)); str erased_free_if_sso{get_sso_input()}; - erase_if(erased_free_if_sso, [](const CharType val) - { return val == CharType{'t'}; }); + erase_if(erased_free_if_sso, [](const CharType val) { return val == CharType{'t'}; }); assert(verify_string(erased_free_if_sso)); #endif // _HAS_CXX20 } @@ -1602,13 +1538,12 @@ void test_removal() } template -void test_misc() -{ +void test_misc() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; - constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t large_size = 20; + constexpr size_t sso_size = 2; constexpr size_t min_large_size = (16 / sizeof(CharType) < 1 ? 1 : 16 / sizeof(CharType)); const str input(large_size, CharType{'b'}); @@ -1712,8 +1647,7 @@ void test_misc() assert(verify_string(replace_sso_to_large)); } - if constexpr (allocator_traits::propagate_on_container_swap::value) - { // swap + if constexpr (allocator_traits::propagate_on_container_swap::value) { // swap str first_large{input}; str second_large = input + str{CharType{'c'}, CharType{'a'}, CharType{'t'}}; @@ -1739,14 +1673,13 @@ void test_misc() } template -void test_sstream() -{ +void test_sstream() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; - using stream = basic_stringbuf, Alloc>; + using str = basic_string, Alloc>; + using stream = basic_stringbuf, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t sso_size = 2; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); @@ -1788,13 +1721,12 @@ void test_sstream() } template -void test_exceptions() -{ +void test_exceptions() { using CharType = typename Alloc::value_type; - using str = basic_string, Alloc>; + using str = basic_string, Alloc>; constexpr size_t large_size = 20; - constexpr size_t sso_size = 2; + constexpr size_t sso_size = 2; const str input(large_size, CharType{'b'}); const str input_sso(sso_size, CharType{'b'}); @@ -1804,14 +1736,11 @@ void test_exceptions() { // append str append_iterator{input}; - try - { + try { assert(verify_string(append_iterator)); append_iterator.append(begin(iter_data), end(iter_data)); assert(false); - } - catch (...) - { + } catch (...) { assert(verify_string(append_iterator)); } @@ -1827,14 +1756,11 @@ void test_exceptions() #endif str append_input_iterator{input}; - try - { + try { assert(verify_string(append_input_iterator)); append_input_iterator.append(input_iter_data.begin(), input_iter_data.end()); assert(false); - } - catch (...) - { + } catch (...) { assert(verify_string(append_input_iterator)); } @@ -1850,14 +1776,11 @@ void test_exceptions() { // assign str assign_iterator{input}; - try - { + try { assert(verify_string(assign_iterator)); assign_iterator.assign(begin(iter_data), end(iter_data)); assert(false); - } - catch (...) - { + } catch (...) { assert(verify_string(assign_iterator)); } @@ -1872,14 +1795,11 @@ void test_exceptions() #endif str assign_input_iterator{input}; - try - { + try { assert(verify_string(assign_input_iterator)); assign_input_iterator.assign(input_iter_data.begin(), input_iter_data.end()); assert(false); - } - catch (...) - { + } catch (...) { assert(verify_string(assign_input_iterator)); } @@ -1897,14 +1817,11 @@ void test_exceptions() { // insert str insert_iterator{input}; - try - { + try { assert(verify_string(insert_iterator)); insert_iterator.insert(insert_iterator.begin(), begin(iter_data), end(iter_data)); assert(false); - } - catch (...) - { + } catch (...) { assert(verify_string(insert_iterator)); } @@ -1920,14 +1837,11 @@ void test_exceptions() #endif str insert_input_iterator{input}; - try - { + try { assert(verify_string(insert_input_iterator)); insert_input_iterator.insert(insert_input_iterator.begin(), input_iter_data.begin(), input_iter_data.end()); assert(false); - } - catch (...) - { + } catch (...) { assert(verify_string(insert_input_iterator)); } @@ -1946,8 +1860,7 @@ void test_exceptions() } template -void run_tests() -{ +void run_tests() { test_construction(); test_append(); test_assign(); @@ -1961,8 +1874,7 @@ void run_tests() } template class Alloc> -void run_custom_allocator_matrix() -{ +void run_custom_allocator_matrix() { run_tests>(); run_tests>(); run_tests>(); @@ -1970,23 +1882,21 @@ void run_custom_allocator_matrix() } template -void run_allocator_matrix() -{ +void run_allocator_matrix() { run_tests>(); run_custom_allocator_matrix(); run_custom_allocator_matrix(); run_custom_allocator_matrix(); } -void test_DevCom_10116361() -{ +void test_DevCom_10116361() { // We failed to null-terminate copies of SSO strings with ASAN annotations active. #ifdef _WIN64 - constexpr const char *text = "testtest"; - constexpr size_t n = 8; + constexpr const char* text = "testtest"; + constexpr size_t n = 8; #else - constexpr const char *text = "test"; - constexpr size_t n = 4; + constexpr const char* text = "test"; + constexpr size_t n = 4; #endif string s0{text}; @@ -1995,14 +1905,13 @@ void test_DevCom_10116361() alignas(string) unsigned char space[sizeof(string)]; memset(space, 0xff, sizeof(space)); - string &s1 = *::new (&space) string{s0}; + string& s1 = *::new (&space) string{s0}; assert(s1.c_str()[n] == '\0'); s1.~string(); } -void test_DevCom_10109507() -{ +void test_DevCom_10109507() { // replace failed to correctly munge asan annotations while working string s("abcd"); s.replace(0, 1, "ef", 2); @@ -2010,8 +1919,7 @@ void test_DevCom_10109507() assert(s == "xyefbcd"); } -int main() -{ +int main() { run_allocator_matrix(); #ifdef __cpp_char8_t run_allocator_matrix(); From cc418a398d47d0f5a909766f34ebfa062f40a3c3 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 8 Nov 2022 15:25:59 -0800 Subject: [PATCH 26/30] zack's CRs --- stl/inc/xmemory | 12 ++++++------ stl/inc/xstring | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/stl/inc/xmemory b/stl/inc/xmemory index 91693198b77..9fc07ed3a84 100644 --- a/stl/inc/xmemory +++ b/stl/inc/xmemory @@ -774,7 +774,8 @@ _NODISCARD constexpr allocation_result::pointe #endif // _HAS_CXX23 // The number of user bytes a single byte of ASAN shadow memory can track. -_INLINE_VAR constexpr size_t _Asan_granularity = 8; +_INLINE_VAR constexpr size_t _Asan_granularity = 8; +_INLINE_VAR constexpr size_t _Asan_granularity_mask = _Asan_granularity - 1; struct _Asan_aligned_pointers { const void* _First; @@ -872,10 +873,9 @@ struct _Asan_aligned_pointers { // Thus, this results in the shadow memory state from the second example. _NODISCARD inline _Asan_aligned_pointers _Get_asan_aligned_first_end( const void* const _First, const void* const _End) noexcept { - static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); return { - reinterpret_cast(reinterpret_cast(_First) & _Mask), - reinterpret_cast(reinterpret_cast(_End) & _Mask), + reinterpret_cast(reinterpret_cast(_First) & ~_Asan_granularity_mask), + reinterpret_cast(reinterpret_cast(_End) & ~_Asan_granularity_mask), }; } @@ -883,8 +883,8 @@ _NODISCARD inline _Asan_aligned_pointers _Get_asan_aligned_first_end( // we can simply push the `_End` pointer to the end of the shadow memory section. // This is _not_ safe in general (see _Get_asan_aligned_first_end's comment for why). _NODISCARD inline const void* _Get_asan_aligned_after(const void* const _End) noexcept { - static constexpr uintptr_t _Mask = ~(_Asan_granularity - 1); - return reinterpret_cast((reinterpret_cast(_End) + _Asan_granularity - 1) & _Mask); + return reinterpret_cast( + (reinterpret_cast(_End) + _Asan_granularity_mask) & ~_Asan_granularity_mask); } template diff --git a/stl/inc/xstring b/stl/inc/xstring index f9ed44253f9..972dd900e7e 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -3969,8 +3969,9 @@ public: if (_Count < _Nx || _Count - _Nx <= _Mypair._Myval2._Myres - _Old_size) { // either we are shrinking, or the growth fits // may temporarily overflow; OK because size_type must be unsigned - _ASAN_STRING_MODIFY(*this, _Old_size, _Old_size + _Count - _Nx); - _Mypair._Myval2._Mysize = _Old_size + _Count - _Nx; + const auto _New_size = _Old_size + _Count - _Nx; + _ASAN_STRING_MODIFY(*this, _Old_size, _New_size); + _Mypair._Myval2._Mysize = _New_size; _Elem* const _Old_ptr = _Mypair._Myval2._Myptr(); _Elem* const _Insert_at = _Old_ptr + _Off; _Traits::move(_Insert_at + _Count, _Insert_at + _Nx, _Old_size - _Nx - _Off + 1); From f6984d8e3ccca768d07c28a7a54aef92d1261151 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 14 Nov 2022 15:08:56 -0800 Subject: [PATCH 27/30] xstring: remove memcpies in `_INSERT_STRING_ANNOTATION` mode --- stl/inc/xstring | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 972dd900e7e..99e4e7eb234 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -3169,6 +3169,7 @@ private: auto& _My_data = _Mypair._Myval2; auto& _Right_data = _Right._Mypair._Myval2; +#if !defined(_INSERT_STRING_ANNOTATION) if constexpr (_Can_memcpy_val) { #if _HAS_CXX20 if (!_STD is_constant_evaluated()) @@ -3183,24 +3184,12 @@ private: } #endif // _ITERATOR_DEBUG_LEVEL != 0 -#ifdef _INSERT_STRING_ANNOTATION - if (!_Right_data._Large_string_engaged()) { - _ASAN_STRING_REMOVE(_Right); - } -#endif // _INSERT_STRING_ANNOTATION - _Memcpy_val_from(_Right); - -#ifdef _INSERT_STRING_ANNOTATION - if (!_Right_data._Large_string_engaged()) { - _ASAN_STRING_CREATE(*this); - } -#endif // _INSERT_STRING_ANNOTATION - _Right._Tidy_init(); return; } } +#endif // !defined(_INSERT_STRING_ANNOTATION) if (_Right_data._Large_string_engaged()) { // steal buffer _Construct_in_place(_My_data._Bx._Ptr, _Right_data._Bx._Ptr); @@ -3304,6 +3293,7 @@ private: _CONSTEXPR20 void _Copy_assign_val_from_small(const basic_string& _Right) { // TRANSITION, VSO-761321; inline into only caller when that's fixed _Tidy_deallocate(); +#if !defined(_INSERT_STRING_ANNOTATION) if constexpr (_Can_memcpy_val) { #if _HAS_CXX20 if (!_STD is_constant_evaluated()) @@ -3313,6 +3303,7 @@ private: return; } } +#endif // !defined(_INSERT_STRING_ANNOTATION) auto& _My_data = _Mypair._Myval2; auto& _Right_data = _Right._Mypair._Myval2; From ace0d8bad390c084ea7fc7b49090d8e3e485e108 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Mon, 21 Nov 2022 11:28:41 -0800 Subject: [PATCH 28/30] ughghghughuhughg --- stl/inc/xstring | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index 60aa5dd3ac8..afb04472f63 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -3197,17 +3197,24 @@ private: if (_Right_data._Large_string_engaged()) { // steal buffer _Construct_in_place(_My_data._Bx._Ptr, _Right_data._Bx._Ptr); _Right_data._Bx._Ptr = nullptr; + + _My_data._Myres = _Right_data._Myres; + _My_data._Mysize = _Right_data._Mysize; + _Swap_proxy_and_iterators(_Right); } else { // copy small string buffer _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); + _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); + _My_data._Myres = _Right_data._Myres; + _My_data._Mysize = _Right_data._Mysize; + _ASAN_STRING_CREATE(*this); + _Right_data._Orphan_all(); } - _My_data._Mysize = _Right_data._Mysize; - _My_data._Myres = _Right_data._Myres; - _ASAN_STRING_CREATE(*this); + _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Right_data); _Right._Tidy_init(); } @@ -5010,7 +5017,9 @@ private: _Traits::assign(_Mypair._Myval2._Myptr()[_Mypair._Myval2._Mysize = _New_size], _Elem()); } - _CONSTEXPR20 void _Tidy_init() noexcept { // initialize basic_string data members + _CONSTEXPR20 void _Tidy_init() noexcept { + // initialize basic_string data members + // pre: *this's SSO buffer is not annotated with ASan data. auto& _My_data = _Mypair._Myval2; _My_data._Mysize = 0; _My_data._Myres = _BUF_SIZE - 1; From 1f55a10b46264c62f9b0b758cdb496df456cf57b Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Wed, 7 Dec 2022 11:14:11 -0800 Subject: [PATCH 29/30] remove SBO annotations we've had issues with them and I do not trust them. --- stl/inc/xstring | 142 ++---------------- .../GH_002030_asan_annotate_string/test.cpp | 22 +-- 2 files changed, 19 insertions(+), 145 deletions(-) diff --git a/stl/inc/xstring b/stl/inc/xstring index e8bc0788e0f..42069a41beb 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2450,25 +2450,6 @@ private: #endif // _HAS_CXX17 #ifdef _INSERT_STRING_ANNOTATION - // this is a function so we can get the alignment of the class while defining it. - _NODISCARD static constexpr bool _Small_string_always_asan_aligned() noexcept { - constexpr bool _Aligned_first = - // The offset of the buffer is a multiple of 8 bytes from the front of the object, and - _Memcpy_val_offset % _Asan_granularity == 0 - // the front of the object is always on an 8-byte boundary. - && alignof(basic_string) >= _Asan_granularity; - - constexpr bool _Aligned_end = - // The size of the buffer is a multiple of 8 bytes, or - sizeof(value_type[_BUF_SIZE]) % _Asan_granularity == 0 - // even though the size of the buffer is not a multiple of 8 bytes, - // _Mysize must be aligned on an 8-byte boundary, - // so there's padding between the end of _Bx._Buf and _Mysize. - || alignof(size_type) >= _Asan_granularity; - - return _Aligned_first && _Aligned_end; - } - _CONSTEXPR20 void _Create_annotation() const noexcept { // Annotates the valid range with shadow memory auto& _My_data = _Mypair._Myval2; @@ -2490,70 +2471,6 @@ private: _Apply_annotation(_My_data._Myptr(), _My_data._Myres, _Old_size, _New_size); } - _CONSTEXPR20 static void _Remove_sso_annotation(const _Scary_val& _My_data) noexcept { - // Removes annotation of the SSO buffer with shadow memory. - // This results in the ability to copy into the SSO buffer without worrying. -#if _HAS_CXX20 - if (_STD is_constant_evaluated()) { - return; - } -#endif // _HAS_CXX20 - const void* const _My_buf = _My_data._Bx._Buf; - const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; - - if constexpr (_Small_string_always_asan_aligned()) { - // old state: - // [_My_buf, _End) unknown - // new state: - // [_My_buf, _End) valid - // [_End, asan_aligned_after(_End)) poison - _CSTD __sanitizer_annotate_contiguous_container(_My_buf, _STD _Get_asan_aligned_after(_End), _My_buf, _End); - } else { - const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); - // old state: - // [_Aligned._First, _My_buf) valid - // [_My_buf, _Aligned._End) unknown - // [_Aligned._End, _End) valid - // new state: - // [_Aligned._First, _End) valid - _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _My_buf, _Aligned._End); - } - } - - _CONSTEXPR20 static void _Annotate_switch_to_large(const _Scary_val& _My_data) noexcept { - // annotates exactly the pointer as valid -#if _HAS_CXX20 - if (_STD is_constant_evaluated()) { - return; - } -#endif // _HAS_CXX20 - - _Remove_sso_annotation(_My_data); - - const void* const _My_buf = _My_data._Bx._Buf; - const void* const _Ptr_last = _STD addressof(_My_data._Bx._Ptr) + 1; - const void* const _End = _My_data._Bx._Buf + _BUF_SIZE; - - if constexpr (_Small_string_always_asan_aligned()) { - // old state: - // [_My_buf, _End) valid (from _Remove_sso_annotation) - // new state: - // [_My_buf, _Ptr_last) valid - // [_Ptr_last, asan_aligned_after(_End)) poison - _CSTD __sanitizer_annotate_contiguous_container( - _My_buf, _STD _Get_asan_aligned_after(_End), _End, _Ptr_last); - } else { - const auto _Aligned = _STD _Get_asan_aligned_first_end(_My_buf, _End); - // old state: - // [_Aligned._First, _End) valid (from _Remove_sso_annotation) - // new state: - // [_Aligned._First, _Ptr_last) valid - // [_Ptr_last, _Aligned._End) poison - // [_Aligned._End, _End) valid - _CSTD __sanitizer_annotate_contiguous_container(_Aligned._First, _Aligned._End, _Aligned._End, _Ptr_last); - } - } - static _CONSTEXPR20 void _Apply_annotation(const value_type* const _First, const size_type _Capacity, const size_type _Old_size, const size_type _New_size) noexcept { #if _HAS_CXX20 @@ -2561,6 +2478,10 @@ private: return; } #endif // _HAS_CXX20 + // Don't annotate small strings; only annotate on the heap. + if (_Capacity == _BUF_SIZE - 1) { + return; + } if (!_Asan_string_should_annotate) { return; @@ -2571,26 +2492,17 @@ private: const void* const _Old_last = _First + _Old_size + 1; const void* const _New_last = _First + _New_size + 1; - _Asan_aligned_pointers _Aligned; - constexpr bool _Large_string_always_asan_aligned = (_Container_allocation_minimum_asan_alignment) >= _Asan_granularity; // for the non-aligned buffer options, the buffer must always have size >= 9 bytes, // so it will always end at least one shadow memory section. - if (_Capacity == _BUF_SIZE - 1) { - if constexpr (_Small_string_always_asan_aligned()) { - _Aligned = {_First, _STD _Get_asan_aligned_after(_End)}; - } else { - _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); - } + _Asan_aligned_pointers _Aligned; + if constexpr (_Large_string_always_asan_aligned) { + _Aligned = {_First, _STD _Get_asan_aligned_after(_End)}; } else { - if constexpr (_Large_string_always_asan_aligned) { - _Aligned = {_First, _STD _Get_asan_aligned_after(_End)}; - } else { - _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); - } + _Aligned = _STD _Get_asan_aligned_first_end(_First, _End); } const void* const _Old_fixed = _Aligned._Clamp_to_end(_Old_last); const void* const _New_fixed = _Aligned._Clamp_to_end(_New_last); @@ -2618,14 +2530,10 @@ private: #define _ASAN_STRING_REMOVE(_Str) (_Str)._Remove_annotation() #define _ASAN_STRING_CREATE(_Str) (_Str)._Create_annotation() #define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) (_Str)._Modify_annotation(_Old_size, _New_size) -#define _ASAN_STRING_SWITCH_TO_LARGE(_Scary) _Annotate_switch_to_large(_Scary) -#define _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Scary) _Remove_sso_annotation(_Scary) #else // ^^^ _INSERT_STRING_ANNOTATION / !_INSERT_STRING_ANNOTATION vvv #define _ASAN_STRING_REMOVE(_Str) #define _ASAN_STRING_CREATE(_Str) #define _ASAN_STRING_MODIFY(_Str, _Old_size, _New_size) -#define _ASAN_STRING_SWITCH_TO_LARGE(_Scary) -#define _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Scary) #endif // !_INSERT_STRING_ANNOTATION public: @@ -2764,10 +2672,8 @@ private: _Container_proxy_ptr<_Alty> _Proxy(_Alproxy, _My_data); if (_Count < _BUF_SIZE) { - _My_data._Myres = _BUF_SIZE - 1; - _My_data._Mysize = _Count; - _ASAN_STRING_CREATE(*this); + _My_data._Myres = _BUF_SIZE - 1; if constexpr (_Strat == _Construct_strategy::_From_char) { _Traits::assign(_My_data._Bx._Buf, _Count, _Arg); @@ -2848,8 +2754,6 @@ private: } } - _ASAN_STRING_REMOVE(*this); - _Tidy_deallocate_guard _Guard{this}; for (; _First != _Last; ++_First) { if constexpr (!is_same_v<_Size, size_type>) { @@ -2872,7 +2776,6 @@ private: _Al.deallocate(_My_data._Bx._Ptr, _My_data._Myres + 1); _My_data._Bx._Ptr = _New_ptr; } else { - _ASAN_STRING_SWITCH_TO_LARGE(_My_data); _Construct_in_place(_My_data._Bx._Ptr, _New_ptr); } _My_data._Myres = _New_capacity; @@ -3083,7 +2986,6 @@ public: _My_data._Myres = _Res - 1; _ASAN_STRING_CREATE(*this); } else { - _ASAN_STRING_MODIFY(*this, _My_data._Mysize, _Size); _Traits::copy(_My_data._Bx._Buf, _Right, _Res); _My_data._Mysize = _Size; _My_data._Myres = _BUF_SIZE - 1; @@ -3203,18 +3105,15 @@ private: _Swap_proxy_and_iterators(_Right); } else { // copy small string buffer - _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); - _My_data._Myres = _Right_data._Myres; + _My_data._Myres = _BUF_SIZE - 1; _My_data._Mysize = _Right_data._Mysize; - _ASAN_STRING_CREATE(*this); _Right_data._Orphan_all(); } - _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Right_data); _Right._Tidy_init(); } @@ -3321,7 +3220,6 @@ private: _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); _My_data._Mysize = _Right_data._Mysize; _My_data._Myres = _Right_data._Myres; - _ASAN_STRING_CREATE(*this); } public: @@ -4433,11 +4331,9 @@ public: const pointer _Ptr = _Starts_large._Bx._Ptr; _Destroy_in_place(_Starts_large._Bx._Ptr); - _ASAN_STRING_REMOVE_SSO_ANNOTATION(_Starts_large); _Starts_large._Activate_SSO_buffer(); _Traits::copy(_Starts_large._Bx._Buf, _Starts_small._Bx._Buf, _Starts_small._Mysize + 1); - _ASAN_STRING_SWITCH_TO_LARGE(_Starts_small); _Construct_in_place(_Starts_small._Bx._Ptr, _Ptr); } @@ -4475,8 +4371,6 @@ public: } else if (_Right_large) { // swap small with large _Swap_bx_large_with_small(_Right_data, _My_data); } else { - _ASAN_STRING_REMOVE(*this); - _ASAN_STRING_REMOVE(_Right); _Elem _Temp_buf[_BUF_SIZE]; _Traits::copy(_Temp_buf, _My_data._Bx._Buf, _My_data._Mysize + 1); _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); @@ -4485,15 +4379,6 @@ public: _STD swap(_My_data._Mysize, _Right_data._Mysize); _STD swap(_My_data._Myres, _Right_data._Myres); - - // intentionally swapped, since if `_Right_large` before, now `*this` is large (and vice versa). - if (!_Right_large) { - _ASAN_STRING_CREATE(*this); - } - - if (!_My_large) { - _ASAN_STRING_CREATE(_Right); - } } _CONSTEXPR20 void swap(basic_string& _Right) noexcept /* strengthened */ { @@ -5004,12 +4889,10 @@ private: const pointer _Ptr = _My_data._Bx._Ptr; auto& _Al = _Getal(); _Destroy_in_place(_My_data._Bx._Ptr); - _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); _Traits::copy(_My_data._Bx._Buf, _Unfancy(_Ptr), _My_data._Mysize + 1); _Al.deallocate(_Ptr, _My_data._Myres + 1); _My_data._Myres = _BUF_SIZE - 1; - _ASAN_STRING_CREATE(*this); } _CONSTEXPR20 void _Eos(const size_type _New_size) { // set new length and null terminator @@ -5019,7 +4902,6 @@ private: _CONSTEXPR20 void _Tidy_init() noexcept { // initialize basic_string data members - // pre: *this's SSO buffer is not annotated with ASan data. auto& _My_data = _Mypair._Myval2; _My_data._Mysize = 0; _My_data._Myres = _BUF_SIZE - 1; @@ -5027,18 +4909,16 @@ private: // the _Traits::assign is last so the codegen doesn't think the char write can alias this _Traits::assign(_My_data._Bx._Buf[0], _Elem()); - _ASAN_STRING_CREATE(*this); } _CONSTEXPR20 void _Tidy_deallocate() noexcept { // initialize buffer, deallocating any storage auto& _My_data = _Mypair._Myval2; _My_data._Orphan_all(); - _ASAN_STRING_REMOVE(*this); if (_My_data._Large_string_engaged()) { + _ASAN_STRING_REMOVE(*this); const pointer _Ptr = _My_data._Bx._Ptr; auto& _Al = _Getal(); _Destroy_in_place(_My_data._Bx._Ptr); - _ASAN_STRING_REMOVE_SSO_ANNOTATION(_My_data); _My_data._Activate_SSO_buffer(); _Al.deallocate(_Ptr, _My_data._Myres + 1); } diff --git a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp index 280adfe7aa0..dcafe097fea 100644 --- a/tests/std/tests/GH_002030_asan_annotate_string/test.cpp +++ b/tests/std/tests/GH_002030_asan_annotate_string/test.cpp @@ -175,27 +175,21 @@ class input_iterator_tester { template bool verify_string(const basic_string, Alloc>& str) { #ifdef __SANITIZE_ADDRESS__ + if (str.capacity() == max_sso_size) { + return true; + } + const void* const buffer = str.data(); const void* const buf_end = str.data() + str.capacity() + 1; - _Asan_aligned_pointers aligned; constexpr bool _Large_string_always_aligned = (_Container_allocation_minimum_asan_alignment>) >= 8; - constexpr bool _Small_string_always_aligned = alignof(CharType*) >= 8 || alignof(CharType) >= 8; - if (str.capacity() == max_sso_size) { - // SSO - if constexpr (_Small_string_always_aligned) { - aligned = {buffer, _Get_asan_aligned_after(buf_end)}; - } else { - aligned = _Get_asan_aligned_first_end(buffer, buf_end); - } + _Asan_aligned_pointers aligned; + if constexpr (_Large_string_always_aligned) { + aligned = {buffer, _Get_asan_aligned_after(buf_end)}; } else { - if constexpr (_Large_string_always_aligned) { - aligned = {buffer, _Get_asan_aligned_after(buf_end)}; - } else { - aligned = _Get_asan_aligned_first_end(buffer, buf_end); - } + aligned = _Get_asan_aligned_first_end(buffer, buf_end); } assert(aligned._First != aligned._End); From e38dd3525307295b491dd1493aad46a7f7470a33 Mon Sep 17 00:00:00 2001 From: Nicole Mazzuca Date: Tue, 13 Dec 2022 09:10:22 -0800 Subject: [PATCH 30/30] minor CRs --- .../__msvc_sanitizer_annotate_container.hpp | 15 +++++++------- stl/inc/xstring | 20 ++++++------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/stl/inc/__msvc_sanitizer_annotate_container.hpp b/stl/inc/__msvc_sanitizer_annotate_container.hpp index fd196e64563..db8cbd852f7 100644 --- a/stl/inc/__msvc_sanitizer_annotate_container.hpp +++ b/stl/inc/__msvc_sanitizer_annotate_container.hpp @@ -18,14 +18,14 @@ _STL_DISABLE_CLANG_WARNINGS #if !defined(_M_CEE_PURE) && !(defined(_DISABLE_STRING_ANNOTATION) && defined(_DISABLE_VECTOR_ANNOTATION)) -#if defined(__SANITIZE_ADDRESS__) +#ifdef __SANITIZE_ADDRESS__ #define _ACTIVATE_STRING_ANNOTATION #define _INSERT_STRING_ANNOTATION #define _ACTIVATE_VECTOR_ANNOTATION #define _INSERT_VECTOR_ANNOTATION -#elif defined(__clang__) && defined(__has_feature) // ^^^ __SANITIZE_ADDRESS__ / __clang__ vvv +#elif defined(__clang__) // ^^^ __SANITIZE_ADDRESS__ / __clang__ vvv #if __has_feature(address_sanitizer) #define _ACTIVATE_STRING_ANNOTATION @@ -46,19 +46,19 @@ _STL_DISABLE_CLANG_WARNINGS #endif // __SANITIZE_ADDRESS__ -#if defined(_DISABLE_STRING_ANNOTATION) +#ifdef _DISABLE_STRING_ANNOTATION #undef _ACTIVATE_STRING_ANNOTATION #undef _INSERT_STRING_ANNOTATION #endif // _DISABLE_STRING_ANNOTATION -#if defined(_DISABLE_VECTOR_ANNOTATION) +#ifdef _DISABLE_VECTOR_ANNOTATION #undef _ACTIVATE_VECTOR_ANNOTATION #undef _INSERT_VECTOR_ANNOTATION #endif // _DISABLE_VECTOR_ANNOTATION -#if !defined(_INSERT_STRING_ANNOTATION) +#ifndef _INSERT_STRING_ANNOTATION #pragma detect_mismatch("annotate_string", "0") #endif // !_INSERT_STRING_ANNOTATION -#if !defined(_INSERT_VECTOR_ANNOTATION) +#ifndef _INSERT_VECTOR_ANNOTATION #pragma detect_mismatch("annotate_vector", "0") #endif // !_INSERT_VECTOR_ANNOTATION @@ -71,7 +71,6 @@ _STL_DISABLE_CLANG_WARNINGS #pragma detect_mismatch("annotate_vector", "1") #endif // _ACTIVATE_VECTOR_ANNOTATION -// only _INSERT_*_ANNOTATION should be used from here on #undef _ACTIVATE_STRING_ANNOTATION #undef _ACTIVATE_VECTOR_ANNOTATION @@ -91,7 +90,7 @@ void __cdecl __sanitizer_annotate_contiguous_container( const void* _First, const void* _End, const void* _Old_last, const void* _New_last); } -#if defined(_M_ARM64EC) +#ifdef _M_ARM64EC #pragma comment(linker, \ "/alternatename:#__sanitizer_annotate_contiguous_container=#__sanitizer_annotate_contiguous_container_default") #pragma comment(linker, \ diff --git a/stl/inc/xstring b/stl/inc/xstring index 42069a41beb..4230d32a064 100644 --- a/stl/inc/xstring +++ b/stl/inc/xstring @@ -2479,11 +2479,7 @@ private: } #endif // _HAS_CXX20 // Don't annotate small strings; only annotate on the heap. - if (_Capacity == _BUF_SIZE - 1) { - return; - } - - if (!_Asan_string_should_annotate) { + if (_Capacity == _BUF_SIZE - 1 || !_Asan_string_should_annotate) { return; } @@ -3099,21 +3095,17 @@ private: if (_Right_data._Large_string_engaged()) { // steal buffer _Construct_in_place(_My_data._Bx._Ptr, _Right_data._Bx._Ptr); _Right_data._Bx._Ptr = nullptr; - - _My_data._Myres = _Right_data._Myres; - _My_data._Mysize = _Right_data._Mysize; - _Swap_proxy_and_iterators(_Right); } else { // copy small string buffer _My_data._Activate_SSO_buffer(); - _Traits::copy(_My_data._Bx._Buf, _Right_data._Bx._Buf, _Right_data._Mysize + 1); - _My_data._Myres = _BUF_SIZE - 1; - _My_data._Mysize = _Right_data._Mysize; - _Right_data._Orphan_all(); } + _My_data._Myres = _Right_data._Myres; + _My_data._Mysize = _Right_data._Mysize; + + _Right._Tidy_init(); } @@ -4332,7 +4324,7 @@ public: _Destroy_in_place(_Starts_large._Bx._Ptr); _Starts_large._Activate_SSO_buffer(); - _Traits::copy(_Starts_large._Bx._Buf, _Starts_small._Bx._Buf, _Starts_small._Mysize + 1); + _Traits::copy(_Starts_large._Bx._Buf, _Starts_small._Bx._Buf, _BUF_SIZE); _Construct_in_place(_Starts_small._Bx._Ptr, _Ptr); }