diff --git a/libcxx/include/__configuration/abi.h b/libcxx/include/__configuration/abi.h index 2d33b9c03090b..c9936df30ff7f 100644 --- a/libcxx/include/__configuration/abi.h +++ b/libcxx/include/__configuration/abi.h @@ -30,8 +30,20 @@ #elif _LIBCPP_ABI_FORCE_MICROSOFT # define _LIBCPP_ABI_MICROSOFT #else +// Windows uses the Microsoft ABI # if defined(_WIN32) && defined(_MSC_VER) # define _LIBCPP_ABI_MICROSOFT + +// 32-bit ARM uses the Itanium ABI with a few differences (array cookies, etc), +// and so does 64-bit ARM on Apple platforms. +# elif defined(__arm__) || (defined(__APPLE__) && defined(__aarch64__)) +# define _LIBCPP_ABI_ITANIUM_WITH_ARM_DIFFERENCES + +// Non-Apple 64-bit ARM uses the vanilla Itanium ABI +# elif defined(__aarch64__) +# define _LIBCPP_ABI_ITANIUM + +// We assume that other architectures also use the vanilla Itanium ABI too # else # define _LIBCPP_ABI_ITANIUM # endif diff --git a/libcxx/include/__memory/array_cookie.h b/libcxx/include/__memory/array_cookie.h index 806a9e99ecafe..6d6240d4e0a5e 100644 --- a/libcxx/include/__memory/array_cookie.h +++ b/libcxx/include/__memory/array_cookie.h @@ -13,6 +13,7 @@ #include <__config> #include <__configuration/abi.h> #include <__cstddef/size_t.h> +#include <__memory/addressof.h> #include <__type_traits/integral_constant.h> #include <__type_traits/is_trivially_destructible.h> #include <__type_traits/negation.h> @@ -26,14 +27,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD // Trait representing whether a type requires an array cookie at the start of its allocation when // allocated as `new T[n]` and deallocated as `delete[] array`. // -// Under the Itanium C++ ABI [1], we know that an array cookie is available unless `T` is trivially -// destructible and the call to `operator delete[]` is not a sized operator delete. Under ABIs other -// than the Itanium ABI, we assume there are no array cookies. +// Under the Itanium C++ ABI [1] and the ARM ABI which derives from it, we know that an array cookie is available +// unless `T` is trivially destructible and the call to `operator delete[]` is not a sized operator delete. Under +// other ABIs, we assume there are no array cookies. // // [1]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#array-cookies -#ifdef _LIBCPP_ABI_ITANIUM +#if defined(_LIBCPP_ABI_ITANIUM) || defined(_LIBCPP_ABI_ITANIUM_WITH_ARM_DIFFERENCES) // TODO: Use a builtin instead -// TODO: We should factor in the choice of the usual deallocation function in this determination. +// TODO: We should factor in the choice of the usual deallocation function in this determination: +// a cookie may be available in more cases but we ignore those for now. template struct __has_array_cookie : _Not > {}; #else @@ -41,13 +43,84 @@ template struct __has_array_cookie : false_type {}; #endif +template sizeof(size_t))> +struct [[__gnu__::__aligned__(_LIBCPP_PREFERRED_ALIGNOF(_Tp))]] __itanium_array_cookie { + size_t __element_count; +}; + +template +struct [[__gnu__::__aligned__(_LIBCPP_PREFERRED_ALIGNOF(_Tp))]] __itanium_array_cookie<_Tp, /* _HasPadding */ true> { + char __padding[_LIBCPP_PREFERRED_ALIGNOF(_Tp) - sizeof(size_t)]; + size_t __element_count; +}; + +template +struct [[__gnu__::__aligned__(_LIBCPP_ALIGNOF(_Tp))]] __arm_array_cookie { + size_t __element_size; + size_t __element_count; +}; + +// Return the element count in the array cookie located before the given pointer. +// +// In the Itanium ABI [1] +// ---------------------- +// The element count is stored immediately before the first element of the array. If the preferred alignment +// of array elements (which is different from the ABI alignment) is more than that of size_t, additional +// padding bytes exist at the beginning of the array cookie. Assuming array elements of size and alignment 16 +// bytes, that gives us the following layout: +// +// |ooooooooxxxxxxxxaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbccccccccccccccccdddddddddddddddd| +// ^^^^^^^^ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// | ^^^^^^^^ | +// | | array elements +// padding | +// element count +// +// In practice, it is sufficient to read the bytes immediately before the first array element. +// +// +// In the Itanium ABI with ARM differences [2] +// ------------------------------------------- +// The array cookie is stored at the very start of the allocation and it has the following form: +// +// struct array_cookie { +// std::size_t element_size; // element_size != 0 +// std::size_t element_count; +// }; +// +// Assuming elements of size and alignment 32 bytes, this gives us the following layout: +// +// |xxxxxxxxXXXXXXXXooooooooooooooooaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb| +// ^^^^^^^^ ^^^^^^^^^^^^^^^^ +// | ^^^^^^^^ | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +// element size | padding | +// element count array elements +// +// We calculate the starting address of the allocation by taking into account the ABI alignment (not +// the preferred alignment) of the type. +// +// [1]: https://itanium-cxx-abi.github.io/cxx-abi/abi.html#array-cookies +// [2]: https://developer.apple.com/documentation/xcode/writing-arm64-code-for-apple-platforms#Handle-C++-differences template // Avoid failures when -fsanitize-address-poison-custom-array-cookie is enabled -_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") size_t __get_array_cookie(_Tp const* __ptr) { +_LIBCPP_HIDE_FROM_ABI _LIBCPP_NO_SANITIZE("address") size_t __get_array_cookie([[__maybe_unused__]] _Tp const* __ptr) { static_assert( __has_array_cookie<_Tp>::value, "Trying to access the array cookie of a type that is not guaranteed to have one"); - size_t const* __cookie = reinterpret_cast(__ptr) - 1; // TODO: Use a builtin instead - return *__cookie; + +#if defined(_LIBCPP_ABI_ITANIUM) + using _ArrayCookie = __itanium_array_cookie<_Tp>; +#elif defined(_LIBCPP_ABI_ITANIUM_WITH_ARM_DIFFERENCES) + using _ArrayCookie = __arm_array_cookie<_Tp>; +#else + static_assert(false, "The array cookie layout is unknown on this ABI"); +#endif + + char const* __allocation_start = reinterpret_cast(__ptr) - sizeof(_ArrayCookie); + _ArrayCookie __cookie; + // This is necessary to avoid violating strict aliasing. It's valid because _ArrayCookie is an + // implicit lifetime type. + __builtin_memcpy(std::addressof(__cookie), __allocation_start, sizeof(_ArrayCookie)); + return __cookie.__element_count; } _LIBCPP_END_NAMESPACE_STD diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp index b7cc12350027b..2de523dfb25cb 100644 --- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp +++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.class/unique.ptr.observers/assert.subscript.pass.cpp @@ -58,15 +58,18 @@ void test() { { { std::unique_ptr ptr(new WithCookie[5]); + assert(&ptr[1] == ptr.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr::operator[](index): index out of range"); } { std::unique_ptr ptr = std::make_unique(5); + assert(&ptr[1] == ptr.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr::operator[](index): index out of range"); } #if TEST_STD_VER >= 20 { std::unique_ptr ptr = std::make_unique_for_overwrite(5); + assert(&ptr[1] == ptr.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(ptr[6] = WithCookie(), "unique_ptr::operator[](index): index out of range"); } #endif @@ -82,11 +85,13 @@ void test() { { { std::unique_ptr ptr = std::make_unique(5); + assert(&ptr[1] == ptr.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(ptr[6], "unique_ptr::operator[](index): index out of range"); } # if TEST_STD_VER >= 20 { std::unique_ptr ptr = std::make_unique_for_overwrite(5); + assert(&ptr[1] == ptr.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(ptr[6] = NoCookie(), "unique_ptr::operator[](index): index out of range"); } # endif @@ -101,6 +106,7 @@ void test() { { std::unique_ptr ptr = std::make_unique(5); std::unique_ptr other(std::move(ptr)); + assert(&other[1] == other.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr::operator[](index): index out of range"); } @@ -109,6 +115,7 @@ void test() { std::unique_ptr ptr = std::make_unique(5); std::unique_ptr other; other = std::move(ptr); + assert(&other[1] == other.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr::operator[](index): index out of range"); } @@ -116,6 +123,7 @@ void test() { { std::unique_ptr ptr = std::make_unique(5); std::unique_ptr other(std::move(ptr)); + assert(&other[1] == other.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr::operator[](index): index out of range"); } @@ -124,6 +132,7 @@ void test() { std::unique_ptr ptr = std::make_unique(5); std::unique_ptr other; other = std::move(ptr); + assert(&other[1] == other.get() + 1); // ensure no assertion TEST_LIBCPP_ASSERT_FAILURE(other[6], "unique_ptr::operator[](index): index out of range"); } }); @@ -144,6 +153,20 @@ struct WithCookie { char padding[Size]; }; +template +struct alignas(128) OveralignedNoCookie { + char padding[Size]; +}; + +template +struct alignas(128) OveralignedWithCookie { + OveralignedWithCookie() = default; + OveralignedWithCookie(OveralignedWithCookie const&) {} + OveralignedWithCookie& operator=(OveralignedWithCookie const&) { return *this; } + ~OveralignedWithCookie() {} + char padding[Size]; +}; + int main(int, char**) { test, NoCookie<1>>(); test, NoCookie<2>>(); @@ -153,6 +176,16 @@ int main(int, char**) { test, NoCookie<16>>(); test, NoCookie<32>>(); test, NoCookie<256>>(); + + test, OveralignedNoCookie<1>>(); + test, OveralignedNoCookie<2>>(); + test, OveralignedNoCookie<3>>(); + test, OveralignedNoCookie<4>>(); + test, OveralignedNoCookie<8>>(); + test, OveralignedNoCookie<16>>(); + test, OveralignedNoCookie<32>>(); + test, OveralignedNoCookie<256>>(); + test(); return 0;