[libc++] Pass type information down to __libcpp_allocate

Currently, places where we call __libcpp_allocate must drop type information
on the ground even when they actually have such information available. That
is unfortunate since some toolchains and system allocators are able to
provide improved security when they know what type is being allocated.

This is the purpose of http://wg21.link/p2719, where we introduce a new
variant of `operator new` which takes a type in its interface. A different
but related issue is that `std::allocator` does not honor any in-class
`T::operator new` since it is specified to call the global `::operator new`
instead.

This patch closes the gap to make it trivial for implementations that
provide typed memory allocators to actually benefit from that information
in more contexts, and also makes libc++ forward-compatible with future
proposals that would fix the existing defects in `std::allocator`.

Since this is a widely-used function and making this a template could
have an impact on debug info sizes, I tried minimizing the number of
templated layers by removing `__do_deallocate_handle_size`, which was
easy to replace with a macro (and IMO this leads to cleaner code).

We could also explore using `_LIBCPP_NODEBUG` on `__libcpp_allocate`
and friends if that proves to be a problem.

Author: ldionne

libcxx/include/CMakeLists.txt

@@ -548,7 +548,6 @@ set(files
   __memory/array_cookie.h
   __memory/assume_aligned.h
   __memory/auto_ptr.h
-  __memory/builtin_new_allocator.h
   __memory/compressed_pair.h
   __memory/concepts.h
   __memory/construct_at.h

libcxx/include/__functional/function.h

@@ -22,7 +22,6 @@
 #include <__memory/allocator.h>
 #include <__memory/allocator_destructor.h>
 #include <__memory/allocator_traits.h>
-#include <__memory/builtin_new_allocator.h>
 #include <__memory/compressed_pair.h>
 #include <__memory/unique_ptr.h>
 #include <__type_traits/aligned_storage.h>
@@ -213,8 +212,10 @@ class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> {
   }
 
   _LIBCPP_HIDE_FROM_ABI __default_alloc_func* __clone() const {
-    __builtin_new_allocator::__holder_t __hold = __builtin_new_allocator::__allocate_type<__default_alloc_func>(1);
-    __default_alloc_func* __res                = ::new ((void*)__hold.get()) __default_alloc_func(__f_);
+    using _Self = __default_alloc_func;
+    unique_ptr<_Self, __deallocating_deleter<_Self>> __hold =
+        std::__libcpp_allocate<_Self>(sizeof(_Self), _LIBCPP_ALIGNOF(_Self));
+    _Self* __res = ::new ((void*)__hold.get()) _Self(__f_);
     (void)__hold.release();
     return __res;
   }
@@ -223,7 +224,7 @@ class __default_alloc_func<_Fp, _Rp(_ArgTypes...)> {
 
   _LIBCPP_HIDE_FROM_ABI static void __destroy_and_delete(__default_alloc_func* __f) {
     __f->destroy();
-    __builtin_new_allocator::__deallocate_type<__default_alloc_func>(__f, 1);
+    __deallocating_deleter<__default_alloc_func>()(__f);
   }
 };
 
@@ -669,8 +670,9 @@ class __policy_func<_Rp(_ArgTypes...)> {
       if (__use_small_storage<_Fun>()) {
         ::new ((void*)&__buf_.__small) _Fun(std::move(__f));
       } else {
-        __builtin_new_allocator::__holder_t __hold = __builtin_new_allocator::__allocate_type<_Fun>(1);
-        __buf_.__large                             = ::new ((void*)__hold.get()) _Fun(std::move(__f));
+        unique_ptr<_Fun, __deallocating_deleter<_Fun>> __hold =
+            std::__libcpp_allocate<_Fun>(sizeof(_Fun), _LIBCPP_ALIGNOF(_Fun));
+        __buf_.__large = ::new ((void*)__hold.get()) _Fun(std::move(__f));
         (void)__hold.release();
       }
     }

libcxx/include/__memory/allocator.h

@@ -101,7 +101,7 @@ class _LIBCPP_TEMPLATE_VIS allocator : private __non_trivial_if<!is_void<_Tp>::v
     if (__libcpp_is_constant_evaluated()) {
       return static_cast<_Tp*>(::operator new(__n * sizeof(_Tp)));
     } else {
-      return static_cast<_Tp*>(std::__libcpp_allocate(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)));
+      return static_cast<_Tp*>(std::__libcpp_allocate<_Tp>(__n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp)));
     }
   }
 
@@ -116,7 +116,7 @@ class _LIBCPP_TEMPLATE_VIS allocator : private __non_trivial_if<!is_void<_Tp>::v
     if (__libcpp_is_constant_evaluated()) {
       ::operator delete(__p);
     } else {
-      std::__libcpp_deallocate((void*)__p, __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
+      std::__libcpp_deallocate<_Tp>((void*)__p, __n * sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
     }
   }
 

libcxx/include/__memory/builtin_new_allocator.h

@@ -39,7 +39,7 @@ struct __temporary_buffer_deleter {
       return;
     }
 
-    std::__libcpp_deallocate_unsized((void*)__ptr, _LIBCPP_ALIGNOF(_Tp));
+    std::__libcpp_deallocate_unsized<_Tp>((void*)__ptr, _LIBCPP_ALIGNOF(_Tp));
   }
 };
 

libcxx/include/__memory/unique_temporary_buffer.h

@@ -29,7 +29,7 @@ _LIBCPP_CONSTEXPR inline _LIBCPP_HIDE_FROM_ABI bool __is_overaligned_for_new(siz
 #endif
 }
 
-template <class... _Args>
+template <class _Tp, class... _Args>
 _LIBCPP_HIDE_FROM_ABI void* __libcpp_operator_new(_Args... __args) {
 #if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
   return __builtin_operator_new(__args...);
@@ -38,7 +38,7 @@ _LIBCPP_HIDE_FROM_ABI void* __libcpp_operator_new(_Args... __args) {
 #endif
 }
 
-template <class... _Args>
+template <class _Tp, class... _Args>
 _LIBCPP_HIDE_FROM_ABI void __libcpp_operator_delete(_Args... __args) _NOEXCEPT {
 #if __has_builtin(__builtin_operator_new) && __has_builtin(__builtin_operator_delete)
   __builtin_operator_delete(__args...);
@@ -47,52 +47,61 @@ _LIBCPP_HIDE_FROM_ABI void __libcpp_operator_delete(_Args... __args) _NOEXCEPT {
 #endif
 }
 
+template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI void* __libcpp_allocate(size_t __size, size_t __align) {
 #if _LIBCPP_HAS_ALIGNED_ALLOCATION
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
-    return __libcpp_operator_new(__size, __align_val);
+    return std::__libcpp_operator_new<_Tp>(__size, __align_val);
   }
 #endif
 
   (void)__align;
-  return __libcpp_operator_new(__size);
+  return std::__libcpp_operator_new<_Tp>(__size);
 }
 
-template <class... _Args>
-_LIBCPP_HIDE_FROM_ABI void __do_deallocate_handle_size(void* __ptr, size_t __size, _Args... __args) _NOEXCEPT {
-#if !_LIBCPP_HAS_SIZED_DEALLOCATION
-  (void)__size;
-  return std::__libcpp_operator_delete(__ptr, __args...);
+#if _LIBCPP_HAS_SIZED_DEALLOCATION
+#  define _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(...) __VA_ARGS__
 #else
-  return std::__libcpp_operator_delete(__ptr, __size, __args...);
+#  define _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(...) /* nothing */
 #endif
-}
 
+template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate(void* __ptr, size_t __size, size_t __align) _NOEXCEPT {
+  (void)__size;
 #if !_LIBCPP_HAS_ALIGNED_ALLOCATION
   (void)__align;
-  return __do_deallocate_handle_size(__ptr, __size);
+  return std::__libcpp_operator_delete<_Tp>(__ptr _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(, __size));
 #else
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
-    return __do_deallocate_handle_size(__ptr, __size, __align_val);
+    return std::__libcpp_operator_delete<_Tp>(__ptr _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(, __size), __align_val);
   } else {
-    return __do_deallocate_handle_size(__ptr, __size);
+    return std::__libcpp_operator_delete<_Tp>(__ptr _LIBCPP_ONLY_IF_SIZED_DEALLOCATION(, __size));
   }
 #endif
 }
 
+template <class _Tp>
+struct __deallocating_deleter {
+  _LIBCPP_HIDE_FROM_ABI void operator()(void* __p) const {
+    std::__libcpp_deallocate<_Tp>(__p, sizeof(_Tp), _LIBCPP_ALIGNOF(_Tp));
+  }
+};
+
+#undef _LIBCPP_ONLY_IF_SIZED_DEALLOCATION
+
+template <class _Tp>
 inline _LIBCPP_HIDE_FROM_ABI void __libcpp_deallocate_unsized(void* __ptr, size_t __align) _NOEXCEPT {
 #if !_LIBCPP_HAS_ALIGNED_ALLOCATION
   (void)__align;
-  return __libcpp_operator_delete(__ptr);
+  return std::__libcpp_operator_delete<_Tp>(__ptr);
 #else
   if (__is_overaligned_for_new(__align)) {
     const align_val_t __align_val = static_cast<align_val_t>(__align);
-    return __libcpp_operator_delete(__ptr, __align_val);
+    return std::__libcpp_operator_delete<_Tp>(__ptr, __align_val);
   } else {
-    return __libcpp_operator_delete(__ptr);
+    return std::__libcpp_operator_delete<_Tp>(__ptr);
   }
 #endif
 }

libcxx/include/__new/allocate.h

@@ -67,7 +67,7 @@ class __small_buffer {
     if constexpr (__fits_in_buffer<_Stored>) {
       return std::launder(reinterpret_cast<_Stored*>(__buffer_));
     } else {
-      byte* __allocation = static_cast<byte*>(std::__libcpp_allocate(sizeof(_Stored), alignof(_Stored)));
+      byte* __allocation = static_cast<byte*>(std::__libcpp_allocate<_Stored>(sizeof(_Stored), alignof(_Stored)));
       std::construct_at(reinterpret_cast<byte**>(__buffer_), __allocation);
       return std::launder(reinterpret_cast<_Stored*>(__allocation));
     }
@@ -76,7 +76,7 @@ class __small_buffer {
   template <class _Stored>
   _LIBCPP_HIDE_FROM_ABI void __dealloc() noexcept {
     if constexpr (!__fits_in_buffer<_Stored>)
-      std::__libcpp_deallocate(*reinterpret_cast<void**>(__buffer_), sizeof(_Stored), alignof(_Stored));
+      std::__libcpp_deallocate<_Stored>(*reinterpret_cast<void**>(__buffer_), sizeof(_Stored), alignof(_Stored));
   }
 
   template <class _Stored, class... _Args>

libcxx/include/__utility/small_buffer.h

@@ -1534,7 +1534,6 @@ module std [system] {
     module array_cookie                       { header "__memory/array_cookie.h" }
     module assume_aligned                     { header "__memory/assume_aligned.h" }
     module auto_ptr                           { header "__memory/auto_ptr.h" }
-    module builtin_new_allocator              { header "__memory/builtin_new_allocator.h" }
     module compressed_pair                    { header "__memory/compressed_pair.h" }
     module concepts                           { header "__memory/concepts.h" }
     module construct_at                       { header "__memory/construct_at.h" }

libcxx/include/module.modulemap

@@ -41,20 +41,22 @@ static bool is_aligned_to(void* ptr, size_t align) {
 class _LIBCPP_EXPORTED_FROM_ABI __new_delete_memory_resource_imp : public memory_resource {
   void* do_allocate(size_t bytes, size_t align) override {
 #if _LIBCPP_HAS_ALIGNED_ALLOCATION
-    return std::__libcpp_allocate(bytes, align);
+    return std::__libcpp_allocate<std::byte>(bytes, align);
 #else
     if (bytes == 0)
       bytes = 1;
-    void* result = std::__libcpp_allocate(bytes, align);
+    void* result = std::__libcpp_allocate<std::byte>(bytes, align);
     if (!is_aligned_to(result, align)) {
-      std::__libcpp_deallocate(result, bytes, align);
+      std::__libcpp_deallocate<std::byte>(result, bytes, align);
       __throw_bad_alloc();
     }
     return result;
 #endif
   }
 
-  void do_deallocate(void* p, size_t bytes, size_t align) override { std::__libcpp_deallocate(p, bytes, align); }
+  void do_deallocate(void* p, size_t bytes, size_t align) override {
+    std::__libcpp_deallocate<std::byte>(p, bytes, align);
+  }
 
   bool do_is_equal(const memory_resource& other) const noexcept override { return &other == this; }
 };