[WIP] Implement std::vector operations in terms of relocation

Author: ldionne

libcxx/docs/ReleaseNotes/20.rst

@@ -52,6 +52,9 @@ Improvements and New Features
 - The ``lexicographical_compare`` and ``ranges::lexicographical_compare`` algorithms have been optimized for trivially
   equality comparable types, resulting in a performance improvement of up to 40x.
 
+- The ``std::vector::erase`` function has been optimized for types that can be relocated trivially (such as ``std::string``),
+  yielding speed ups witnessed to be around 2x for these types (but subject to the use case).
+
 - The ``_LIBCPP_ENABLE_CXX20_REMOVED_TEMPORARY_BUFFER`` macro has been added to make ``std::get_temporary_buffer`` and
   ``std::return_temporary_buffer`` available.
 

libcxx/include/__memory/temp_value.h

@@ -21,6 +21,24 @@
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
+template <class _Tp>
+struct __temporary_emplace_value {
+  union {
+    _Tp __value_;
+  };
+
+  template <class _Allocator, class... _Args>
+  _LIBCPP_HIDE_FROM_ABI
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __temporary_emplace_value(_Allocator& __alloc, _Args&&... __args) {
+    allocator_traits<_Allocator>::construct(__alloc, std::addressof(__value_), std::forward<_Args>(__args)...);
+  }
+
+  // Don't destroy anything, since we assume that the value gets relocated by whoever uses this type
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 ~__temporary_emplace_value() {}
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _Tp* __address() { return std::addressof(__value_); }
+};
+
 template <class _Tp, class _Alloc>
 struct __temp_value {
   typedef allocator_traits<_Alloc> _Traits;

libcxx/include/__vector/vector.h

@@ -34,11 +34,14 @@
 #include <__memory/allocator.h>
 #include <__memory/allocator_traits.h>
 #include <__memory/compressed_pair.h>
+#include <__memory/destroy.h>
+#include <__memory/is_trivially_allocator_relocatable.h>
 #include <__memory/noexcept_move_assign_container.h>
 #include <__memory/pointer_traits.h>
 #include <__memory/swap_allocator.h>
 #include <__memory/temp_value.h>
 #include <__memory/uninitialized_algorithms.h>
+#include <__memory/uninitialized_relocate.h>
 #include <__ranges/access.h>
 #include <__ranges/concepts.h>
 #include <__ranges/container_compatible_range.h>
@@ -472,11 +475,68 @@ class _LIBCPP_TEMPLATE_VIS vector {
     this->__destruct_at_end(this->__end_ - 1);
   }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, const_reference __x) {
+    return emplace(std::move(__position), __x);
+  }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator insert(const_iterator __position, value_type&& __x) {
+    return emplace(std::move(__position), std::move(__x));
+  }
   template <class... _Args>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __position, _Args&&... __args);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator emplace(const_iterator __cposition, _Args&&... __args) {
+    iterator __position = begin() + (__cposition - cbegin());
+    if (__end_ < __cap_) {
+      if (__position == end()) {
+        allocator_traits<_Allocator>::construct(
+            __alloc_, std::__to_address(__position), std::forward<_Args>(__args)...);
+        ++__end_;
+      } else {
+        // Construct a temporary value on the stack, so that in case this throws we haven't modified
+        // the vector yet. This also takes care of the corner case where we'd be trying to insert
+        // from an element located in the vector itself, in which case we'd otherwise have to be
+        // careful about reference invalidation if we didn't make a temporary value.
+        __temporary_emplace_value<value_type> __tmp(__alloc_, std::forward<_Args>(__args)...);
+        auto __guard = std::__make_exception_guard([&] {
+          std::allocator_traits<_Allocator>::destroy(__alloc_, __tmp.__address());
+        });
+
+        // Open up a gap inside the vector by relocating everything to the right and insert the new
+        // element into the right spot.
+        //
+        // If this fails, the relocation operation guarantees that the whole tail of the vector has
+        // been destroyed. So we set the "new end" of the vector accordingly, which means we basically
+        // erased the whole tail of the vector. This provides the basic exception guarantee.
+        try {
+          std::__uninitialized_allocator_relocate_backward(__alloc_, __position, end(), __position + 1);
+        } catch (...) {
+          __end_ = __to_pointer(__position);
+          throw;
+        }
+
+        // Relocate the temporary value into its final location. If that throws, we know from __allocator_relocate_at
+        // that the temporary will have been destroyed, but we must still clear the tail of the vector, since otherwise
+        // we'd leave a gap in the middle of the vector.
+        __guard.__complete(); // we know the temporary value gets destroyed by the relocation no matter what happens
+        try {
+          std::__allocator_relocate_at(__alloc_, __tmp.__address(), std::__to_address(__position));
+        } catch (...) {
+          std::__allocator_destroy(__alloc_, __position + 1, end() + 1);
+          __end_ = __to_pointer(__position);
+          throw;
+        }
+
+        ++__end_;
+      }
+      __annotate_increase(1);
+      return __position;
+    } else {
+      __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __position - begin(), __alloc_);
+      __v.emplace_back(std::forward<_Args>(__args)...);
+      pointer __p = __to_pointer(__position);
+      __p         = __swap_out_circular_buffer(__v, __p);
+      return __make_iter(__p);
+    }
+  }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator
   insert(const_iterator __position, size_type __n, const_reference __x);
@@ -520,8 +580,40 @@ class _LIBCPP_TEMPLATE_VIS vector {
   }
 #endif
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position);
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __first, const_iterator __last);
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __position) {
+    _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
+        __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
+    return erase(__position, __position + 1);
+  }
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI iterator erase(const_iterator __cfirst, const_iterator __clast) {
+    _LIBCPP_ASSERT_VALID_INPUT_RANGE(__cfirst <= __clast, "vector::erase(first, last) called with invalid range");
+
+    iterator __first = begin() + std::distance(cbegin(), __cfirst);
+    iterator __last  = begin() + std::distance(cbegin(), __clast);
+    if (__first == __last)
+      return __last;
+
+    auto const __n = std::distance(__first, __last);
+
+    // If the value_type is nothrow relocatable, we destroy the range being erased and we relocate the tail
+    // of the vector into the created gap. This is especially efficient if the elements are trivially relocatable.
+    // Otherwise, we use the standard technique with move-assignments.
+    //
+    // Note that unlike vector::insert, we can't use relocation when it is potentially throwing, because
+    // vector::erase is required not to throw an exception unless T's assignment operator throws. So we
+    // can bypass the assignment with a relocation, but only when that definitely doesn't throw.
+    if constexpr (__is_nothrow_allocator_relocatable<_Allocator, value_type>::value) {
+      std::__allocator_destroy(__alloc_, __first, __last);
+      std::__uninitialized_allocator_relocate(__alloc_, __last, end(), __first);
+    } else {
+      auto __new_end = std::move(__last, end(), __first);
+      std::__allocator_destroy(__alloc_, __new_end, end());
+    }
+
+    __end_ -= __n;
+    __annotate_shrink(size() + __n);
+    return __first;
+  }
 
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void clear() _NOEXCEPT {
     size_type __old_size = size();
@@ -662,6 +754,11 @@ class _LIBCPP_TEMPLATE_VIS vector {
     __annotate_shrink(__old_size);
   }
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI pointer __to_pointer(iterator __it) const {
+    auto __index = __it - begin();
+    return this->__begin_ + __index;
+  }
+
   template <class... _Args>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI inline pointer __emplace_back_slow_path(_Args&&... __args);
 
@@ -1115,28 +1212,6 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 inline
 #endif
 }
 
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 inline _LIBCPP_HIDE_FROM_ABI typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::erase(const_iterator __position) {
-  _LIBCPP_ASSERT_VALID_ELEMENT_ACCESS(
-      __position != end(), "vector::erase(iterator) called with a non-dereferenceable iterator");
-  difference_type __ps = __position - cbegin();
-  pointer __p          = this->__begin_ + __ps;
-  this->__destruct_at_end(std::move(__p + 1, this->__end_, __p));
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::erase(const_iterator __first, const_iterator __last) {
-  _LIBCPP_ASSERT_VALID_INPUT_RANGE(__first <= __last, "vector::erase(first, last) called with invalid range");
-  pointer __p = this->__begin_ + (__first - begin());
-  if (__first != __last) {
-    this->__destruct_at_end(std::move(__p + (__last - __first), this->__end_, __p));
-  }
-  return __make_iter(__p);
-}
-
 template <class _Tp, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void
 vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointer __to) {
@@ -1152,68 +1227,6 @@ vector<_Tp, _Allocator>::__move_range(pointer __from_s, pointer __from_e, pointe
   std::move_backward(__from_s, __from_s + __n, __old_last);
 }
 
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, const_reference __x) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (this->__end_ < this->__cap_) {
-    if (__p == this->__end_) {
-      __construct_one_at_end(__x);
-    } else {
-      __move_range(__p, this->__end_, __p + 1);
-      const_pointer __xr = pointer_traits<const_pointer>::pointer_to(__x);
-      if (std::__is_pointer_in_range(std::__to_address(__p), std::__to_address(__end_), std::addressof(__x)))
-        ++__xr;
-      *__p = *__xr;
-    }
-  } else {
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
-    __v.emplace_back(__x);
-    __p = __swap_out_circular_buffer(__v, __p);
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::insert(const_iterator __position, value_type&& __x) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (this->__end_ < this->__cap_) {
-    if (__p == this->__end_) {
-      __construct_one_at_end(std::move(__x));
-    } else {
-      __move_range(__p, this->__end_, __p + 1);
-      *__p = std::move(__x);
-    }
-  } else {
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
-    __v.emplace_back(std::move(__x));
-    __p = __swap_out_circular_buffer(__v, __p);
-  }
-  return __make_iter(__p);
-}
-
-template <class _Tp, class _Allocator>
-template <class... _Args>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
-vector<_Tp, _Allocator>::emplace(const_iterator __position, _Args&&... __args) {
-  pointer __p = this->__begin_ + (__position - begin());
-  if (this->__end_ < this->__cap_) {
-    if (__p == this->__end_) {
-      __construct_one_at_end(std::forward<_Args>(__args)...);
-    } else {
-      __temp_value<value_type, _Allocator> __tmp(this->__alloc_, std::forward<_Args>(__args)...);
-      __move_range(__p, this->__end_, __p + 1);
-      *__p = std::move(__tmp.get());
-    }
-  } else {
-    __split_buffer<value_type, allocator_type&> __v(__recommend(size() + 1), __p - this->__begin_, this->__alloc_);
-    __v.emplace_back(std::forward<_Args>(__args)...);
-    __p = __swap_out_circular_buffer(__v, __p);
-  }
-  return __make_iter(__p);
-}
-
 template <class _Tp, class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 typename vector<_Tp, _Allocator>::iterator
 vector<_Tp, _Allocator>::insert(const_iterator __position, size_type __n, const_reference __x) {

libcxx/test/std/containers/sequences/vector/vector.modifiers/common.h

@@ -40,28 +40,6 @@ struct Throws {
 bool Throws::sThrows = false;
 #endif
 
-struct Tracker {
-  int copy_assignments = 0;
-  int move_assignments = 0;
-};
-
-struct TrackedAssignment {
-  Tracker* tracker_;
-  TEST_CONSTEXPR_CXX14 explicit TrackedAssignment(Tracker* tracker) : tracker_(tracker) {}
-
-  TrackedAssignment(TrackedAssignment const&) = default;
-  TrackedAssignment(TrackedAssignment&&)      = default;
-
-  TEST_CONSTEXPR_CXX14 TrackedAssignment& operator=(TrackedAssignment const&) {
-    tracker_->copy_assignments++;
-    return *this;
-  }
-  TEST_CONSTEXPR_CXX14 TrackedAssignment& operator=(TrackedAssignment&&) {
-    tracker_->move_assignments++;
-    return *this;
-  }
-};
-
 struct NonTriviallyRelocatable {
   int value_;
   TEST_CONSTEXPR NonTriviallyRelocatable() : value_(0) {}

libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter.pass.cpp

@@ -107,31 +107,5 @@ int main(int, char**) {
   }
 #endif
 
-  // Make sure we satisfy the complexity requirement in terms of the number of times the assignment
-  // operator is called.
-  //
-  // There is currently ambiguity as to whether this is truly mandated by the Standard, so we only
-  // test it for libc++.
-#ifdef _LIBCPP_VERSION
-  {
-    Tracker tracker;
-    std::vector<TrackedAssignment> v;
-
-    // Set up the vector with 5 elements.
-    for (int i = 0; i != 5; ++i) {
-      v.emplace_back(&tracker);
-    }
-    assert(tracker.copy_assignments == 0);
-    assert(tracker.move_assignments == 0);
-
-    // Erase element [1] from it. Elements [2] [3] [4] should be shifted, so we should
-    // see 3 move assignments (and nothing else).
-    v.erase(v.begin() + 1);
-    assert(v.size() == 4);
-    assert(tracker.copy_assignments == 0);
-    assert(tracker.move_assignments == 3);
-  }
-#endif
-
   return 0;
 }

libcxx/test/std/containers/sequences/vector/vector.modifiers/erase_iter_iter.pass.cpp

@@ -196,31 +196,5 @@ int main(int, char**) {
     assert(it == v.begin() + 2);
   }
 
-  // Make sure we satisfy the complexity requirement in terms of the number of times the assignment
-  // operator is called.
-  //
-  // There is currently ambiguity as to whether this is truly mandated by the Standard, so we only
-  // test it for libc++.
-#ifdef _LIBCPP_VERSION
-  {
-    Tracker tracker;
-    std::vector<TrackedAssignment> v;
-
-    // Set up the vector with 5 elements.
-    for (int i = 0; i != 5; ++i) {
-      v.emplace_back(&tracker);
-    }
-    assert(tracker.copy_assignments == 0);
-    assert(tracker.move_assignments == 0);
-
-    // Erase elements [1] and [2] from it. Elements [3] [4] should be shifted, so we should
-    // see 2 move assignments (and nothing else).
-    v.erase(v.begin() + 1, v.begin() + 3);
-    assert(v.size() == 3);
-    assert(tracker.copy_assignments == 0);
-    assert(tracker.move_assignments == 2);
-  }
-#endif
-
   return 0;
 }