[libc++] Optimize copy construction and assignment of __tree

Author: philnik777

libcxx/include/__tree

@@ -1213,6 +1213,86 @@ private:
     __node_pointer __cache_root_;
     __node_pointer __cache_elem_;
   };
+
+  class __tree_deleter {
+    __node_allocator& __alloc_;
+
+  public:
+    using pointer = __node_pointer;
+
+    _LIBCPP_HIDE_FROM_ABI __tree_deleter(__node_allocator& __alloc) : __alloc_(__alloc) {}
+
+    void operator()(__node_pointer __ptr) {
+      if (!__ptr)
+        return;
+
+      (*this)(static_cast<__node_pointer>(__ptr->__left_));
+
+      auto __right = __ptr->__right_;
+
+      __node_traits::destroy(__alloc_, std::addressof(__ptr->__value_));
+      __node_traits::deallocate(__alloc_, __ptr, 1);
+
+      (*this)(static_cast<__node_pointer>(__right));
+    }
+  };
+
+  class __allocate_node_builder {
+    __tree& __base_;
+
+  public:
+    __allocate_node_builder(__tree& __base) : __base_(__base) {}
+
+    template <class _Up>
+    __node_holder operator()(_Up&& __v) {
+      return __base_.__construct_node(std::forward<_Up>(__v));
+    }
+  };
+
+  class __cache_node_builder {
+    __tree& __base_;
+    _DetachedTreeCache& __cache_;
+
+  public:
+    __cache_node_builder(__tree& __base, _DetachedTreeCache& __cache) : __base_(__base), __cache_(__cache) {}
+
+    template <class _Up>
+    __node_holder operator()(_Up&& __v) {
+      __node_pointer __ret = __cache_.__get();
+      if (__ret) {
+        __assign_value(__ret->__value_, std::forward<_Up>(__v));
+        __cache_.__advance();
+      }
+      return __node_holder(__ret, _Dp(__base_.__node_alloc_));
+    }
+  };
+
+  template <class _NodeBuilder>
+  _LIBCPP_HIDE_FROM_ABI __node_pointer __copy_tree(_NodeBuilder& __builder, __node_pointer __src) {
+    if (!__src)
+      return nullptr;
+
+    __node_holder __new_node = __builder(__src->__value_);
+    if (!__new_node) {
+      __allocate_node_builder __alloc_builder(*this);
+      return __copy_tree(__alloc_builder, __src);
+    }
+
+    unique_ptr<__node, __tree_deleter> __left(
+        __copy_tree(__builder, static_cast<__node_pointer>(__src->__left_)), __node_alloc_);
+    __node_pointer __right = __copy_tree(__builder, static_cast<__node_pointer>(__src->__right_));
+
+    __node_pointer __new_node_ptr = __new_node.release();
+
+    __new_node_ptr->__is_black_ = __src->__is_black_;
+    __new_node_ptr->__left_     = static_cast<__node_base_pointer>(__left.release());
+    __new_node_ptr->__right_    = static_cast<__node_base_pointer>(__right);
+    if (__new_node_ptr->__left_)
+      __new_node_ptr->__left_->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
+    if (__right)
+      __right->__parent_ = static_cast<__end_node_pointer>(__new_node_ptr);
+    return __new_node_ptr;
+  }
 };
 
 template <class _Tp, class _Compare, class _Allocator>
@@ -1277,11 +1357,31 @@ __tree<_Tp, _Compare, _Allocator>::_DetachedTreeCache::__detach_next(__node_poin
 
 template <class _Tp, class _Compare, class _Allocator>
 __tree<_Tp, _Compare, _Allocator>& __tree<_Tp, _Compare, _Allocator>::operator=(const __tree& __t) {
-  if (this != std::addressof(__t)) {
-    value_comp() = __t.value_comp();
-    __copy_assign_alloc(__t);
-    __assign_multi(__t.begin(), __t.end());
+  if (this == std::addressof(__t))
+    return *this;
+
+  value_comp() = __t.value_comp();
+  __copy_assign_alloc(__t);
+
+  if (__t.size() == 0) {
+    clear();
+    return *this;
+  }
+
+  if (__size_ != 0) {
+    _DetachedTreeCache __cache(this);
+    __cache_node_builder __builder(*this, __cache);
+    __end_node_.__left_ =
+        static_cast<__node_base_pointer>(__copy_tree(__builder, static_cast<__node_pointer>(__t.__end_node_.__left_)));
+  } else {
+    __allocate_node_builder __builder(*this);
+    __end_node_.__left_ =
+        static_cast<__node_base_pointer>(__copy_tree(__builder, static_cast<__node_pointer>(__t.__end_node_.__left_)));
   }
+  __end_node_.__left_->__parent_ = __end_node();
+  __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
+  __size_       = __t.__size_;
+
   return *this;
 }
 
@@ -1327,11 +1427,19 @@ void __tree<_Tp, _Compare, _Allocator>::__assign_multi(_InputIterator __first, _
 
 template <class _Tp, class _Compare, class _Allocator>
 __tree<_Tp, _Compare, _Allocator>::__tree(const __tree& __t)
-    : __begin_node_(),
+    : __begin_node_(__end_node()),
       __node_alloc_(__node_traits::select_on_container_copy_construction(__t.__node_alloc())),
       __size_(0),
       __value_comp_(__t.value_comp()) {
-  __begin_node_ = __end_node();
+  if (__t.__size_ == 0)
+    return;
+
+  __allocate_node_builder __builder(*this);
+  __end_node_.__left_ =
+      static_cast<__node_base_pointer>(__copy_tree(__builder, static_cast<__node_pointer>(__t.__end_node_.__left_)));
+  __end_node_.__left_->__parent_ = __end_node();
+  __begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(static_cast<__node_base_pointer>(__end_node())));
+  __size_       = __t.__size_;
 }
 
 template <class _Tp, class _Compare, class _Allocator>
@@ -1430,13 +1538,7 @@ __tree<_Tp, _Compare, _Allocator>::~__tree() {
 
 template <class _Tp, class _Compare, class _Allocator>
 void __tree<_Tp, _Compare, _Allocator>::destroy(__node_pointer __nd) _NOEXCEPT {
-  if (__nd != nullptr) {
-    destroy(static_cast<__node_pointer>(__nd->__left_));
-    destroy(static_cast<__node_pointer>(__nd->__right_));
-    __node_allocator& __na = __node_alloc();
-    __node_traits::destroy(__na, std::addressof(__nd->__value_));
-    __node_traits::deallocate(__na, __nd, 1);
-  }
+  (__tree_deleter(__node_alloc_))(__nd);
 }
 
 template <class _Tp, class _Compare, class _Allocator>

libcxx/include/map

@@ -970,7 +970,7 @@ public:
       : map(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI map(const map& __m) : __tree_(__m.__tree_) { insert(__m.begin(), __m.end()); }
+  _LIBCPP_HIDE_FROM_ABI map(const map& __m) = default;
 
   _LIBCPP_HIDE_FROM_ABI map& operator=(const map& __m) = default;
 
@@ -1637,11 +1637,7 @@ public:
       : multimap(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m)
-      : __tree_(__m.__tree_.value_comp(),
-                __alloc_traits::select_on_container_copy_construction(__m.__tree_.__alloc())) {
-    insert(__m.begin(), __m.end());
-  }
+  _LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m) = default;
 
   _LIBCPP_HIDE_FROM_ABI multimap& operator=(const multimap& __m) = default;
 

libcxx/include/set

@@ -660,7 +660,7 @@ public:
       : set(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI set(const set& __s) : __tree_(__s.__tree_) { insert(__s.begin(), __s.end()); }
+  _LIBCPP_HIDE_FROM_ABI set(const set& __s) = default;
 
   _LIBCPP_HIDE_FROM_ABI set& operator=(const set& __s) = default;
 
@@ -1119,11 +1119,7 @@ public:
       : multiset(from_range, std::forward<_Range>(__range), key_compare(), __a) {}
 #  endif
 
-  _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s)
-      : __tree_(__s.__tree_.value_comp(),
-                __alloc_traits::select_on_container_copy_construction(__s.__tree_.__alloc())) {
-    insert(__s.begin(), __s.end());
-  }
+  _LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s) = default;
 
   _LIBCPP_HIDE_FROM_ABI multiset& operator=(const multiset& __s) = default;
 

libcxx/test/benchmarks/containers/associative/associative_container_benchmarks.h

@@ -151,7 +151,7 @@ void associative_container_benchmarks(std::string container) {
   /////////////////////////
   // Assignment
   /////////////////////////
-  bench("operator=(const&)", [=](auto& st) {
+  bench("operator=(const&) (into cleared Container)", [=](auto& st) {
     const std::size_t size = st.range(0);
     std::vector<Value> in  = make_value_types(generate_unique_keys(size));
     Container src(in.begin(), in.end());
@@ -172,6 +172,21 @@ void associative_container_benchmarks(std::string container) {
     }
   });
 
+  bench("operator=(const&) (into populated Container)", [=](auto& st) {
+    const std::size_t size = st.range(0);
+    std::vector<Value> in  = make_value_types(generate_unique_keys(size));
+    Container src(in.begin(), in.end());
+    Container c[BatchSize];
+
+    while (st.KeepRunningBatch(BatchSize)) {
+      for (std::size_t i = 0; i != BatchSize; ++i) {
+        c[i] = src;
+        benchmark::DoNotOptimize(c[i]);
+        benchmark::ClobberMemory();
+      }
+    }
+  });
+
   /////////////////////////
   // Insertion
   /////////////////////////

libcxx/test/benchmarks/containers/associative/map.bench.cpp

@@ -29,6 +29,7 @@ struct support::adapt_operations<std::map<K, V>> {
 
 int main(int argc, char** argv) {
   support::associative_container_benchmarks<std::map<int, int>>("std::map<int, int>");
+  support::associative_container_benchmarks<std::map<std::string, int>>("std::map<std::string, int>");
 
   benchmark::Initialize(&argc, argv);
   benchmark::RunSpecifiedBenchmarks();