[libc++] Optimize multi{map,set}::insert(InputIterator, InputIterator)

libcxx/docs/ReleaseNotes/22.rst

@@ -51,6 +51,9 @@ Improvements and New Features
 - The performance of ``map::erase`` and ``set::erase`` has been improved by up to 2x
 - The performance of ``find(key)`` in ``map``, ``set``, ``multimap`` and ``multiset`` has been improved by up to 2.3x
 - Some reallocations are now avoided in `std::filesystem::path::lexically_relative`, resulting in a performance improvement of up to 1.7x.
+- The performance of the ``(iterator, iterator)`` constructors of ``multimap`` and ``multiset``
+  has been improved by up to 3x
+- The performance of ``insert(iterator, iterator)`` of ``multimap`` and ``multiset`` has been improved by up to 2.5x
 
 Deprecations and Removals
 -------------------------

libcxx/include/__tree

@@ -1005,6 +1005,35 @@ public:
     __emplace_hint_multi(__p, std::move(__value));
   }
 
+  template <class _InIter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI void __insert_range_multi(_InIter __first, _Sent __last) {
+    if (__first == __last)
+      return;
+
+    if (__root() == nullptr) { // Make sure we always have a root node
+      __insert_node_at(
+          __end_node(), __end_node()->__left_, static_cast<__node_base_pointer>(__construct_node(*__first).release()));
+      ++__first;
+    }
+
+    auto __max_node = static_cast<__node_pointer>(std::__tree_max(static_cast<__node_base_pointer>(__root())));
+
+    for (; __first != __last; ++__first) {
+      __node_holder __nd = __construct_node(*__first);
+      // Always check the max node first. This optimizes for sorted ranges inserted at the end.
+      if (!value_comp()(__nd->__get_value(), __max_node->__get_value())) { // __node >= __max_val
+        __insert_node_at(static_cast<__end_node_pointer>(__max_node),
+                         __max_node->__right_,
+                         static_cast<__node_base_pointer>(__nd.get()));
+        __max_node = __nd.release();
+      } else {
+        __end_node_pointer __parent;
+        __node_base_pointer& __child = __find_leaf_high(__parent, __nd->__get_value());
+        __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__nd.release()));
+      }
+    }
+  }
+
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __node_assign_unique(const value_type& __v, __node_pointer __dest);
 
   _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);

libcxx/include/map

@@ -593,6 +593,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/unique_ptr.h>
 #  include <__memory_resource/polymorphic_allocator.h>
 #  include <__node_handle>
+#  include <__ranges/access.h>
 #  include <__ranges/concepts.h>
 #  include <__ranges/container_compatible_range.h>
 #  include <__ranges/from_range.h>
@@ -1750,17 +1751,13 @@ public:
 
   template <class _InputIterator>
   _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
-    for (const_iterator __e = cend(); __f != __l; ++__f)
-      __tree_.__emplace_hint_multi(__e.__i_, *__f);
+    __tree_.__insert_range_multi(__f, __l);
   }
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
-    const_iterator __end = cend();
-    for (auto&& __element : __range) {
-      __tree_.__emplace_hint_multi(__end.__i_, std::forward<decltype(__element)>(__element));
-    }
+    __tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 

libcxx/include/set

@@ -530,6 +530,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/allocator_traits.h>
 #  include <__memory_resource/polymorphic_allocator.h>
 #  include <__node_handle>
+#  include <__ranges/access.h>
 #  include <__ranges/concepts.h>
 #  include <__ranges/container_compatible_range.h>
 #  include <__ranges/from_range.h>
@@ -1205,18 +1206,14 @@ public:
   }
 
   template <class _InputIterator>
-  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
-    for (const_iterator __e = cend(); __f != __l; ++__f)
-      __tree_.__emplace_hint_multi(__e, *__f);
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+    __tree_.__insert_range_multi(__first, __last);
   }
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
-    const_iterator __end = cend();
-    for (auto&& __element : __range) {
-      __tree_.__emplace_hint_multi(__end, std::forward<decltype(__element)>(__element));
-    }
+    __tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
   }
 #  endif
 

libcxx/test/std/containers/associative/multimap/multimap.modifiers/insert_iter_iter.pass.cpp

@@ -13,88 +13,195 @@
 // template <class InputIterator>
 //   void insert(InputIterator first, InputIterator last);
 
-#include <map>
+#include <array>
 #include <cassert>
+#include <map>
 
-#include "test_macros.h"
-#include "test_iterators.h"
 #include "min_allocator.h"
+#include "test_iterators.h"
+#include "test_macros.h"
 
-int main(int, char**) {
-  {
-    typedef std::multimap<int, double> M;
-    typedef std::pair<int, double> P;
-    P ar[] = {
-        P(1, 1),
-        P(1, 1.5),
-        P(1, 2),
-        P(2, 1),
-        P(2, 1.5),
-        P(2, 2),
-        P(3, 1),
-        P(3, 1.5),
-        P(3, 2),
-    };
-    M m;
-    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
-    assert(m.size() == 9);
-    assert(m.begin()->first == 1);
-    assert(m.begin()->second == 1);
-    assert(std::next(m.begin())->first == 1);
-    assert(std::next(m.begin())->second == 1.5);
-    assert(std::next(m.begin(), 2)->first == 1);
-    assert(std::next(m.begin(), 2)->second == 2);
-    assert(std::next(m.begin(), 3)->first == 2);
-    assert(std::next(m.begin(), 3)->second == 1);
-    assert(std::next(m.begin(), 4)->first == 2);
-    assert(std::next(m.begin(), 4)->second == 1.5);
-    assert(std::next(m.begin(), 5)->first == 2);
-    assert(std::next(m.begin(), 5)->second == 2);
-    assert(std::next(m.begin(), 6)->first == 3);
-    assert(std::next(m.begin(), 6)->second == 1);
-    assert(std::next(m.begin(), 7)->first == 3);
-    assert(std::next(m.begin(), 7)->second == 1.5);
-    assert(std::next(m.begin(), 8)->first == 3);
-    assert(std::next(m.begin(), 8)->second == 2);
+template <class Iter, class Alloc>
+void test_alloc() {
+  {   // Check that an empty range works correctly
+    { // Without elements in the container
+      using Map = std::multimap<int, int, std::less<int>, Alloc>;
+
+      std::array<std::pair<const int, int>, 0> arr;
+
+      Map map;
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 0);
+      assert(map.begin() == map.end());
+    }
+    { // With 1 element in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      std::array<Pair, 0> arr;
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 1);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(std::next(map.begin(), 1) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      std::array<Pair, 0> arr;
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Pair(1, 1));
+      map.insert(Pair(2, 2));
+      map.insert(Iter(arr.data()), Iter(arr.data() + arr.size()));
+      assert(map.size() == 3);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(2, 2));
+      assert(std::next(map.begin(), 3) == map.end());
+    }
   }
-#if TEST_STD_VER >= 11
-  {
-    typedef std::multimap<int, double, std::less<int>, min_allocator<std::pair<const int, double>>> M;
-    typedef std::pair<int, double> P;
-    P ar[] = {
-        P(1, 1),
-        P(1, 1.5),
-        P(1, 2),
-        P(2, 1),
-        P(2, 1.5),
-        P(2, 2),
-        P(3, 1),
-        P(3, 1.5),
-        P(3, 2),
-    };
-    M m;
-    m.insert(cpp17_input_iterator<P*>(ar), cpp17_input_iterator<P*>(ar + sizeof(ar) / sizeof(ar[0])));
-    assert(m.size() == 9);
-    assert(m.begin()->first == 1);
-    assert(m.begin()->second == 1);
-    assert(std::next(m.begin())->first == 1);
-    assert(std::next(m.begin())->second == 1.5);
-    assert(std::next(m.begin(), 2)->first == 1);
-    assert(std::next(m.begin(), 2)->second == 2);
-    assert(std::next(m.begin(), 3)->first == 2);
-    assert(std::next(m.begin(), 3)->second == 1);
-    assert(std::next(m.begin(), 4)->first == 2);
-    assert(std::next(m.begin(), 4)->second == 1.5);
-    assert(std::next(m.begin(), 5)->first == 2);
-    assert(std::next(m.begin(), 5)->second == 2);
-    assert(std::next(m.begin(), 6)->first == 3);
-    assert(std::next(m.begin(), 6)->second == 1);
-    assert(std::next(m.begin(), 7)->first == 3);
-    assert(std::next(m.begin(), 7)->second == 1.5);
-    assert(std::next(m.begin(), 8)->first == 3);
-    assert(std::next(m.begin(), 8)->second == 2);
+  {   // Check that 1 element is inserted correctly
+    { // Without elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 1);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(std::next(map.begin(), 1) == map.end());
+    }
+    { // With 1 element in the container - a different key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 2);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(std::next(map.begin(), 2) == map.end());
+    }
+    { // With 1 element in the container - the same key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Pair(1, 1));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 2);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(std::next(map.begin(), 2) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Pair(1, 1));
+      map.insert(Pair(2, 2));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(1, 1));
+      assert(*std::next(map.begin(), 3) == Pair(2, 2));
+      assert(std::next(map.begin(), 4) == map.end());
+    }
   }
-#endif
+  {   // Check that multiple elements are inserted correctly
+    { // Without elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(1, 1), Pair(3, 3)};
+
+      Map map;
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 3);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(3, 3));
+      assert(std::next(map.begin(), 3) == map.end());
+    }
+    { // With 1 element in the container - a different key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(1, 1), Pair(3, 3)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(1, 1));
+      assert(*std::next(map.begin(), 3) == Pair(3, 3));
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+    { // With 1 element in the container - the same key
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(2, 2), Pair(3, 3)};
+
+      Map map;
+      map.insert(Pair(1, 1));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 4);
+      assert(*std::next(map.begin(), 0) == Pair(1, 1));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(2, 2));
+      assert(*std::next(map.begin(), 3) == Pair(3, 3));
+      assert(std::next(map.begin(), 4) == map.end());
+    }
+    { // With multiple elements in the container
+      using Map  = std::multimap<int, int, std::less<int>, Alloc>;
+      using Pair = std::pair<const int, int>;
+
+      Pair arr[] = {Pair(1, 1), Pair(3, 3), Pair(4, 4)};
+
+      Map map;
+      map.insert(Pair(0, 0));
+      map.insert(Pair(1, 1));
+      map.insert(Pair(2, 2));
+      map.insert(Iter(std::begin(arr)), Iter(std::end(arr)));
+      assert(map.size() == 6);
+      assert(*std::next(map.begin(), 0) == Pair(0, 0));
+      assert(*std::next(map.begin(), 1) == Pair(1, 1));
+      assert(*std::next(map.begin(), 2) == Pair(1, 1));
+      assert(*std::next(map.begin(), 3) == Pair(2, 2));
+      assert(*std::next(map.begin(), 4) == Pair(3, 3));
+      assert(*std::next(map.begin(), 5) == Pair(4, 4));
+      assert(std::next(map.begin(), 6) == map.end());
+    }
+  }
+}
+
+void test() {
+  test_alloc<cpp17_input_iterator<std::pair<const int, int>*>, std::allocator<std::pair<const int, int> > >();
+  test_alloc<cpp17_input_iterator<std::pair<const int, int>*>, min_allocator<std::pair<const int, int> > >();
+}
+
+int main(int, char**) {
+  test();
 
   return 0;
 }