[libc++] Optimize multi{map,set}::insert(InputIterator, InputIterator)

[philnik777]

--------------------------------------------------------------------------------------------------------------------
Benchmark                                                                                        old             new
--------------------------------------------------------------------------------------------------------------------
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/0                           14.8 ns         16.0 ns
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/32                           513 ns          536 ns
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/1024                       50523 ns        49099 ns
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/8192                      712956 ns       773317 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/0                             14.6 ns         16.0 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/32                             505 ns          404 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/1024                         24907 ns        14673 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/8192                        317355 ns       115852 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/0                               449 ns          458 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/32                             1036 ns          965 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/1024                          29504 ns        20190 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/8192                         313616 ns       150854 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/0                              456 ns          459 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/32                            1013 ns          971 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/1024                         41227 ns        24008 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/8192                        478994 ns       343475 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/0                    459 ns          456 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/32                   732 ns          720 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/1024               10027 ns         8581 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/8192               74175 ns        65850 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/0                   15.2 ns         17.1 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/32                  3093 ns         3015 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/1024              164606 ns       178851 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/8192             2239105 ns      2458905 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/0                     15.1 ns         16.9 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/32                    1293 ns         2497 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/1024                 94936 ns        68618 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/8192               1252379 ns      1136337 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/0                       456 ns          463 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/32                     2024 ns         1807 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/1024                 159767 ns       105691 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/8192                1424748 ns       762697 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/0                      461 ns          458 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/32                    2013 ns         2040 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/1024                166583 ns       141373 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/8192               1566078 ns      1358940 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/0            454 ns          454 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/32           841 ns          813 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/1024       18878 ns        20836 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/8192      136735 ns       135926 ns

[github-actions]

✅ With the latest revision this PR passed the C/C++ code formatter.

[ldionne]

As you suggested, let's see what happens if we drop the hint here.

[ldionne]

Let's document what this function does and in particular what optimizations it performs (i.e. it is optimized for already-sorted inputs).

[ldionne]

Looks like you forgot to update multimap. Please also include the numbers for multimap in the PR description.

[ldionne]

I would also suggest taking a tag here and handling the (very similar) unique/multi cases uniformly with a if constexpr. Then we can report improvements for map and set as well.

[llvmbot]

@llvm/pr-subscribers-libcxx

Author: Nikolas Klauser (philnik777)

Changes

--------------------------------------------------------------------------------------------------------------------
Benchmark                                                                                        old             new
--------------------------------------------------------------------------------------------------------------------
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/0                           14.8 ns         16.0 ns
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/32                           513 ns          536 ns
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/1024                       50523 ns        49099 ns
std::multiset<int>::ctor(iterator, iterator) (unsorted sequence)/8192                      712956 ns       773317 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/0                             14.6 ns         16.0 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/32                             505 ns          404 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/1024                         24907 ns        14673 ns
std::multiset<int>::ctor(iterator, iterator) (sorted sequence)/8192                        317355 ns       115852 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/0                               449 ns          458 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/32                             1036 ns          965 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/1024                          29504 ns        20190 ns
std::multiset<int>::insert(iterator, iterator) (all new keys)/8192                         313616 ns       150854 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/0                              456 ns          459 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/32                            1013 ns          971 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/1024                         41227 ns        24008 ns
std::multiset<int>::insert(iterator, iterator) (half new keys)/8192                        478994 ns       343475 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/0                    459 ns          456 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/32                   732 ns          720 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/1024               10027 ns         8581 ns
std::multiset<int>::erase(iterator, iterator) (erase half the container)/8192               74175 ns        65850 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/0                   15.2 ns         17.1 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/32                  3093 ns         3015 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/1024              164606 ns       178851 ns
std::multiset<std::string>::ctor(iterator, iterator) (unsorted sequence)/8192             2239105 ns      2458905 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/0                     15.1 ns         16.9 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/32                    1293 ns         2497 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/1024                 94936 ns        68618 ns
std::multiset<std::string>::ctor(iterator, iterator) (sorted sequence)/8192               1252379 ns      1136337 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/0                       456 ns          463 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/32                     2024 ns         1807 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/1024                 159767 ns       105691 ns
std::multiset<std::string>::insert(iterator, iterator) (all new keys)/8192                1424748 ns       762697 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/0                      461 ns          458 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/32                    2013 ns         2040 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/1024                166583 ns       141373 ns
std::multiset<std::string>::insert(iterator, iterator) (half new keys)/8192               1566078 ns      1358940 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/0            454 ns          454 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/32           841 ns          813 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/1024       18878 ns        20836 ns
std::multiset<std::string>::erase(iterator, iterator) (erase half the container)/8192      136735 ns       135926 ns

---

Full diff: https://github.com/llvm/llvm-project/pull/152691.diff

3 Files Affected:

• (modified) libcxx/include/__tree (+30)
• (modified) libcxx/include/map (+1)
• (modified) libcxx/include/set (+4-7)

diff --git a/libcxx/include/__tree b/libcxx/include/__tree
index 6dadd0915c984..5baadca719ac4 100644
--- a/libcxx/include/__tree
+++ b/libcxx/include/__tree
@@ -970,6 +970,36 @@ public:
     __emplace_hint_multi(__p, std::move(__value));
   }
 
+  template <class _InIter, class _Sent>
+  _LIBCPP_HIDE_FROM_ABI void __insert_range_multi(_InIter __first, _Sent __last) {
+    if (__first == __last)
+      return;
+
+    if (__root() == nullptr) { // Make sure we always have a root node
+      __node_holder __node = __construct_node(*__first);
+      __insert_node_at(__end_node(), __end_node()->__left_, static_cast<__node_base_pointer>(__node.release()));
+      ++__first;
+    }
+
+    auto __max_node = static_cast<__node_pointer>(std::__tree_max(static_cast<__node_base_pointer>(__root())));
+    __node_pointer __last_insertion = __root();
+
+    for (; __first != __last; ++__first) {
+      __node_holder __node = __construct_node(*__first);
+      if (!value_comp()(__node->__get_value(), __max_node->__get_value())) { // __node >= __max_val
+        __insert_node_at(static_cast<__end_node_pointer>(__max_node),
+                         __max_node->__right_,
+                         static_cast<__node_base_pointer>(__node.get()));
+        __max_node = __node.release();
+      } else {
+        __end_node_pointer __parent;
+        __node_base_pointer& __child = __find_leaf(++iterator(__last_insertion), __parent, __node->__value_);
+        __last_insertion             = __node.get();
+        __insert_node_at(__parent, __child, static_cast<__node_base_pointer>(__node.release()));
+      }
+    }
+  }
+
   _LIBCPP_HIDE_FROM_ABI pair<iterator, bool> __node_assign_unique(const value_type& __v, __node_pointer __dest);
 
   _LIBCPP_HIDE_FROM_ABI iterator __node_insert_multi(__node_pointer __nd);
diff --git a/libcxx/include/map b/libcxx/include/map
index 9f98abef9afe0..a84b5592244fa 100644
--- a/libcxx/include/map
+++ b/libcxx/include/map
@@ -593,6 +593,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/unique_ptr.h>
 #  include <__memory_resource/polymorphic_allocator.h>
 #  include <__node_handle>
+#  include <__ranges/access.h>
 #  include <__ranges/concepts.h>
 #  include <__ranges/container_compatible_range.h>
 #  include <__ranges/from_range.h>
diff --git a/libcxx/include/set b/libcxx/include/set
index c77345bc5dc1f..18488b56e7dd2 100644
--- a/libcxx/include/set
+++ b/libcxx/include/set
@@ -530,6 +530,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred);  // C++20
 #  include <__memory/allocator_traits.h>
 #  include <__memory_resource/polymorphic_allocator.h>
 #  include <__node_handle>
+#  include <__ranges/access.h>
 #  include <__ranges/concepts.h>
 #  include <__ranges/container_compatible_range.h>
 #  include <__ranges/from_range.h>
@@ -1205,18 +1206,14 @@ public:
   }
 
   template <class _InputIterator>
-  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __f, _InputIterator __l) {
-    for (const_iterator __e = cend(); __f != __l; ++__f)
-      __tree_.__emplace_hint_multi(__e, *__f);
+  _LIBCPP_HIDE_FROM_ABI void insert(_InputIterator __first, _InputIterator __last) {
+    __tree_.__insert_range_multi(__first, __last);
   }
 
 #  if _LIBCPP_STD_VER >= 23
   template <_ContainerCompatibleRange<value_type> _Range>
   _LIBCPP_HIDE_FROM_ABI void insert_range(_Range&& __range) {
-    const_iterator __end = cend();
-    for (auto&& __element : __range) {
-      __tree_.__emplace_hint_multi(__end, std::forward<decltype(__element)>(__element));
-    }
+    __tree_.__insert_range_multi(ranges::begin(__range), ranges::end(__range));
   }
 #  endif