[libc++] Optimize input_iterator-pair insert for std::vector (#113768)

As a follow-up to #113852, this PR optimizes the performance of the
`insert(const_iterator pos, InputIt first, InputIt last)` function for
`input_iterator`-pair inputs in `std::vector` for cases where
reallocation occurs during insertion. Additionally, this optimization
enhances exception safety by replacing the traditional `try-catch`
mechanism with a modern exception guard for the `insert` function.

The optimization targets cases where insertion trigger reallocation. In
scenarios without reallocation, the implementation remains unchanged.

Previous implementation
-----------------------
The previous implementation of `insert` is inefficient in reallocation
scenarios because it performs the following steps separately:
- `reserve()`: This leads to the first round of relocating old
elements to new memory;
- `rotate()`: This leads to the second round of reorganizing the
existing elements;
- Move-forward: Moves the elements after the insertion position to
their final positions.
- Insert: performs the actual insertion.

This approach results in a lot of redundant operations, requiring the
elements to undergo three rounds of relocations/reorganizations to be
placed in their final positions.

Proposed implementation
-----------------------
The proposed implementation jointly optimize the above 4 steps in the
previous implementation such that each element is placed in its final
position in just one round of relocation. Specifically, this
optimization reduces the total cost from 2 relocations + 1 std::rotate
call to just 1 relocation, without needing to call `std::rotate`,
thereby significantly improving overall performance.

Author: winner245

libcxx/docs/ReleaseNotes/20.rst

@@ -69,9 +69,13 @@ Improvements and New Features
 - The ``_LIBCPP_ABI_BOUNDED_ITERATORS_IN_STD_ARRAY`` ABI configuration was added, which allows storing valid bounds
   in ``std::array::iterator`` and detecting OOB accesses when the appropriate hardening mode is enabled.
 
-- The input iterator overload of `assign(_InputIterator, _InputIterator)` in `std::vector<_Tp, _Allocator>` has been
-  optimized, resulting in a performance improvement of up to 2x for trivial element types (e.g., `std::vector<int>`),
-  and up to 3.4x for non-trivial element types (e.g., `std::vector<std::vector<int>>`).
+- The ``input_iterator``-pair overload of ``void assign(InputIt, InputIt)`` has been optimized for ``std::vector``, 
+  resulting in a performance improvement of up to 2x for trivial element types (e.g., ``std::vector<int>``), and up 
+  to 3.4x for non-trivial element types (e.g., ``std::vector<std::vector<int>>``).
+
+- The ``input_iterator``-pair overload of ``iterator insert(const_iterator, InputIt, InputIt)`` has been optimized 
+  for ``std::vector``, resulting in a performance improvement of up to 10x for ``std::vector<int>``, and up to 2.3x 
+  for ``std::vector<std::vector<int>>``.
 
 - On Windows, ``<system_error>``'s ``std::system_category`` is now distinct from ``std::generic_category``. The behavior
   on other operating systems is unchanged.

libcxx/include/__vector/vector.h

@@ -1250,30 +1250,30 @@ vector<_Tp, _Allocator>::__insert_with_sentinel(const_iterator __position, _Inpu
   difference_type __off = __position - begin();
   pointer __p           = this->__begin_ + __off;
   pointer __old_last    = this->__end_;
-  for (; this->__end_ != this->__cap_ && __first != __last; ++__first) {
+  for (; this->__end_ != this->__cap_ && __first != __last; ++__first)
     __construct_one_at_end(*__first);
+
+  if (__first == __last)
+    (void)std::rotate(__p, __old_last, this->__end_);
+  else {
+    __split_buffer<value_type, allocator_type&> __v(__alloc_);
+    auto __guard = std::__make_exception_guard(
+        _AllocatorDestroyRangeReverse<allocator_type, pointer>(__alloc_, __old_last, this->__end_));
+    __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
+    __split_buffer<value_type, allocator_type&> __merged(
+        __recommend(size() + __v.size()), __off, __alloc_); // has `__off` positions available at the front
+    std::__uninitialized_allocator_relocate(
+        __alloc_, std::__to_address(__old_last), std::__to_address(this->__end_), std::__to_address(__merged.__end_));
+    __guard.__complete(); // Release the guard once objects in [__old_last_, __end_) have been successfully relocated.
+    __merged.__end_ += this->__end_ - __old_last;
+    this->__end_ = __old_last;
+    std::__uninitialized_allocator_relocate(
+        __alloc_, std::__to_address(__v.__begin_), std::__to_address(__v.__end_), std::__to_address(__merged.__end_));
+    __merged.__end_ += __v.size();
+    __v.__end_ = __v.__begin_;
+    __p        = __swap_out_circular_buffer(__merged, __p);
   }
-  __split_buffer<value_type, allocator_type&> __v(this->__alloc_);
-  if (__first != __last) {
-#if _LIBCPP_HAS_EXCEPTIONS
-    try {
-#endif // _LIBCPP_HAS_EXCEPTIONS
-      __v.__construct_at_end_with_sentinel(std::move(__first), std::move(__last));
-      difference_type __old_size = __old_last - this->__begin_;
-      difference_type __old_p    = __p - this->__begin_;
-      reserve(__recommend(size() + __v.size()));
-      __p        = this->__begin_ + __old_p;
-      __old_last = this->__begin_ + __old_size;
-#if _LIBCPP_HAS_EXCEPTIONS
-    } catch (...) {
-      erase(__make_iter(__old_last), end());
-      throw;
-    }
-#endif // _LIBCPP_HAS_EXCEPTIONS
-  }
-  __p = std::rotate(__p, __old_last, this->__end_);
-  insert(__make_iter(__p), std::make_move_iterator(__v.begin()), std::make_move_iterator(__v.end()));
-  return begin() + __off;
+  return __make_iter(__p);
 }
 
 template <class _Tp, class _Allocator>

libcxx/test/benchmarks/GenerateInput.h

@@ -134,7 +134,14 @@ std::vector<std::vector<IntT>> getRandomIntegerInputsWithLength(std::size_t N, s
   return inputs;
 }
 
-inline std::vector<std::string> getPrefixedRandomStringInputs(std::size_t N) {
+inline std::vector<std::string> getSSORandomStringInputs(size_t N) {
+  std::vector<std::string> inputs;
+  for (size_t i = 0; i < N; ++i)
+    inputs.push_back(getRandomString(10)); // SSO
+  return inputs;
+}
+
+inline std::vector<std::string> getPrefixedRandomStringInputs(size_t N) {
   std::vector<std::string> inputs;
   inputs.reserve(N);
   constexpr int kSuffixLength = 32;

libcxx/test/benchmarks/containers/ContainerBenchmarks.h

@@ -135,6 +135,66 @@ void BM_InsertValueRehash(benchmark::State& st, Container c, GenInputs gen) {
   }
 }
 
+template <class Container, class GenInputs>
+void BM_Insert_InputIterIter_NoRealloc(benchmark::State& st, Container c, GenInputs gen) {
+  auto in = gen(st.range(0));
+  DoNotOptimizeData(in);
+  const auto size = c.size();
+  const auto beg  = cpp17_input_iterator(in.begin());
+  const auto end  = cpp17_input_iterator(in.end());
+  c.reserve(size + in.size()); // force no reallocation
+  for (auto _ : st) {
+    benchmark::DoNotOptimize(&(*c.insert(c.begin(), beg, end)));
+    st.PauseTiming();
+    c.erase(c.begin() + size, c.end()); // avoid the container to grow indefinitely
+    st.ResumeTiming();
+    DoNotOptimizeData(c);
+    benchmark::ClobberMemory();
+  }
+}
+
+template <class Container, class GenInputs>
+void BM_Insert_InputIterIter_Realloc_HalfFilled(benchmark::State& st, Container, GenInputs gen) {
+  const auto size = st.range(0);
+  Container a     = gen(size);
+  Container in    = gen(size + 10);
+  DoNotOptimizeData(a);
+  DoNotOptimizeData(in);
+  const auto beg = cpp17_input_iterator(in.begin());
+  const auto end = cpp17_input_iterator(in.end());
+  for (auto _ : st) {
+    st.PauseTiming();
+    Container c;
+    c.reserve(size * 2); // Reallocation with half-filled container
+    c = a;
+    st.ResumeTiming();
+    benchmark::DoNotOptimize(&(*c.insert(c.begin(), beg, end)));
+    DoNotOptimizeData(c);
+    benchmark::ClobberMemory();
+  }
+}
+
+template <class Container, class GenInputs>
+void BM_Insert_InputIterIter_Realloc_NearFull(benchmark::State& st, Container, GenInputs gen) {
+  const auto size = st.range(0);
+  Container a     = gen(size);
+  Container in    = gen(10);
+  DoNotOptimizeData(a);
+  DoNotOptimizeData(in);
+  const auto beg = cpp17_input_iterator(in.begin());
+  const auto end = cpp17_input_iterator(in.end());
+  for (auto _ : st) {
+    st.PauseTiming();
+    Container c;
+    c.reserve(size + 5); // Reallocation almost-full container
+    c = a;
+    st.ResumeTiming();
+    benchmark::DoNotOptimize(&(*c.insert(c.begin(), beg, end)));
+    DoNotOptimizeData(c);
+    benchmark::ClobberMemory();
+  }
+}
+
 template <class Container, class GenInputs>
 void BM_InsertDuplicate(benchmark::State& st, Container c, GenInputs gen) {
   auto in        = gen(st.range(0));

libcxx/test/benchmarks/containers/vector_operations.bench.cpp

@@ -91,4 +91,18 @@ BENCHMARK_CAPTURE(BM_AssignInputIterIter<100>,
                   getRandomIntegerInputsWithLength<int>)
     ->Args({TestNumInputs, TestNumInputs});
 
+BENCHMARK_CAPTURE(BM_Insert_InputIterIter_NoRealloc, vector_int, std::vector<int>(100, 1), getRandomIntegerInputs<int>)
+    ->Arg(514048);
+BENCHMARK_CAPTURE(
+    BM_Insert_InputIterIter_Realloc_HalfFilled, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
+    ->Arg(514048);
+BENCHMARK_CAPTURE(BM_Insert_InputIterIter_Realloc_NearFull, vector_int, std::vector<int>{}, getRandomIntegerInputs<int>)
+    ->Arg(514048);
+BENCHMARK_CAPTURE(
+    BM_Insert_InputIterIter_Realloc_HalfFilled, vector_string, std::vector<std::string>{}, getSSORandomStringInputs)
+    ->Arg(514048);
+BENCHMARK_CAPTURE(
+    BM_Insert_InputIterIter_Realloc_NearFull, vector_string, std::vector<std::string>{}, getSSORandomStringInputs)
+    ->Arg(514048);
+
 BENCHMARK_MAIN();