Speed-up and refactor move-assignment operator for vector<bool>

Author: winner245

libcxx/include/__vector/vector_bool.h

@@ -398,6 +398,8 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
     __guard.__complete();
   }
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __realloc_and_copy(const vector& __v);
+
   template <class _Iterator, class _Sentinel>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
 
@@ -527,7 +529,7 @@ vector<bool, _Allocator>::__recommend(size_type __new_size) const {
   const size_type __cap = capacity();
   if (__cap >= __ms / 2)
     return __ms;
-  return std::max(2 * __cap, __align_it(__new_size));
+  return std::max<size_type>(2 * __cap, __align_it(__new_size));
 }
 
 //  Default constructs __n objects starting at __end_
@@ -695,18 +697,26 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v
   }
 }
 
+// This function copies each storage word as a whole, which is substantially more efficient than copying
+// individual bits within each word.
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void
+vector<bool, _Allocator>::__realloc_and_copy(const vector& __v) {
+  if (__v.__size_) {
+    if (__v.__size_ > capacity()) {
+      __vdeallocate();
+      __vallocate(__v.__size_);
+    }
+    std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
+  }
+  __size_ = __v.__size_;
+}
+
 template <class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
   if (this != std::addressof(__v)) {
     __copy_assign_alloc(__v);
-    if (__v.__size_) {
-      if (__v.__size_ > capacity()) {
-        __vdeallocate();
-        __vallocate(__v.__size_);
-      }
-      std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
-    }
-    __size_ = __v.__size_;
+    __realloc_and_copy(__v);
   }
   return *this;
 }
@@ -754,7 +764,7 @@ vector<bool, _Allocator>::operator=(vector&& __v)
 template <class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
   if (__alloc_ != __c.__alloc_)
-    assign(__c.begin(), __c.end());
+    __realloc_and_copy(__c);
   else
     __move_assign(__c, true_type());
 }

libcxx/test/benchmarks/containers/ContainerBenchmarks.h

@@ -51,6 +51,20 @@ void BM_Assignment(benchmark::State& st, Container) {
   }
 }
 
+template <class Container, class Allocator>
+void BM_Move_Assignment(benchmark::State& st, Container, Allocator) {
+  auto size = st.range(0);
+  Container c1(Allocator{1});
+  Container c2(Allocator{2});
+  // c1.reserve(size);
+  c2.resize(size);
+  for (auto _ : st) {
+    c1 = std::move(c2);
+    DoNotOptimizeData(c1);
+    DoNotOptimizeData(c2);
+  }
+}
+
 template <std::size_t... sz, typename Container, typename GenInputs>
 void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
   auto v = gen(1, sz...);

libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp

@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "ContainerBenchmarks.h"
+#include "../GenerateInput.h"
+#include "sized_allocator.h"
+#include "test_allocator.h"
+
+using namespace ContainerBenchmarks;
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+                  vector_bool_uint16_t,
+                  std::vector<bool, sized_allocator<bool, std::uint16_t, std::int16_t>>{},
+                  sized_allocator<bool, std::uint16_t, std::int16_t>{})
+    ->Arg(5140480);
+
+BENCHMARK_CAPTURE(BM_Move_Assignment,
+                  vector_bool_size_t,
+                  std::vector<bool, sized_allocator<bool, std::size_t, std::ptrdiff_t>>{},
+                  sized_allocator<bool, std::size_t, std::ptrdiff_t>{})
+    ->Arg(5140480);
+
+BENCHMARK_MAIN();

libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp

@@ -10,46 +10,65 @@
 
 // vector& operator=(const vector& c);
 
-#include <vector>
 #include <cassert>
-#include "test_macros.h"
-#include "test_allocator.h"
+#include <vector>
+
 #include "min_allocator.h"
+#include "test_allocator.h"
+#include "test_macros.h"
 
-TEST_CONSTEXPR_CXX20 bool tests()
-{
-    {
-        std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
-        std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == test_allocator<bool>(3));
-    }
-    {
-        std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
-        std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == other_allocator<bool>(5));
-    }
+TEST_CONSTEXPR_CXX20 bool tests() {
+  // Test with insufficient space where reallocation occurs during assignment
+  {
+    std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<bool>(3));
+  }
+  {
+    std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
+    std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == other_allocator<bool>(5));
+  }
 #if TEST_STD_VER >= 11
-    {
-        std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
-        std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
-        l2 = l;
-        assert(l2 == l);
-        assert(l2.get_allocator() == min_allocator<bool>());
-    }
+  {
+    std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
+    std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == min_allocator<bool>());
+  }
 #endif
 
-    return true;
+  // Test with sufficient size where no reallocation occurs during assignment
+  {
+    std::vector<bool, test_allocator<bool> > l(4, true, test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > l2(5, false, test_allocator<bool>(3));
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<bool>(3));
+  }
+
+  // Test with sufficient spare space where no reallocation occurs during assignment
+  {
+    std::vector<bool, test_allocator<bool> > l(4, true, test_allocator<bool>(5));
+    std::vector<bool, test_allocator<bool> > l2(2, false, test_allocator<bool>(3));
+    l2.reserve(5);
+    l2 = l;
+    assert(l2 == l);
+    assert(l2.get_allocator() == test_allocator<bool>(3));
+  }
+
+  return true;
 }
 
-int main(int, char**)
-{
-    tests();
+int main(int, char**) {
+  tests();
 #if TEST_STD_VER > 17
-    static_assert(tests());
+  static_assert(tests());
 #endif
-    return 0;
+  return 0;
 }