Speed-up copy/move-ctors for vector<bool>

Author: winner245

libcxx/include/__vector/vector_bool.h

@@ -417,6 +417,8 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
     __guard.__complete();
   }
 
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __alloc_and_copy(const vector& __v);
+
   template <class _Iterator, class _Sentinel>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
 
@@ -692,25 +694,30 @@ vector<bool, _Allocator>::vector(initializer_list<value_type> __il, const alloca
 
 #endif // _LIBCPP_CXX03_LANG
 
+// This function copies each storage word as a whole, which is substantially more efficient than copying
+// individual bits within each word
+template <class _Allocator>
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void vector<bool, _Allocator>::__alloc_and_copy(const vector& __v) {
+  if (__v.__size_) {
+    __vallocate(__v.__size_);
+    std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
+  }
+  __size_ = __v.__size_;
+}
+
 template <class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v)
     : __begin_(nullptr),
       __size_(0),
       __cap_(0),
       __alloc_(__storage_traits::select_on_container_copy_construction(__v.__alloc_)) {
-  if (__v.size() > 0) {
-    __vallocate(__v.size());
-    __construct_at_end(__v.begin(), __v.end(), __v.size());
-  }
+  __alloc_and_copy(__v);
 }
 
 template <class _Allocator>
 _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v, const allocator_type& __a)
     : __begin_(nullptr), __size_(0), __cap_(0), __alloc_(__a) {
-  if (__v.size() > 0) {
-    __vallocate(__v.size());
-    __construct_at_end(__v.begin(), __v.end(), __v.size());
-  }
+  __alloc_and_copy(__v);
 }
 
 template <class _Allocator>
@@ -755,9 +762,8 @@ vector<bool, _Allocator>::vector(vector&& __v, const __type_identity_t<allocator
     this->__cap_   = __v.__cap_;
     __v.__begin_   = nullptr;
     __v.__cap_ = __v.__size_ = 0;
-  } else if (__v.size() > 0) {
-    __vallocate(__v.size());
-    __construct_at_end(__v.begin(), __v.end(), __v.size());
+  } else {
+    __alloc_and_copy(__v);
   }
 }
 

libcxx/test/benchmarks/containers/ContainerBenchmarks.h

@@ -39,6 +39,36 @@ void BM_CopyConstruct(benchmark::State& st, Container) {
   }
 }
 
+template <class Container>
+void BM_MoveConstruct(benchmark::State& st, Container) {
+  auto size = st.range(0);
+  Container c(size);
+  for (auto _ : st) {
+    auto v = std::move(c);
+    DoNotOptimizeData(v);
+  }
+}
+
+template <class Container, class Allocator>
+void BM_CopyConstruct_Alloc(benchmark::State& st, Container, Allocator a) {
+  auto size = st.range(0);
+  Container c(size);
+  for (auto _ : st) {
+    Container v(c, a);
+    DoNotOptimizeData(v);
+  }
+}
+
+template <class Container, class Allocator>
+void BM_MoveConstruct_Alloc(benchmark::State& st, Container, Allocator a) {
+  auto size = st.range(0);
+  Container c(size);
+  for (auto _ : st) {
+    Container v(std::move(c), a);
+    DoNotOptimizeData(v);
+  }
+}
+
 template <class Container>
 void BM_Assignment(benchmark::State& st, Container) {
   auto size = st.range(0);

libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp

@@ -0,0 +1,36 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <deque>
+#include <functional>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "benchmark/benchmark.h"
+#include "ContainerBenchmarks.h"
+#include "../GenerateInput.h"
+#include "test_allocator.h"
+
+using namespace ContainerBenchmarks;
+
+BENCHMARK_CAPTURE(BM_CopyConstruct, vector_bool, std::vector<bool>{})->Arg(5140480);
+BENCHMARK_CAPTURE(BM_MoveConstruct, vector_bool, std::vector<bool>{})->Arg(5140480);
+BENCHMARK_CAPTURE(
+    BM_CopyConstruct_Alloc, vector_bool, std::vector<bool, test_allocator<bool>>(), test_allocator<bool>(3))
+    ->Arg(5140480);
+BENCHMARK_CAPTURE(
+    BM_MoveConstruct_Alloc, vector_bool, std::vector<bool, test_allocator<bool>>(), test_allocator<bool>(3))
+    ->Arg(5140480);
+
+BENCHMARK_MAIN();