Optimize ranges::move{,_backward} for vector<bool>::iterator

Author: winner245

libcxx/docs/ReleaseNotes/21.rst

@@ -43,9 +43,8 @@ Implemented Papers
 Improvements and New Features
 -----------------------------
 
-- The ``std::ranges::{copy, copy_n, copy_backward}`` algorithms have been optimized for ``std::vector<bool>::iterator``\s,
-  resulting in a performance improvement of up to 2000x.
-
+- The ``std::ranges::{copy, copy_n, copy_backward, move, move_backward}`` algorithms have been optimized for
+  ``std::vector<bool>::iterator``, resulting in a performance improvement of up to 2000x.
 
 Deprecations and Removals
 -------------------------

libcxx/include/__algorithm/move.h

@@ -9,11 +9,13 @@
 #ifndef _LIBCPP___ALGORITHM_MOVE_H
 #define _LIBCPP___ALGORITHM_MOVE_H
 
+#include <__algorithm/copy.h>
 #include <__algorithm/copy_move_common.h>
 #include <__algorithm/for_each_segment.h>
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__fwd/bit_reference.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
 #include <__type_traits/common_type.h>
@@ -98,6 +100,14 @@ struct __move_impl {
     }
   }
 
+  template <class _Cp, bool _IsConst>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
+  operator()(__bit_iterator<_Cp, _IsConst> __first,
+             __bit_iterator<_Cp, _IsConst> __last,
+             __bit_iterator<_Cp, false> __result) {
+    return std::__copy(__first, __last, __result);
+  }
+
   // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
   template <class _In, class _Out, __enable_if_t<__can_lower_move_assignment_to_memmove<_In, _Out>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>

libcxx/include/__algorithm/move_backward.h

@@ -9,10 +9,12 @@
 #ifndef _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
 #define _LIBCPP___ALGORITHM_MOVE_BACKWARD_H
 
+#include <__algorithm/copy_backward.h>
 #include <__algorithm/copy_move_common.h>
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/min.h>
 #include <__config>
+#include <__fwd/bit_reference.h>
 #include <__iterator/iterator_traits.h>
 #include <__iterator/segmented_iterator.h>
 #include <__type_traits/common_type.h>
@@ -107,6 +109,14 @@ struct __move_backward_impl {
     }
   }
 
+  template <class _Cp, bool _IsConst>
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, _IsConst>, __bit_iterator<_Cp, false> >
+  operator()(__bit_iterator<_Cp, _IsConst> __first,
+             __bit_iterator<_Cp, _IsConst> __last,
+             __bit_iterator<_Cp, false> __result) {
+    return std::__copy_backward<_ClassicAlgPolicy>(__first, __last, __result);
+  }
+
   // At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
   template <class _In, class _Out, __enable_if_t<__can_lower_move_assignment_to_memmove<_In, _Out>::value, int> = 0>
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>

libcxx/include/__bit_reference

@@ -186,22 +186,6 @@ private:
         __mask_(__m) {}
 };
 
-// move
-
-template <class _Cp, bool _IsConst>
-inline _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false>
-move(__bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  return std::copy(__first, __last, __result);
-}
-
-// move_backward
-
-template <class _Cp, bool _IsConst>
-inline _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> move_backward(
-    __bit_iterator<_Cp, _IsConst> __first, __bit_iterator<_Cp, _IsConst> __last, __bit_iterator<_Cp, false> __result) {
-  return std::copy_backward(__first, __last, __result);
-}
-
 // swap_ranges
 
 template <class _Cl, class _Cr>

libcxx/test/benchmarks/algorithms/move.bench.cpp

@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <vector>
+
+static void bm_ranges_move_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto dst = aligned ? out.begin() : out.begin() + 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::ranges::move(in, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_move_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto beg = in.begin();
+  auto end = in.end();
+  auto dst = aligned ? out.begin() : out.begin() + 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::move(beg, end, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_ranges_move_vb_aligned(benchmark::State& state) { bm_ranges_move_vb(state, true); }
+static void bm_ranges_move_vb_unaligned(benchmark::State& state) { bm_ranges_move_vb(state, false); }
+
+static void bm_move_vb_aligned(benchmark::State& state) { bm_move_vb(state, true); }
+static void bm_move_vb_unaligned(benchmark::State& state) { bm_move_vb(state, false); }
+
+// Test std::ranges::move for vector<bool>::iterator
+BENCHMARK(bm_ranges_move_vb_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096);
+BENCHMARK(bm_ranges_move_vb_unaligned)->Range(8, 1 << 20);
+
+// Test std::move for vector<bool>::iterator
+BENCHMARK(bm_move_vb_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_move_vb_unaligned)->Range(8, 1 << 20);
+
+BENCHMARK_MAIN();

libcxx/test/benchmarks/algorithms/move_backward.bench.cpp

@@ -0,0 +1,55 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <vector>
+
+static void bm_ranges_move_backward_vb(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto dst = aligned ? out.end() : out.end() - 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::ranges::move_backward(in, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_move_backward(benchmark::State& state, bool aligned) {
+  auto n = state.range();
+  std::vector<bool> in(n, true);
+  std::vector<bool> out(aligned ? n : n + 8);
+  benchmark::DoNotOptimize(&in);
+  auto beg = in.begin();
+  auto end = in.end();
+  auto dst = aligned ? out.end() : out.end() - 4;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::move_backward(beg, end, dst));
+    benchmark::DoNotOptimize(&out);
+  }
+}
+
+static void bm_ranges_move_backward_vb_aligned(benchmark::State& state) { bm_ranges_move_backward_vb(state, true); }
+static void bm_ranges_move_backward_vb_unaligned(benchmark::State& state) { bm_ranges_move_backward_vb(state, false); }
+
+static void bm_move_backward_vb_aligned(benchmark::State& state) { bm_move_backward(state, true); }
+static void bm_move_backward_vb_unaligned(benchmark::State& state) { bm_move_backward(state, false); }
+
+// Test std::ranges::move_backward for vector<bool>::iterator
+BENCHMARK(bm_ranges_move_backward_vb_aligned)->Range(8, 1 << 16)->DenseRange(102400, 204800, 4096);
+BENCHMARK(bm_ranges_move_backward_vb_unaligned)->Range(8, 1 << 20);
+
+// Test std::move_backward for vector<bool>::iterator
+BENCHMARK(bm_move_backward_vb_aligned)->Range(8, 1 << 20);
+BENCHMARK(bm_move_backward_vb_unaligned)->Range(8, 1 << 20);
+
+BENCHMARK_MAIN();

libcxx/test/std/algorithms/alg.modifying.operations/alg.move/move.pass.cpp

@@ -20,6 +20,7 @@
 #include <cassert>
 #include <iterator>
 #include <memory>
+#include <vector>
 
 #include "MoveOnly.h"
 #include "test_iterators.h"
@@ -45,15 +46,15 @@ struct Test {
   template <class OutIter>
   TEST_CONSTEXPR_CXX20 void operator()() {
     const unsigned N = 1000;
-    int ia[N] = {};
+    int ia[N]        = {};
     for (unsigned i = 0; i < N; ++i)
-        ia[i] = i;
+      ia[i] = i;
     int ib[N] = {0};
 
-    OutIter r = std::move(InIter(ia), InIter(ia+N), OutIter(ib));
-    assert(base(r) == ib+N);
+    OutIter r = std::move(InIter(ia), InIter(ia + N), OutIter(ib));
+    assert(base(r) == ib + N);
     for (unsigned i = 0; i < N; ++i)
-        assert(ia[i] == ib[i]);
+      assert(ia[i] == ib[i]);
   }
 };
 
@@ -73,13 +74,13 @@ struct Test1 {
     const unsigned N = 100;
     std::unique_ptr<int> ia[N];
     for (unsigned i = 0; i < N; ++i)
-        ia[i].reset(new int(i));
+      ia[i].reset(new int(i));
     std::unique_ptr<int> ib[N];
 
-    OutIter r = std::move(InIter(ia), InIter(ia+N), OutIter(ib));
-    assert(base(r) == ib+N);
+    OutIter r = std::move(InIter(ia), InIter(ia + N), OutIter(ib));
+    assert(base(r) == ib + N);
     for (unsigned i = 0; i < N; ++i)
-        assert(*ib[i] == static_cast<int>(i));
+      assert(*ib[i] == static_cast<int>(i));
   }
 };
 
@@ -92,6 +93,26 @@ struct Test1OutIters {
   }
 };
 
+TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) {
+  std::vector<bool> in(N, false);
+  for (std::size_t i = 0; i < N; i += 2)
+    in[i] = true;
+
+  { // Test move with aligned bytes
+    std::vector<bool> out(N);
+    std::move(in.begin(), in.end(), out.begin());
+    assert(in == out);
+  }
+  { // Test move with unaligned bytes
+    std::vector<bool> out(N + 8);
+    std::move(in.begin(), in.end(), out.begin() + 4);
+    for (std::size_t i = 0; i < N; ++i)
+      assert(out[i + 4] == in[i]);
+  }
+
+  return true;
+}
+
 TEST_CONSTEXPR_CXX20 bool test() {
   types::for_each(types::cpp17_input_iterator_list<int*>(), TestOutIters());
   if (TEST_STD_AT_LEAST_23_OR_RUNTIME_EVALUATED)
@@ -118,7 +139,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
     // When non-trivial
     {
       MoveOnly from[3] = {1, 2, 3};
-      MoveOnly to[3] = {};
+      MoveOnly to[3]   = {};
       std::move(std::begin(from), std::end(from), std::begin(to));
       assert(to[0] == MoveOnly(1));
       assert(to[1] == MoveOnly(2));
@@ -127,14 +148,24 @@ TEST_CONSTEXPR_CXX20 bool test() {
     // When trivial
     {
       TrivialMoveOnly from[3] = {1, 2, 3};
-      TrivialMoveOnly to[3] = {};
+      TrivialMoveOnly to[3]   = {};
       std::move(std::begin(from), std::end(from), std::begin(to));
       assert(to[0] == TrivialMoveOnly(1));
       assert(to[1] == TrivialMoveOnly(2));
       assert(to[2] == TrivialMoveOnly(3));
     }
   }
 
+  { // Test vector<bool>::iterator optimization
+    assert(test_vector_bool(8));
+    assert(test_vector_bool(19));
+    assert(test_vector_bool(32));
+    assert(test_vector_bool(49));
+    assert(test_vector_bool(64));
+    assert(test_vector_bool(199));
+    assert(test_vector_bool(256));
+  }
+
   return true;
 }