Optimize ranges::rotate for vector<bool>::iterator

Author: winner245

libcxx/include/__algorithm/rotate.h

@@ -9,12 +9,17 @@
 #ifndef _LIBCPP___ALGORITHM_ROTATE_H
 #define _LIBCPP___ALGORITHM_ROTATE_H
 
+#include <__algorithm/copy.h>
+#include <__algorithm/copy_backward.h>
 #include <__algorithm/iterator_operations.h>
 #include <__algorithm/move.h>
 #include <__algorithm/move_backward.h>
 #include <__algorithm/swap_ranges.h>
 #include <__config>
 #include <__iterator/iterator_traits.h>
+#include <__memory/construct_at.h>
+#include <__memory/pointer_traits.h>
+#include <__type_traits/is_constant_evaluated.h>
 #include <__type_traits/is_trivially_assignable.h>
 #include <__utility/move.h>
 #include <__utility/pair.h>
@@ -185,6 +190,75 @@ __rotate(_Iterator __first, _Iterator __middle, _Sentinel __last) {
   return _Ret(std::move(__result), std::move(__last_iter));
 }
 
+template <class _Cp>
+struct __bit_array {
+  using difference_type   = typename _Cp::difference_type;
+  using __storage_type    = typename _Cp::__storage_type;
+  using __storage_pointer = typename _Cp::__storage_pointer;
+  using iterator          = typename _Cp::iterator;
+
+  static const unsigned __bits_per_word = _Cp::__bits_per_word;
+  static const unsigned _Np             = 4;
+
+  difference_type __size_;
+  __storage_type __word_[_Np];
+
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static difference_type capacity() {
+    return static_cast<difference_type>(_Np * __bits_per_word);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bit_array(difference_type __s) : __size_(__s) {
+    if (__libcpp_is_constant_evaluated()) {
+      for (size_t __i = 0; __i != __bit_array<_Cp>::_Np; ++__i)
+        std::__construct_at(__word_ + __i, 0);
+    }
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() {
+    return iterator(pointer_traits<__storage_pointer>::pointer_to(__word_[0]), 0);
+  }
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() {
+    return iterator(pointer_traits<__storage_pointer>::pointer_to(__word_[0]) + __size_ / __bits_per_word,
+                    static_cast<unsigned>(__size_ % __bits_per_word));
+  }
+};
+
+template <class, class _Cp>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 pair<__bit_iterator<_Cp, false>, __bit_iterator<_Cp, false> >
+__rotate(__bit_iterator<_Cp, false> __first, __bit_iterator<_Cp, false> __middle, __bit_iterator<_Cp, false> __last) {
+  using _I1             = __bit_iterator<_Cp, false>;
+  using difference_type = typename _I1::difference_type;
+  difference_type __d1  = __middle - __first;
+  difference_type __d2  = __last - __middle;
+  _I1 __r               = __first + __d2;
+  while (__d1 != 0 && __d2 != 0) {
+    if (__d1 <= __d2) {
+      if (__d1 <= __bit_array<_Cp>::capacity()) {
+        __bit_array<_Cp> __b(__d1);
+        std::copy(__first, __middle, __b.begin());
+        std::copy(__b.begin(), __b.end(), std::copy(__middle, __last, __first));
+        break;
+      } else {
+        __bit_iterator<_Cp, false> __mp = std::swap_ranges(__first, __middle, __middle);
+        __first                         = __middle;
+        __middle                        = __mp;
+        __d2 -= __d1;
+      }
+    } else {
+      if (__d2 <= __bit_array<_Cp>::capacity()) {
+        __bit_array<_Cp> __b(__d2);
+        std::copy(__middle, __last, __b.begin());
+        std::copy_backward(__b.begin(), __b.end(), std::copy_backward(__first, __middle, __last));
+        break;
+      } else {
+        __bit_iterator<_Cp, false> __mp = __first + __d2;
+        std::swap_ranges(__first, __mp, __middle);
+        __first = __mp;
+        __d1 -= __d2;
+      }
+    }
+  }
+  return std::make_pair(__r, __last);
+}
+
 template <class _ForwardIterator>
 inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _ForwardIterator
 rotate(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIterator __last) {

libcxx/include/__bit_reference

@@ -12,6 +12,7 @@
 
 #include <__algorithm/copy_n.h>
 #include <__algorithm/min.h>
+#include <__algorithm/rotate.h>
 #include <__bit/countr.h>
 #include <__compare/ordering.h>
 #include <__config>
@@ -22,6 +23,7 @@
 #include <__memory/pointer_traits.h>
 #include <__type_traits/conditional.h>
 #include <__type_traits/is_constant_evaluated.h>
+#include <__utility/pair.h>
 #include <__utility/swap.h>
 
 #if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -583,78 +585,6 @@ inline _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cr, false> swap_ranges(
   return std::__swap_ranges_unaligned(__first1, __last1, __first2);
 }
 
-// rotate
-
-template <class _Cp>
-struct __bit_array {
-  using difference_type   = typename _Cp::difference_type;
-  using __storage_type    = typename _Cp::__storage_type;
-  using __storage_pointer = typename _Cp::__storage_pointer;
-  using iterator          = typename _Cp::iterator;
-
-  static const unsigned __bits_per_word = _Cp::__bits_per_word;
-  static const unsigned _Np             = 4;
-
-  difference_type __size_;
-  __storage_type __word_[_Np];
-
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 static difference_type capacity() {
-    return static_cast<difference_type>(_Np * __bits_per_word);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit __bit_array(difference_type __s) : __size_(__s) {
-    if (__libcpp_is_constant_evaluated()) {
-      for (size_t __i = 0; __i != __bit_array<_Cp>::_Np; ++__i)
-        std::__construct_at(__word_ + __i, 0);
-    }
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator begin() {
-    return iterator(pointer_traits<__storage_pointer>::pointer_to(__word_[0]), 0);
-  }
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 iterator end() {
-    return iterator(pointer_traits<__storage_pointer>::pointer_to(__word_[0]) + __size_ / __bits_per_word,
-                    static_cast<unsigned>(__size_ % __bits_per_word));
-  }
-};
-
-template <class _Cp>
-_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false>
-rotate(__bit_iterator<_Cp, false> __first, __bit_iterator<_Cp, false> __middle, __bit_iterator<_Cp, false> __last) {
-  using _I1             = __bit_iterator<_Cp, false>;
-  using difference_type = typename _I1::difference_type;
-
-  difference_type __d1 = __middle - __first;
-  difference_type __d2 = __last - __middle;
-  _I1 __r              = __first + __d2;
-  while (__d1 != 0 && __d2 != 0) {
-    if (__d1 <= __d2) {
-      if (__d1 <= __bit_array<_Cp>::capacity()) {
-        __bit_array<_Cp> __b(__d1);
-        std::copy(__first, __middle, __b.begin());
-        std::copy(__b.begin(), __b.end(), std::copy(__middle, __last, __first));
-        break;
-      } else {
-        __bit_iterator<_Cp, false> __mp = std::swap_ranges(__first, __middle, __middle);
-        __first                         = __middle;
-        __middle                        = __mp;
-        __d2 -= __d1;
-      }
-    } else {
-      if (__d2 <= __bit_array<_Cp>::capacity()) {
-        __bit_array<_Cp> __b(__d2);
-        std::copy(__middle, __last, __b.begin());
-        std::copy_backward(__b.begin(), __b.end(), std::copy_backward(__first, __middle, __last));
-        break;
-      } else {
-        __bit_iterator<_Cp, false> __mp = __first + __d2;
-        std::swap_ranges(__first, __mp, __middle);
-        __first = __mp;
-        __d1 -= __d2;
-      }
-    }
-  }
-  return __r;
-}
-
 // equal
 
 template <class _Cp, bool _IC1, bool _IC2>
@@ -995,9 +925,9 @@ private:
   template <class _Cl, class _Cr>
   friend __bit_iterator<_Cr, false>
       swap_ranges(__bit_iterator<_Cl, false>, __bit_iterator<_Cl, false>, __bit_iterator<_Cr, false>);
-  template <class _Dp>
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, false>
-      rotate(__bit_iterator<_Dp, false>, __bit_iterator<_Dp, false>, __bit_iterator<_Dp, false>);
+  template <class, class _Dp>
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 friend pair<__bit_iterator<_Cp, false>, __bit_iterator<_Cp, false> >
+      __rotate(__bit_iterator<_Dp, false>, __bit_iterator<_Dp, false>, __bit_iterator<_Dp, false>);
   template <class _Dp, bool _IC1, bool _IC2>
   _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
       __equal_aligned(__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>);

libcxx/test/benchmarks/algorithms/rotate.bench.cpp

@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <algorithm>
+#include <benchmark/benchmark.h>
+#include <vector>
+
+static void bm_ranges_rotate(benchmark::State& state) {
+  auto n = state.range();
+  std::vector<bool> v(n);
+  auto mid = std::ranges::begin(v) + n / 2;
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::ranges::rotate(v, mid));
+    benchmark::DoNotOptimize(&v);
+  }
+}
+
+static void bm_rotate(benchmark::State& state) {
+  auto n = state.range();
+  std::vector<bool> v(n);
+  auto beg = v.begin();
+  auto mid = v.begin() + n / 2;
+  auto end = v.end();
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(std::rotate(beg, mid, end));
+    benchmark::DoNotOptimize(&v);
+  }
+}
+
+BENCHMARK(bm_ranges_rotate)->Range(8, 1 << 20);
+BENCHMARK(bm_rotate)->Range(8, 1 << 20);
+
+BENCHMARK_MAIN();

libcxx/test/std/algorithms/alg.modifying.operations/alg.rotate/ranges_rotate.pass.cpp

@@ -21,6 +21,7 @@
 #include <array>
 #include <cassert>
 #include <ranges>
+#include <vector>
 
 #include "almost_satisfies_types.h"
 #include "test_iterators.h"
@@ -29,10 +30,9 @@
 // ======================================================
 
 template <class Iter = int*, class Sent = int*>
-concept HasRotateIter =
-    requires(Iter&& iter, Sent&& sent) {
-      std::ranges::rotate(std::forward<Iter>(iter), std::forward<Iter>(iter), std::forward<Sent>(sent));
-    };
+concept HasRotateIter = requires(Iter&& iter, Sent&& sent) {
+  std::ranges::rotate(std::forward<Iter>(iter), std::forward<Iter>(iter), std::forward<Sent>(sent));
+};
 
 static_assert(HasRotateIter<int*, int*>);
 
@@ -48,10 +48,9 @@ static_assert(!HasRotateIter<int*, SentinelForNotWeaklyEqualityComparableWith>);
 // =========================================
 
 template <class Range>
-concept HasRotateRange =
-    requires(Range&& range, std::ranges::iterator_t<Range> iter) {
-      std::ranges::rotate(std::forward<Range>(range), iter);
-    };
+concept HasRotateRange = requires(Range&& range, std::ranges::iterator_t<Range> iter) {
+  std::ranges::rotate(std::forward<Range>(range), iter);
+};
 
 template <class T>
 using R = UncheckedRange<T>;
@@ -73,10 +72,10 @@ constexpr void test_one(const std::array<int, N> input, std::size_t mid_index, s
   assert(mid_index <= N);
 
   { // (iterator, sentinel) overload.
-    auto in = input;
+    auto in    = input;
     auto begin = Iter(in.data());
-    auto mid = Iter(in.data() + mid_index);
-    auto end = Sent(Iter(in.data() + in.size()));
+    auto mid   = Iter(in.data() + mid_index);
+    auto end   = Sent(Iter(in.data() + in.size()));
 
     std::same_as<std::ranges::subrange<Iter>> decltype(auto) result = std::ranges::rotate(begin, mid, end);
     assert(base(result.begin()) == in.data() + in.size() - mid_index);
@@ -85,10 +84,10 @@ constexpr void test_one(const std::array<int, N> input, std::size_t mid_index, s
   }
 
   { // (range) overload.
-    auto in = input;
+    auto in    = input;
     auto begin = Iter(in.data());
-    auto mid = Iter(in.data() + mid_index);
-    auto end = Sent(Iter(in.data() + in.size()));
+    auto mid   = Iter(in.data() + mid_index);
+    auto end   = Sent(Iter(in.data() + in.size()));
     auto range = std::ranges::subrange(std::move(begin), std::move(end));
 
     std::same_as<std::ranges::subrange<Iter>> decltype(auto) result = std::ranges::rotate(range, mid);
@@ -146,18 +145,39 @@ constexpr void test_iterators() {
   test_iter<int*>();
 }
 
+template <std::size_t N>
+TEST_CONSTEXPR_CXX20 bool test_bit_iterator() {
+  for (int offset = -1; offset <= 1; ++offset) {
+    std::vector<bool> a(N, false);
+    std::size_t mid = N / 2 + offset;
+    for (std::size_t i = mid; i < N; ++i)
+      a[i] = true;
+
+    // (iterator, sentinel)-overload
+    std::ranges::rotate(std::ranges::begin(a), std::ranges::begin(a) + mid, std::ranges::end(a));
+    for (std::size_t i = 0; i < N; ++i)
+      assert(a[i] == (i < N - mid));
+
+    // range-overload
+    std::ranges::rotate(a, std::ranges::begin(a) + (N - mid));
+    for (std::size_t i = 0; i < N; ++i)
+      assert(a[i] == (i >= mid));
+  }
+  return true;
+};
+
 constexpr bool test() {
   test_iterators();
 
   { // Complexity: at most `last - first` swaps.
     const std::array input = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
-    auto expected = static_cast<int>(input.size());
+    auto expected          = static_cast<int>(input.size());
 
     {
-      auto in = input;
-      int swaps = 0;
+      auto in    = input;
+      int swaps  = 0;
       auto begin = adl::Iterator::TrackSwaps(in.data(), swaps);
-      auto end = adl::Iterator::TrackSwaps(in.data() + in.size(), swaps);
+      auto end   = adl::Iterator::TrackSwaps(in.data() + in.size(), swaps);
 
       for (std::size_t mid = 0; mid != input.size(); ++mid) {
         std::ranges::rotate(begin, begin + mid, end);
@@ -166,10 +186,10 @@ constexpr bool test() {
     }
 
     {
-      auto in = input;
-      int swaps = 0;
+      auto in    = input;
+      int swaps  = 0;
       auto begin = adl::Iterator::TrackSwaps(in.data(), swaps);
-      auto end = adl::Iterator::TrackSwaps(in.data() + in.size(), swaps);
+      auto end   = adl::Iterator::TrackSwaps(in.data() + in.size(), swaps);
       auto range = std::ranges::subrange(begin, end);
 
       for (std::size_t mid = 0; mid != input.size(); ++mid) {
@@ -179,6 +199,11 @@ constexpr bool test() {
     }
   }
 
+  test_bit_iterator<8>();
+  test_bit_iterator<16>();
+  test_bit_iterator<64>();
+  test_bit_iterator<100>();
+
   return true;
 }