[libc++] Optimize rotate

Author: philnik777

libcxx/docs/ReleaseNotes/20.rst

@@ -73,6 +73,9 @@ Improvements and New Features
   optimized, resulting in a performance improvement of up to 2x for trivial element types (e.g., `std::vector<int>`),
   and up to 3.4x for non-trivial element types (e.g., `std::vector<std::vector<int>>`).
 
+- ``rotate`` has been optimized, resulting in a performance improvement of up to 2.2x for trivially move assignable
+  types.
+
 Deprecations and Removals
 -------------------------
 

libcxx/include/__algorithm/rotate.h

@@ -82,46 +82,39 @@ __rotate_forward(_ForwardIterator __first, _ForwardIterator __middle, _ForwardIt
   return __r;
 }
 
-template <typename _Integral>
-inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Integral __algo_gcd(_Integral __x, _Integral __y) {
-  do {
-    _Integral __t = __x % __y;
-    __x           = __y;
-    __y           = __t;
-  } while (__y);
-  return __x;
-}
+template <class _AlgPolicy, class _Iter, class _Sent>
+_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _Iter __rotate_random(_Iter __first, _Iter __middle, _Sent __last) {
+  auto __left = _IterOps<_AlgPolicy>::distance(__first, __middle);
+  auto __right = _IterOps<_AlgPolicy>::distance(__middle, __last);
+  auto __end = __first + __right;
 
-template <class _AlgPolicy, typename _RandomAccessIterator>
-_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 _RandomAccessIterator
-__rotate_gcd(_RandomAccessIterator __first, _RandomAccessIterator __middle, _RandomAccessIterator __last) {
-  typedef typename iterator_traits<_RandomAccessIterator>::difference_type difference_type;
-  typedef typename iterator_traits<_RandomAccessIterator>::value_type value_type;
-  using _Ops = _IterOps<_AlgPolicy>;
+  if (__left == 0 || __first == __last)
+    return __first;
 
-  const difference_type __m1 = __middle - __first;
-  const difference_type __m2 = _Ops::distance(__middle, __last);
-  if (__m1 == __m2) {
-    std::__swap_ranges<_AlgPolicy>(__first, __middle, __middle, __last);
+  if (__left == __right) {
+    std::__swap_ranges<_AlgPolicy>(__first, __middle, __middle);
     return __middle;
   }
-  const difference_type __g = std::__algo_gcd(__m1, __m2);
-  for (_RandomAccessIterator __p = __first + __g; __p != __first;) {
-    value_type __t(_Ops::__iter_move(--__p));
-    _RandomAccessIterator __p1 = __p;
-    _RandomAccessIterator __p2 = __p1 + __m1;
-    do {
-      *__p1                     = _Ops::__iter_move(__p2);
-      __p1                      = __p2;
-      const difference_type __d = _Ops::distance(__p2, __last);
-      if (__m1 < __d)
-        __p2 += __m1;
-      else
-        __p2 = __first + (__m1 - __d);
-    } while (__p2 != __p);
-    *__p1 = std::move(__t);
+
+  auto __min_len = std::min(__left, __right);
+
+  while (__min_len > 0) {
+    if (__left <= __right) {
+      do {
+        std::__swap_ranges<_AlgPolicy>(__first, __first + __left, __first + __left);
+        __first += __left;
+        __right -= __left;
+      } while (__left <= __right);
+      __min_len = __right;
+    } else {
+      do {
+        std::__swap_ranges<_AlgPolicy>(__first + (__left - __right), __first + __left, __first + __left);
+        __left -= __right;
+      } while (__left > __right);
+      __min_len = __left;
+    }
   }
-  return __first + __m2;
+  return __end;
 }
 
 template <class _AlgPolicy, class _ForwardIterator>
@@ -163,7 +156,7 @@ inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 _RandomAccessIterator
       return std::__rotate_left<_AlgPolicy>(__first, __last);
     if (_IterOps<_AlgPolicy>::next(__middle) == __last)
       return std::__rotate_right<_AlgPolicy>(__first, __last);
-    return std::__rotate_gcd<_AlgPolicy>(__first, __middle, __last);
+    return std::__rotate_random<_AlgPolicy>(__first, __middle, __last);
   }
   return std::__rotate_forward<_AlgPolicy>(__first, __middle, __last);
 }

libcxx/test/benchmarks/algorithms/rotate.bench.cpp

@@ -0,0 +1,35 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+#include <algorithm>
+#include <cassert>
+
+#include <benchmark/benchmark.h>
+
+void run_sizes(auto benchmark) {
+  benchmark->Arg(1)->Arg(2)->Arg(3)->Arg(4)->Arg(32)->Arg(64)->Arg(512)->Arg(4096)->Arg(65536);
+}
+
+template <class T>
+static void BM_std_rotate(benchmark::State& state) {
+  std::vector<T> vec(state.range(), T());
+
+  for (auto _ : state) {
+    benchmark::DoNotOptimize(vec);
+    benchmark::DoNotOptimize(std::rotate(vec.begin(), vec.begin() + vec.size() / 3, vec.end()));
+  }
+}
+BENCHMARK(BM_std_rotate<int>)->Apply(run_sizes);
+#ifndef TEST_HAS_NO_INT128
+BENCHMARK(BM_std_rotate<__int128>)->Apply(run_sizes);
+#endif
+BENCHMARK(BM_std_rotate<std::string>)->Apply(run_sizes);
+
+BENCHMARK_MAIN();