Skip to content

Commit 2b9442a

Browse files
winner245winner245
committed
Optimize ranges::equal for vector<bool>::iterator
1 parent ee92122 commit 2b9442a

File tree

7 files changed

+418
-217
lines changed

7 files changed

+418
-217
lines changed

‎libcxx/docs/ReleaseNotes/21.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,8 @@ Improvements and New Features
4646
- The ``std::ranges::{copy, copy_n, copy_backward}`` algorithms have been optimized for ``std::vector<bool>::iterator``\s,
4747
resulting in a performance improvement of up to 2000x.
4848

49+
- The ``std::ranges::equal`` algorithm has been optimized for ``std::vector<bool>::iterator``, resulting in a performance
50+
improvement of up to 188x.
4951

5052
Deprecations and Removals
5153
-------------------------

‎libcxx/include/__algorithm/equal.h‎

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,27 @@
1111
#define _LIBCPP___ALGORITHM_EQUAL_H
1212

1313
#include <__algorithm/comp.h>
14+
#include <__algorithm/min.h>
1415
#include <__algorithm/unwrap_iter.h>
1516
#include <__config>
1617
#include <__functional/identity.h>
18+
#include <__fwd/bit_reference.h>
1719
#include <__iterator/distance.h>
1820
#include <__iterator/iterator_traits.h>
21+
#include <__memory/pointer_traits.h>
1922
#include <__string/constexpr_c_functions.h>
2023
#include <__type_traits/desugars_to.h>
2124
#include <__type_traits/enable_if.h>
2225
#include <__type_traits/invoke.h>
2326
#include <__type_traits/is_equality_comparable.h>
27+
#include <__type_traits/is_same.h>
2428
#include <__type_traits/is_volatile.h>
2529
#include <__utility/move.h>
2630

31+
#if _LIBCPP_STD_VER >= 20
32+
# include <__functional/ranges_operations.h>
33+
#endif
34+
2735
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
2836
# pragma GCC system_header
2937
#endif
@@ -33,6 +41,132 @@ _LIBCPP_PUSH_MACROS
3341

3442
_LIBCPP_BEGIN_NAMESPACE_STD
3543

44+
template <class _Cp, bool _IC1, bool _IC2>
45+
[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __equal_unaligned(
46+
__bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
47+
using _It = __bit_iterator<_Cp, _IC1>;
48+
using difference_type = typename _It::difference_type;
49+
using __storage_type = typename _It::__storage_type;
50+
51+
const int __bits_per_word = _It::__bits_per_word;
52+
difference_type __n = __last1 - __first1;
53+
if (__n > 0) {
54+
// do first word
55+
if (__first1.__ctz_ != 0) {
56+
unsigned __clz_f = __bits_per_word - __first1.__ctz_;
57+
difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
58+
__n -= __dn;
59+
__storage_type __m = (~__storage_type(0) << __first1.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
60+
__storage_type __b = *__first1.__seg_ & __m;
61+
unsigned __clz_r = __bits_per_word - __first2.__ctz_;
62+
__storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
63+
__m = (~__storage_type(0) << __first2.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
64+
if (__first2.__ctz_ > __first1.__ctz_) {
65+
if ((*__first2.__seg_ & __m) != (__b << (__first2.__ctz_ - __first1.__ctz_)))
66+
return false;
67+
} else {
68+
if ((*__first2.__seg_ & __m) != (__b >> (__first1.__ctz_ - __first2.__ctz_)))
69+
return false;
70+
}
71+
__first2.__seg_ += (__ddn + __first2.__ctz_) / __bits_per_word;
72+
__first2.__ctz_ = static_cast<unsigned>((__ddn + __first2.__ctz_) % __bits_per_word);
73+
__dn -= __ddn;
74+
if (__dn > 0) {
75+
__m = ~__storage_type(0) >> (__bits_per_word - __dn);
76+
if ((*__first2.__seg_ & __m) != (__b >> (__first1.__ctz_ + __ddn)))
77+
return false;
78+
__first2.__ctz_ = static_cast<unsigned>(__dn);
79+
}
80+
++__first1.__seg_;
81+
// __first1.__ctz_ = 0;
82+
}
83+
// __first1.__ctz_ == 0;
84+
// do middle words
85+
unsigned __clz_r = __bits_per_word - __first2.__ctz_;
86+
__storage_type __m = ~__storage_type(0) << __first2.__ctz_;
87+
for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first1.__seg_) {
88+
__storage_type __b = *__first1.__seg_;
89+
if ((*__first2.__seg_ & __m) != (__b << __first2.__ctz_))
90+
return false;
91+
++__first2.__seg_;
92+
if ((*__first2.__seg_ & ~__m) != (__b >> __clz_r))
93+
return false;
94+
}
95+
// do last word
96+
if (__n > 0) {
97+
__m = ~__storage_type(0) >> (__bits_per_word - __n);
98+
__storage_type __b = *__first1.__seg_ & __m;
99+
__storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
100+
__m = (~__storage_type(0) << __first2.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
101+
if ((*__first2.__seg_ & __m) != (__b << __first2.__ctz_))
102+
return false;
103+
__first2.__seg_ += (__dn + __first2.__ctz_) / __bits_per_word;
104+
__first2.__ctz_ = static_cast<unsigned>((__dn + __first2.__ctz_) % __bits_per_word);
105+
__n -= __dn;
106+
if (__n > 0) {
107+
__m = ~__storage_type(0) >> (__bits_per_word - __n);
108+
if ((*__first2.__seg_ & __m) != (__b >> __dn))
109+
return false;
110+
}
111+
}
112+
}
113+
return true;
114+
}
115+
116+
template <class _Cp, bool _IC1, bool _IC2>
117+
[[__nodiscard__]] _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __equal_aligned(
118+
__bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
119+
using _It = __bit_iterator<_Cp, _IC1>;
120+
using difference_type = typename _It::difference_type;
121+
using __storage_type = typename _It::__storage_type;
122+
123+
const int __bits_per_word = _It::__bits_per_word;
124+
difference_type __n = __last1 - __first1;
125+
if (__n > 0) {
126+
// do first word
127+
if (__first1.__ctz_ != 0) {
128+
unsigned __clz = __bits_per_word - __first1.__ctz_;
129+
difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
130+
__n -= __dn;
131+
__storage_type __m = (~__storage_type(0) << __first1.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
132+
if ((*__first2.__seg_ & __m) != (*__first1.__seg_ & __m))
133+
return false;
134+
++__first2.__seg_;
135+
++__first1.__seg_;
136+
// __first1.__ctz_ = 0;
137+
// __first2.__ctz_ = 0;
138+
}
139+
// __first1.__ctz_ == 0;
140+
// __first2.__ctz_ == 0;
141+
// do middle words
142+
for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first1.__seg_, ++__first2.__seg_)
143+
if (*__first2.__seg_ != *__first1.__seg_)
144+
return false;
145+
// do last word
146+
if (__n > 0) {
147+
__storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
148+
if ((*__first2.__seg_ & __m) != (*__first1.__seg_ & __m))
149+
return false;
150+
}
151+
}
152+
return true;
153+
}
154+
155+
template <class _Cp,
156+
bool _IC1,
157+
bool _IC2,
158+
class _BinaryPredicate,
159+
__enable_if_t<std::is_same<_BinaryPredicate, __equal_to>::value, int> = 0>
160+
[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_iter_impl(
161+
__bit_iterator<_Cp, _IC1> __first1,
162+
__bit_iterator<_Cp, _IC1> __last1,
163+
__bit_iterator<_Cp, _IC2> __first2,
164+
_BinaryPredicate) {
165+
if (__first1.__ctz_ == __first2.__ctz_)
166+
return std::__equal_aligned(__first1, __last1, __first2);
167+
return std::__equal_unaligned(__first1, __last1, __first2);
168+
}
169+
36170
template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
37171
[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_iter_impl(
38172
_InputIterator1 __first1, _InputIterator1 __last1, _InputIterator2 __first2, _BinaryPredicate& __pred) {
@@ -94,6 +228,32 @@ __equal_impl(_Tp* __first1, _Tp* __last1, _Up* __first2, _Up*, _Pred&, _Proj1&,
94228
return std::__constexpr_memcmp_equal(__first1, __first2, __element_count(__last1 - __first1));
95229
}
96230

231+
template <class _Cp,
232+
bool _IC1,
233+
bool _IC2,
234+
class _Pred,
235+
class _Proj1,
236+
class _Proj2,
237+
__enable_if_t<(is_same<_Pred, __equal_to>::value
238+
# if _LIBCPP_STD_VER >= 20
239+
|| is_same<_Pred, ranges::equal_to>::value
240+
# endif
241+
) &&
242+
__desugars_to_v<__equal_tag, _Pred, bool, bool> && __is_identity<_Proj1>::value &&
243+
__is_identity<_Proj2>::value,
244+
int> = 0>
245+
[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __equal_impl(
246+
__bit_iterator<_Cp, _IC1> __first1,
247+
__bit_iterator<_Cp, _IC1> __last1,
248+
__bit_iterator<_Cp, _IC2> __first2,
249+
__bit_iterator<_Cp, _IC2>,
250+
_Pred&,
251+
_Proj1&,
252+
_Proj2&) {
253+
__equal_to __pred;
254+
return std::__equal_iter_impl(__first1, __last1, __first2, __pred);
255+
}
256+
97257
template <class _InputIterator1, class _InputIterator2, class _BinaryPredicate>
98258
[[__nodiscard__]] inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
99259
equal(_InputIterator1 __first1,

‎libcxx/include/__bit_reference‎

Lines changed: 11 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@
1212

1313
#include <__algorithm/copy.h>
1414
#include <__algorithm/copy_backward.h>
15+
#include <__algorithm/comp.h>
1516
#include <__algorithm/copy_n.h>
17+
#include <__algorithm/equal.h>
1618
#include <__algorithm/min.h>
1719
#include <__bit/countr.h>
1820
#include <__compare/ordering.h>
@@ -24,7 +26,9 @@
2426
#include <__memory/construct_at.h>
2527
#include <__memory/pointer_traits.h>
2628
#include <__type_traits/conditional.h>
29+
#include <__type_traits/enable_if.h>
2730
#include <__type_traits/is_constant_evaluated.h>
31+
#include <__type_traits/is_same.h>
2832
#include <__type_traits/void_t.h>
2933
#include <__utility/pair.h>
3034
#include <__utility/swap.h>
@@ -420,127 +424,6 @@ rotate(__bit_iterator<_Cp, false> __first, __bit_iterator<_Cp, false> __middle,
420424
return __r;
421425
}
422426

423-
// equal
424-
425-
template <class _Cp, bool _IC1, bool _IC2>
426-
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __equal_unaligned(
427-
__bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
428-
using _It = __bit_iterator<_Cp, _IC1>;
429-
using difference_type = typename _It::difference_type;
430-
using __storage_type = typename _It::__storage_type;
431-
432-
const int __bits_per_word = _It::__bits_per_word;
433-
difference_type __n = __last1 - __first1;
434-
if (__n > 0) {
435-
// do first word
436-
if (__first1.__ctz_ != 0) {
437-
unsigned __clz_f = __bits_per_word - __first1.__ctz_;
438-
difference_type __dn = std::min(static_cast<difference_type>(__clz_f), __n);
439-
__n -= __dn;
440-
__storage_type __m = (~__storage_type(0) << __first1.__ctz_) & (~__storage_type(0) >> (__clz_f - __dn));
441-
__storage_type __b = *__first1.__seg_ & __m;
442-
unsigned __clz_r = __bits_per_word - __first2.__ctz_;
443-
__storage_type __ddn = std::min<__storage_type>(__dn, __clz_r);
444-
__m = (~__storage_type(0) << __first2.__ctz_) & (~__storage_type(0) >> (__clz_r - __ddn));
445-
if (__first2.__ctz_ > __first1.__ctz_) {
446-
if ((*__first2.__seg_ & __m) != (__b << (__first2.__ctz_ - __first1.__ctz_)))
447-
return false;
448-
} else {
449-
if ((*__first2.__seg_ & __m) != (__b >> (__first1.__ctz_ - __first2.__ctz_)))
450-
return false;
451-
}
452-
__first2.__seg_ += (__ddn + __first2.__ctz_) / __bits_per_word;
453-
__first2.__ctz_ = static_cast<unsigned>((__ddn + __first2.__ctz_) % __bits_per_word);
454-
__dn -= __ddn;
455-
if (__dn > 0) {
456-
__m = ~__storage_type(0) >> (__bits_per_word - __dn);
457-
if ((*__first2.__seg_ & __m) != (__b >> (__first1.__ctz_ + __ddn)))
458-
return false;
459-
__first2.__ctz_ = static_cast<unsigned>(__dn);
460-
}
461-
++__first1.__seg_;
462-
// __first1.__ctz_ = 0;
463-
}
464-
// __first1.__ctz_ == 0;
465-
// do middle words
466-
unsigned __clz_r = __bits_per_word - __first2.__ctz_;
467-
__storage_type __m = ~__storage_type(0) << __first2.__ctz_;
468-
for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first1.__seg_) {
469-
__storage_type __b = *__first1.__seg_;
470-
if ((*__first2.__seg_ & __m) != (__b << __first2.__ctz_))
471-
return false;
472-
++__first2.__seg_;
473-
if ((*__first2.__seg_ & ~__m) != (__b >> __clz_r))
474-
return false;
475-
}
476-
// do last word
477-
if (__n > 0) {
478-
__m = ~__storage_type(0) >> (__bits_per_word - __n);
479-
__storage_type __b = *__first1.__seg_ & __m;
480-
__storage_type __dn = std::min(__n, static_cast<difference_type>(__clz_r));
481-
__m = (~__storage_type(0) << __first2.__ctz_) & (~__storage_type(0) >> (__clz_r - __dn));
482-
if ((*__first2.__seg_ & __m) != (__b << __first2.__ctz_))
483-
return false;
484-
__first2.__seg_ += (__dn + __first2.__ctz_) / __bits_per_word;
485-
__first2.__ctz_ = static_cast<unsigned>((__dn + __first2.__ctz_) % __bits_per_word);
486-
__n -= __dn;
487-
if (__n > 0) {
488-
__m = ~__storage_type(0) >> (__bits_per_word - __n);
489-
if ((*__first2.__seg_ & __m) != (__b >> __dn))
490-
return false;
491-
}
492-
}
493-
}
494-
return true;
495-
}
496-
497-
template <class _Cp, bool _IC1, bool _IC2>
498-
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI bool __equal_aligned(
499-
__bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
500-
using _It = __bit_iterator<_Cp, _IC1>;
501-
using difference_type = typename _It::difference_type;
502-
using __storage_type = typename _It::__storage_type;
503-
504-
const int __bits_per_word = _It::__bits_per_word;
505-
difference_type __n = __last1 - __first1;
506-
if (__n > 0) {
507-
// do first word
508-
if (__first1.__ctz_ != 0) {
509-
unsigned __clz = __bits_per_word - __first1.__ctz_;
510-
difference_type __dn = std::min(static_cast<difference_type>(__clz), __n);
511-
__n -= __dn;
512-
__storage_type __m = (~__storage_type(0) << __first1.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
513-
if ((*__first2.__seg_ & __m) != (*__first1.__seg_ & __m))
514-
return false;
515-
++__first2.__seg_;
516-
++__first1.__seg_;
517-
// __first1.__ctz_ = 0;
518-
// __first2.__ctz_ = 0;
519-
}
520-
// __first1.__ctz_ == 0;
521-
// __first2.__ctz_ == 0;
522-
// do middle words
523-
for (; __n >= __bits_per_word; __n -= __bits_per_word, ++__first1.__seg_, ++__first2.__seg_)
524-
if (*__first2.__seg_ != *__first1.__seg_)
525-
return false;
526-
// do last word
527-
if (__n > 0) {
528-
__storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
529-
if ((*__first2.__seg_ & __m) != (*__first1.__seg_ & __m))
530-
return false;
531-
}
532-
}
533-
return true;
534-
}
535-
536-
template <class _Cp, bool _IC1, bool _IC2>
537-
inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool
538-
equal(__bit_iterator<_Cp, _IC1> __first1, __bit_iterator<_Cp, _IC1> __last1, __bit_iterator<_Cp, _IC2> __first2) {
539-
if (__first1.__ctz_ == __first2.__ctz_)
540-
return std::__equal_aligned(__first1, __last1, __first2);
541-
return std::__equal_unaligned(__first1, __last1, __first2);
542-
}
543-
544427
template <class _Cp, bool _IsConst, typename _Cp::__storage_type>
545428
class __bit_iterator {
546429
public:
@@ -769,9 +652,13 @@ private:
769652
template <class _Dp, bool _IC1, bool _IC2>
770653
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
771654
__equal_unaligned(__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>);
772-
template <class _Dp, bool _IC1, bool _IC2>
773-
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool
774-
equal(__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>);
655+
template <class _Dp,
656+
bool _IC1,
657+
bool _IC2,
658+
class _BinaryPredicate,
659+
__enable_if_t<std::is_same<_BinaryPredicate, __equal_to>::value, int> >
660+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 friend bool __equal_iter_impl(
661+
__bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC1>, __bit_iterator<_Dp, _IC2>, _BinaryPredicate);
775662
template <bool _ToFind, class _Dp, bool _IC>
776663
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend __bit_iterator<_Dp, _IC>
777664
__find_bool(__bit_iterator<_Dp, _IC>, typename __size_difference_type_traits<_Dp>::size_type);

‎libcxx/include/bitset‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,7 @@ template <size_t N> struct hash<std::bitset<N>>;
132132
# include <__algorithm/copy.h>
133133
# include <__algorithm/copy_backward.h>
134134
# include <__algorithm/count.h>
135+
# include <__algorithm/equal.h>
135136
# include <__algorithm/fill.h>
136137
# include <__algorithm/fill_n.h>
137138
# include <__algorithm/find.h>

0 commit comments

Comments
 (0)