Skip to content

Commit fd82b4e

Browse files
winner245winner245
committed
Speed-up range operations in vector<bool>
1 parent ee3e17d commit fd82b4e

File tree

6 files changed

+271
-1
lines changed

6 files changed

+271
-1
lines changed

‎libcxx/include/__algorithm/copy.h‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414
#include <__algorithm/min.h>
1515
#include <__config>
1616
#include <__fwd/bit_reference.h>
17+
#include <__iterator/distance.h>
1718
#include <__iterator/iterator_traits.h>
1819
#include <__iterator/segmented_iterator.h>
1920
#include <__memory/pointer_traits.h>
2021
#include <__type_traits/common_type.h>
2122
#include <__type_traits/enable_if.h>
23+
#include <__type_traits/is_convertible.h>
2224
#include <__utility/move.h>
2325
#include <__utility/pair.h>
2426

@@ -154,6 +156,25 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI __bit_iterator<_Cp, false> _
154156
return __result;
155157
}
156158

159+
template <class _InIter,
160+
class _Sent,
161+
__enable_if_t<__has_input_iterator_category<_InIter>::value &&
162+
!__has_random_access_iterator_category<_InIter>::value,
163+
int> = 0>
164+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InIter>::difference_type
165+
__iter_sent_distance(_InIter __first, _Sent __last) {
166+
typename iterator_traits<_InIter>::difference_type __r(0);
167+
for (; __first != __last; ++__first)
168+
++__r;
169+
return __r;
170+
}
171+
172+
template <class _InIter, class _Sent, __enable_if_t<__has_random_access_iterator_category<_InIter>::value, int> = 0>
173+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX17 typename iterator_traits<_InIter>::difference_type
174+
__iter_sent_distance(_InIter __first, _Sent __last) {
175+
return static_cast<typename iterator_traits<_InIter>::difference_type>(__last - __first);
176+
}
177+
157178
struct __copy_impl {
158179
template <class _InIter, class _Sent, class _OutIter>
159180
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, _OutIter>
@@ -227,6 +248,64 @@ struct __copy_impl {
227248
return std::make_pair(__last, std::__copy_unaligned(__first, __last, __result));
228249
}
229250

251+
template <class _InIter,
252+
class _Sent,
253+
class _Cp,
254+
__enable_if_t<(__has_forward_iterator_category<_InIter>::value ||
255+
__has_iterator_concept_convertible_to<_InIter, forward_iterator_tag>::value) &&
256+
is_convertible<typename iterator_traits<_InIter>::value_type, bool>::value,
257+
int> = 0>
258+
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_InIter, __bit_iterator<_Cp, false> >
259+
operator()(_InIter __first, _Sent __last, __bit_iterator<_Cp, false> __result) const {
260+
using _It = __bit_iterator<_Cp, false>;
261+
using __storage_type = typename _It::__storage_type;
262+
#if _LIBCPP_STD_VER >= 20
263+
__storage_type __n = static_cast<__storage_type>(std::ranges::distance(__first, __last));
264+
#else
265+
__storage_type __n = static_cast<__storage_type>(std::__iter_sent_distance(__first, __last));
266+
#endif
267+
const unsigned __bits_per_word = _It::__bits_per_word;
268+
269+
if (__first != __last) {
270+
// do first partial word, if present
271+
if (__result.__ctz_ != 0) {
272+
__storage_type __clz = static_cast<__storage_type>(__bits_per_word - __result.__ctz_);
273+
__storage_type __dn = std::min(__clz, __n);
274+
__storage_type __w = *__result.__seg_;
275+
__storage_type __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
276+
__w &= ~__m;
277+
for (__storage_type __i = 0; __i < __dn; ++__i, ++__first)
278+
__w |= static_cast<__storage_type>(*__first) << __result.__ctz_++;
279+
*__result.__seg_ = __w;
280+
if (__result.__ctz_ == __bits_per_word) {
281+
__result.__ctz_ = 0;
282+
++__result.__seg_;
283+
}
284+
__n -= __dn;
285+
}
286+
}
287+
// do middle whole words, if present
288+
__storage_type __nw = __n / __bits_per_word;
289+
__n -= __nw * __bits_per_word;
290+
for (; __nw; --__nw) {
291+
__storage_type __w = 0;
292+
for (__storage_type __i = 0; __i < __bits_per_word; ++__i, ++__first)
293+
__w |= static_cast<__storage_type>(*__first) << __i;
294+
*__result.__seg_++ = __w;
295+
}
296+
// do last partial word, if present
297+
if (__n) {
298+
__storage_type __w = 0;
299+
for (__storage_type __i = 0; __i < __n; ++__i, ++__first)
300+
__w |= static_cast<__storage_type>(*__first) << __i;
301+
__storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
302+
*__result.__seg_ &= ~__m;
303+
*__result.__seg_ |= __w;
304+
__result.__ctz_ = __n;
305+
}
306+
return std::make_pair(std::move(__first), std::move(__result));
307+
}
308+
230309
// At this point, the iterators have been unwrapped so any `contiguous_iterator` has been unwrapped to a pointer.
231310
template <class _In, class _Out, __enable_if_t<__can_lower_copy_assignment_to_memmove<_In, _Out>::value, int> = 0>
232311
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 pair<_In*, _Out*>

‎libcxx/include/__bit_reference‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
#include <__type_traits/is_constant_evaluated.h>
3636
#include <__type_traits/is_same.h>
3737
#include <__type_traits/is_unsigned.h>
38+
#include <__type_traits/is_convertible.h>
3839
#include <__type_traits/void_t.h>
3940
#include <__utility/pair.h>
4041
#include <__utility/swap.h>
@@ -463,6 +464,16 @@ private:
463464
template <class _Dp>
464465
friend struct __bit_array;
465466

467+
template <class _InIter,
468+
class _Sent,
469+
class _Dp,
470+
__enable_if_t<(__has_forward_iterator_category<_InIter>::value ||
471+
__has_iterator_concept_convertible_to<_InIter, forward_iterator_tag>::value) &&
472+
is_convertible<typename iterator_traits<_InIter>::value_type, bool>::value,
473+
int> >
474+
_LIBCPP_CONSTEXPR_SINCE_CXX14 friend pair<_InIter, __bit_iterator<_Dp, false> >
475+
__copy_impl::operator()(_InIter __first, _Sent __last, __bit_iterator<_Dp, false> __result) const;
476+
466477
template <bool _FillVal, class _Dp>
467478
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend void
468479
__fill_n_bool(__bit_iterator<_Dp, false> __first, typename __size_difference_type_traits<_Dp>::size_type __n);

‎libcxx/include/__vector/vector_bool.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,6 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
174174
if constexpr (ranges::forward_range<_Range> || ranges::sized_range<_Range>) {
175175
auto __n = static_cast<size_type>(ranges::distance(__range));
176176
__init_with_size(ranges::begin(__range), ranges::end(__range), __n);
177-
178177
} else {
179178
__init_with_sentinel(ranges::begin(__range), ranges::end(__range));
180179
}

‎libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
10+
11+
#include <cstdint>
12+
#include <cstdlib>
13+
#include <cstring>
14+
#include <deque>
15+
#include <functional>
16+
#include <memory>
17+
#include <string>
18+
#include <vector>
19+
20+
#include "benchmark/benchmark.h"
21+
#include "ContainerBenchmarks.h"
22+
#include "../../std/containers/from_range_helpers.h"
23+
#include "../GenerateInput.h"
24+
#include "test_iterators.h"
25+
26+
using namespace ContainerBenchmarks;
27+
28+
using vb_iter = std::vector<bool>::iterator;
29+
30+
// Benchmarks for forward_iterator or forward_range
31+
32+
BENCHMARK_CAPTURE(BM_ConstructIterIter<std::vector<bool>>,
33+
forward_iterator,
34+
getRandomIntegerInputs<bool>,
35+
forward_iterator<vb_iter>())
36+
->Arg(5140480);
37+
38+
BENCHMARK_CAPTURE(BM_ConstructFromRange<std::vector<bool>>,
39+
forward_range,
40+
getRandomIntegerInputs<bool>,
41+
forward_range_wrapper<vb_iter>())
42+
->Arg(5140480);
43+
44+
BENCHMARK_CAPTURE(
45+
BM_AssignIterIter<std::vector<bool>>, forward_iterator, getRandomIntegerInputs<bool>, forward_iterator<vb_iter>())
46+
->Arg(5140480);
47+
48+
BENCHMARK_CAPTURE(
49+
BM_AssignRange<std::vector<bool>>, forward_range, getRandomIntegerInputs<bool>, forward_range_wrapper<vb_iter>())
50+
->Arg(5140480);
51+
52+
BENCHMARK_CAPTURE(BM_InsertIterIterIter<std::vector<bool>>,
53+
forward_iterator,
54+
getRandomIntegerInputs<bool>,
55+
forward_iterator<vb_iter>())
56+
->Arg(5140480);
57+
58+
BENCHMARK_CAPTURE(
59+
BM_InsertRange<std::vector<bool>>, forward_range, getRandomIntegerInputs<bool>, forward_range_wrapper<vb_iter>())
60+
->Arg(5140480);
61+
62+
BENCHMARK_CAPTURE(
63+
BM_AppendRange<std::vector<bool>>, forward_range, getRandomIntegerInputs<bool>, forward_range_wrapper<vb_iter>())
64+
->Arg(5140480);
65+
66+
// Benchmarks for random_access_iterator or random_access_range
67+
68+
BENCHMARK_CAPTURE(BM_ConstructIterIter<std::vector<bool>>,
69+
random_access_iterator,
70+
getRandomIntegerInputs<bool>,
71+
random_access_iterator<vb_iter>())
72+
->Arg(5140480);
73+
74+
BENCHMARK_CAPTURE(BM_ConstructFromRange<std::vector<bool>>,
75+
random_access_range,
76+
getRandomIntegerInputs<bool>,
77+
random_access_range_wrapper<vb_iter>())
78+
->Arg(5140480);
79+
80+
BENCHMARK_CAPTURE(BM_AssignIterIter<std::vector<bool>>,
81+
random_access_iterator,
82+
getRandomIntegerInputs<bool>,
83+
random_access_iterator<vb_iter>())
84+
->Arg(5140480);
85+
86+
BENCHMARK_CAPTURE(BM_AssignRange<std::vector<bool>>,
87+
random_access_range,
88+
getRandomIntegerInputs<bool>,
89+
random_access_range_wrapper<vb_iter>())
90+
->Arg(5140480);
91+
92+
BENCHMARK_CAPTURE(BM_InsertIterIterIter<std::vector<bool>>,
93+
random_access_iterator,
94+
getRandomIntegerInputs<bool>,
95+
random_access_iterator<vb_iter>())
96+
->Arg(5140480);
97+
98+
BENCHMARK_CAPTURE(BM_InsertRange<std::vector<bool>>,
99+
random_access_range,
100+
getRandomIntegerInputs<bool>,
101+
random_access_range_wrapper<vb_iter>())
102+
->Arg(5140480);
103+
104+
BENCHMARK_CAPTURE(BM_AppendRange<std::vector<bool>>,
105+
random_access_range,
106+
getRandomIntegerInputs<bool>,
107+
random_access_range_wrapper<vb_iter>())
108+
->Arg(5140480);
109+
110+
BENCHMARK_MAIN();

‎libcxx/test/std/algorithms/alg.modifying.operations/alg.copy/copy.pass.cpp‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
// XFAIL: FROZEN-CXX03-HEADERS-FIXME
1616

1717
#include <algorithm>
18+
#include <array>
1819
#include <cassert>
1920
#include <vector>
2021

@@ -64,6 +65,30 @@ struct TestInIters {
6465
}
6566
};
6667

68+
template <std::size_t N>
69+
struct TestFwdIterInBitIterOut {
70+
std::array<bool, N> in = {};
71+
template <class FwdIter>
72+
TEST_CONSTEXPR_CXX20 void operator()() {
73+
for (std::size_t i = 0; i < in.size(); i += 2)
74+
in[i] = true;
75+
76+
{ // Test with full bytes
77+
std::vector<bool> out(N);
78+
std::copy(FwdIter(in.data()), FwdIter(in.data() + N), out.begin());
79+
for (std::size_t i = 0; i < N; ++i)
80+
assert(out[i] == static_cast<bool>(in[i]));
81+
}
82+
{ // Test with partial bytes in both front and back
83+
std::vector<bool> out(N + 8);
84+
std::copy(FwdIter(in.data()), FwdIter(in.data() + N), out.begin() + 4);
85+
for (std::size_t i = 0; i < N; ++i)
86+
assert(out[i + 4] == static_cast<bool>(in[i]));
87+
}
88+
}
89+
};
90+
91+
6792
TEST_CONSTEXPR_CXX20 bool test_vector_bool(std::size_t N) {
6893
std::vector<bool> in(N, false);
6994
for (std::size_t i = 0; i < N; i += 2)
@@ -259,6 +284,14 @@ TEST_CONSTEXPR_CXX20 bool test() {
259284
}
260285
}
261286

287+
{ // Test std::copy() with forward_iterator-pair inputs and vector<bool>::iterator output
288+
types::for_each(types::forward_iterator_list<bool*>(), TestFwdIterInBitIterOut<8>());
289+
types::for_each(types::forward_iterator_list<bool*>(), TestFwdIterInBitIterOut<19>());
290+
types::for_each(types::forward_iterator_list<bool*>(), TestFwdIterInBitIterOut<32>());
291+
types::for_each(types::forward_iterator_list<bool*>(), TestFwdIterInBitIterOut<64>());
292+
types::for_each(types::forward_iterator_list<bool*>(), TestFwdIterInBitIterOut<299>());
293+
}
294+
262295
return true;
263296
}
264297

‎libcxx/test/std/containers/from_range_helpers.h‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,44 @@ constexpr auto wrap_input(std::vector<T>& input) {
5050
return std::ranges::subrange(std::move(b), std::move(e));
5151
}
5252

53+
template <class Iter, class Sent = Iter>
54+
class forward_range_wrapper {
55+
public:
56+
using _Iter = forward_iterator<Iter>;
57+
using _Sent = sentinel_wrapper<_Iter>;
58+
59+
forward_range_wrapper() = default;
60+
forward_range_wrapper(Iter begin, Iter end) : begin_(std::move(begin)), end_(std::move(end)) {}
61+
_Iter begin() { return _Iter(std::move(begin_)); }
62+
_Sent end() { return _Sent(_Iter(std::move(end_))); }
63+
64+
private:
65+
Iter begin_;
66+
Sent end_;
67+
};
68+
69+
template <class Iter, class Sent>
70+
forward_range_wrapper(Iter, Sent) -> forward_range_wrapper<Iter, Sent>;
71+
72+
template <class Iter, class Sent = Iter>
73+
class random_access_range_wrapper {
74+
public:
75+
using _Iter = cpp20_random_access_iterator<Iter>;
76+
using _Sent = sized_sentinel<_Iter>;
77+
78+
random_access_range_wrapper() = default;
79+
random_access_range_wrapper(Iter begin, Iter end) : begin_(std::move(begin)), end_(std::move(end)) {}
80+
_Iter begin() { return _Iter(std::move(begin_)); }
81+
_Sent end() { return _Sent(_Iter(std::move(end_))); }
82+
83+
private:
84+
Iter begin_;
85+
Sent end_;
86+
};
87+
88+
template <class Iter, class Sent>
89+
random_access_range_wrapper(Iter, Sent) -> random_access_range_wrapper<Iter, Sent>;
90+
5391
struct KeyValue {
5492
int key; // Only the key is considered for equality comparison.
5593
char value; // Allows distinguishing equivalent instances.

0 commit comments

Comments
 (0)