Skip to content

Commit 54abc4f

Browse files
winner245winner245
committed
Speed-up range operations in vector<bool>
1 parent 76714be commit 54abc4f

File tree

6 files changed

+218
-0
lines changed

6 files changed

+218
-0
lines changed

‎libcxx/include/__algorithm/copy.h‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
#include <__algorithm/for_each_segment.h>
1414
#include <__algorithm/min.h>
1515
#include <__config>
16+
#include <__fwd/bit_reference.h>
17+
#include <__iterator/distance.h>
1618
#include <__iterator/iterator_traits.h>
1719
#include <__iterator/segmented_iterator.h>
1820
#include <__type_traits/common_type.h>
@@ -103,6 +105,60 @@ struct __copy_impl {
103105
}
104106
};
105107

108+
template <class _InIter, class _Sent, class _Cp>
109+
_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI pair<_InIter, __bit_iterator<_Cp, false>>
110+
__copy_bool(_InIter __first, _Sent __last, __bit_iterator<_Cp, false> __result) {
111+
using _It = __bit_iterator<_Cp, false>;
112+
using __storage_type = typename _It::__storage_type;
113+
__storage_type __n = static_cast<__storage_type>(std::distance(__first, __last));
114+
const unsigned __bits_per_word = _It::__bits_per_word;
115+
116+
if (__n) {
117+
// do first partial word, if present
118+
if (__result.__ctz_ != 0) {
119+
__storage_type __clz = static_cast<__storage_type>(__bits_per_word - __result.__ctz_);
120+
__storage_type __dn = std::min(__clz, __n);
121+
__storage_type __w = *__result.__seg_;
122+
__storage_type __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
123+
__w &= ~__m;
124+
for (__storage_type __i = 0; __i < __dn; ++__i, ++__first)
125+
__w |= static_cast<__storage_type>(*__first) << __result.__ctz_++;
126+
*__result.__seg_ = __w;
127+
if (__result.__ctz_ == __bits_per_word) {
128+
__result.__ctz_ = 0;
129+
++__result.__seg_;
130+
}
131+
__n -= __dn;
132+
}
133+
}
134+
// do middle whole words, if present
135+
__storage_type __nw = __n / __bits_per_word;
136+
__n -= __nw * __bits_per_word;
137+
for (; __nw; --__nw) {
138+
__storage_type __w = 0;
139+
for (__storage_type __i = 0; __i < __bits_per_word; ++__i, ++__first)
140+
__w |= static_cast<__storage_type>(*__first) << __i;
141+
*__result.__seg_++ = __w;
142+
}
143+
// do last partial word, if present
144+
if (__n) {
145+
__storage_type __w = 0;
146+
for (__storage_type __i = 0; __i < __n; ++__i, ++__first)
147+
__w |= static_cast<__storage_type>(*__first) << __i;
148+
__storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
149+
*__result.__seg_ &= ~__m;
150+
*__result.__seg_ |= __w;
151+
__result.__ctz_ = __n;
152+
}
153+
return std::make_pair(std::move(__first), std::move(__result));
154+
}
155+
156+
template <class _InIter, class _Cp, __enable_if_t<__has_forward_iterator_category<_InIter>::value, int> = 0>
157+
pair<_InIter, __bit_iterator<_Cp, false>> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
158+
__copy(_InIter __first, _InIter __last, __bit_iterator<_Cp, false> __result) {
159+
return std::__copy_bool(std::move(__first), std::move(__last), std::move(__result));
160+
}
161+
106162
template <class _InIter, class _Sent, class _OutIter>
107163
pair<_InIter, _OutIter> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
108164
__copy(_InIter __first, _Sent __last, _OutIter __result) {

‎libcxx/include/__bit_reference‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#ifndef _LIBCPP___BIT_REFERENCE
1111
#define _LIBCPP___BIT_REFERENCE
1212

13+
#include <__algorithm/copy.h>
1314
#include <__algorithm/copy_n.h>
1415
#include <__algorithm/fill_n.h>
1516
#include <__algorithm/min.h>
@@ -966,6 +967,10 @@ private:
966967
template <class _Dp>
967968
friend struct __bit_array;
968969

970+
template <class _InIter, class _Sent, class _Dp>
971+
_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI friend pair<_InIter, __bit_iterator<_Dp, false>>
972+
__copy_bool(_InIter __first, _Sent __last, __bit_iterator<_Dp, false> __result);
973+
969974
template <bool _FillVal, class _Dp>
970975
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend void
971976
__fill_n_bool(__bit_iterator<_Dp, false> __first, typename _Dp::size_type __n);

‎libcxx/include/__cxx03/__algorithm/copy.h‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
#include <__cxx03/__algorithm/iterator_operations.h>
1515
#include <__cxx03/__algorithm/min.h>
1616
#include <__cxx03/__config>
17+
#include <__cxx03/__fwd/bit_reference.h>
18+
#include <__cxx03/__iterator/distance.h>
19+
#include <__cxx03/__iterator/iterator_traits.h>
1720
#include <__cxx03/__iterator/segmented_iterator.h>
1821
#include <__cxx03/__type_traits/common_type.h>
1922
#include <__cxx03/__utility/move.h>
@@ -103,6 +106,60 @@ struct __copy_impl {
103106
}
104107
};
105108

109+
template <class _InIter, class _Sent, class _Cp>
110+
_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI pair<_InIter, __bit_iterator<_Cp, false>>
111+
__copy_bool(_InIter __first, _Sent __last, __bit_iterator<_Cp, false> __result) {
112+
using _It = __bit_iterator<_Cp, false>;
113+
using __storage_type = typename _It::__storage_type;
114+
__storage_type __n = static_cast<__storage_type>(std::distance(__first, __last));
115+
const unsigned __bits_per_word = _It::__bits_per_word;
116+
117+
if (__n) {
118+
// do first partial word, if present
119+
if (__result.__ctz_ != 0) {
120+
__storage_type __clz = static_cast<__storage_type>(__bits_per_word - __result.__ctz_);
121+
__storage_type __dn = std::min(__clz, __n);
122+
__storage_type __w = *__result.__seg_;
123+
__storage_type __m = (~__storage_type(0) << __result.__ctz_) & (~__storage_type(0) >> (__clz - __dn));
124+
__w &= ~__m;
125+
for (__storage_type __i = 0; __i < __dn; ++__i, ++__first)
126+
__w |= static_cast<__storage_type>(*__first) << __result.__ctz_++;
127+
*__result.__seg_ = __w;
128+
if (__result.__ctz_ == __bits_per_word) {
129+
__result.__ctz_ = 0;
130+
++__result.__seg_;
131+
}
132+
__n -= __dn;
133+
}
134+
}
135+
// do middle whole words, if present
136+
__storage_type __nw = __n / __bits_per_word;
137+
__n -= __nw * __bits_per_word;
138+
for (; __nw; --__nw) {
139+
__storage_type __w = 0;
140+
for (__storage_type __i = 0; __i < __bits_per_word; ++__i, ++__first)
141+
__w |= static_cast<__storage_type>(*__first) << __i;
142+
*__result.__seg_++ = __w;
143+
}
144+
// do last partial word, if present
145+
if (__n) {
146+
__storage_type __w = 0;
147+
for (__storage_type __i = 0; __i < __n; ++__i, ++__first)
148+
__w |= static_cast<__storage_type>(*__first) << __i;
149+
__storage_type __m = ~__storage_type(0) >> (__bits_per_word - __n);
150+
*__result.__seg_ &= ~__m;
151+
*__result.__seg_ |= __w;
152+
__result.__ctz_ = __n;
153+
}
154+
return std::make_pair(std::move(__first), std::move(__result));
155+
}
156+
157+
template <class _InIter, class _Cp, __enable_if_t<__has_forward_iterator_category<_InIter>::value, int> = 0>
158+
pair<_InIter, __bit_iterator<_Cp, false>> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
159+
__copy(_InIter __first, _InIter __last, __bit_iterator<_Cp, false> __result) {
160+
return std::__copy_bool(std::move(__first), std::move(__last), std::move(__result));
161+
}
162+
106163
template <class _AlgPolicy, class _InIter, class _Sent, class _OutIter>
107164
pair<_InIter, _OutIter> inline _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14
108165
__copy(_InIter __first, _Sent __last, _OutIter __result) {

‎libcxx/include/__cxx03/__bit_reference‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#ifndef _LIBCPP___CXX03___BIT_REFERENCE
1111
#define _LIBCPP___CXX03___BIT_REFERENCE
1212

13+
#include <__cxx03/__algorithm/copy.h>
1314
#include <__cxx03/__algorithm/copy_n.h>
1415
#include <__cxx03/__algorithm/fill_n.h>
1516
#include <__cxx03/__algorithm/min.h>
@@ -966,6 +967,10 @@ private:
966967
template <class _Dp>
967968
friend struct __bit_array;
968969

970+
template <class _InIter, class _Sent, class _Dp>
971+
_LIBCPP_CONSTEXPR_SINCE_CXX14 _LIBCPP_HIDE_FROM_ABI friend pair<_InIter, __bit_iterator<_Dp, false>>
972+
__copy_bool(_InIter __first, _Sent __last, __bit_iterator<_Dp, false> __result);
973+
969974
template <bool _FillVal, class _Dp>
970975
_LIBCPP_CONSTEXPR_SINCE_CXX20 friend void
971976
__fill_n_bool(__bit_iterator<_Dp, false> __first, typename _Dp::size_type __n);

‎libcxx/test/benchmarks/containers/ContainerBenchmarks.h‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,30 @@ void BM_Assignment(benchmark::State& st, Container) {
5151
}
5252
}
5353

54+
template <class Container, class GenInputs>
55+
void BM_assign_iter_iter(benchmark::State& st, Container c, GenInputs gen) {
56+
auto in = gen(st.range(0));
57+
auto beg = in.begin();
58+
auto end = in.end();
59+
for (auto _ : st) {
60+
c.assign(beg, end);
61+
DoNotOptimizeData(c);
62+
DoNotOptimizeData(in);
63+
benchmark::ClobberMemory();
64+
}
65+
}
66+
67+
template <std::size_t... sz, typename Container, typename GenInputs>
68+
void BM_assign_range(benchmark::State& st, Container c, GenInputs gen) {
69+
auto in = gen(st.range(0));
70+
for (auto _ : st) {
71+
c.assign_range(in);
72+
DoNotOptimizeData(c);
73+
DoNotOptimizeData(in);
74+
benchmark::ClobberMemory();
75+
}
76+
}
77+
5478
template <std::size_t... sz, typename Container, typename GenInputs>
5579
void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
5680
auto v = gen(1, sz...);
@@ -108,6 +132,40 @@ void BM_Pushback_no_grow(benchmark::State& state, Container c) {
108132
}
109133
}
110134

135+
template <class Container, class GenInputs>
136+
void BM_insert_iter_iter_iter(benchmark::State& st, Container c, GenInputs gen) {
137+
auto in = gen(st.range(0));
138+
const auto beg = in.begin();
139+
const auto end = in.end();
140+
for (auto _ : st) {
141+
c.resize(100);
142+
c.insert(c.begin() + 50, beg, end);
143+
DoNotOptimizeData(c);
144+
benchmark::ClobberMemory();
145+
}
146+
}
147+
148+
template <class Container, class GenInputs>
149+
void BM_insert_range(benchmark::State& st, Container c, GenInputs gen) {
150+
auto in = gen(st.range(0));
151+
for (auto _ : st) {
152+
c.resize(100);
153+
c.insert_range(c.begin() + 50, in);
154+
DoNotOptimizeData(c);
155+
benchmark::ClobberMemory();
156+
}
157+
}
158+
159+
template <class Container, class GenInputs>
160+
void BM_append_range(benchmark::State& st, Container c, GenInputs gen) {
161+
auto in = gen(st.range(0));
162+
for (auto _ : st) {
163+
c.append_range(in);
164+
DoNotOptimizeData(c);
165+
benchmark::ClobberMemory();
166+
}
167+
}
168+
111169
template <class Container, class GenInputs>
112170
void BM_InsertValue(benchmark::State& st, Container c, GenInputs gen) {
113171
auto in = gen(st.range(0));

‎libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
10+
11+
#include <cstdint>
12+
#include <cstdlib>
13+
#include <cstring>
14+
#include <deque>
15+
#include <functional>
16+
#include <memory>
17+
#include <string>
18+
#include <vector>
19+
20+
#include "benchmark/benchmark.h"
21+
#include "ContainerBenchmarks.h"
22+
#include "../GenerateInput.h"
23+
24+
using namespace ContainerBenchmarks;
25+
26+
BENCHMARK_CAPTURE(BM_ConstructIterIter, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)->Arg(5140480);
27+
BENCHMARK_CAPTURE(BM_ConstructFromRange, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)->Arg(5140480);
28+
29+
BENCHMARK_CAPTURE(BM_assign_iter_iter, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)->Arg(5140480);
30+
BENCHMARK_CAPTURE(BM_assign_range, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)->Arg(5140480);
31+
32+
BENCHMARK_CAPTURE(BM_insert_iter_iter_iter, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)
33+
->Arg(5140480);
34+
BENCHMARK_CAPTURE(BM_insert_range, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)->Arg(5140480);
35+
BENCHMARK_CAPTURE(BM_append_range, vector_bool, std::vector<bool>{}, getRandomIntegerInputs<bool>)->Arg(5140480);
36+
37+
BENCHMARK_MAIN();

0 commit comments

Comments
 (0)