Skip to content

Commit 56ce456

Browse files
winner245winner245
committed
Speed-up and refactor move-assignment operator for vector<bool>
1 parent 15c2d4b commit 56ce456

File tree

6 files changed

+284
-112
lines changed

6 files changed

+284
-112
lines changed

‎libcxx/include/__vector/vector_bool.h‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -417,6 +417,8 @@ class _LIBCPP_TEMPLATE_VIS vector<bool, _Allocator> {
417417
__guard.__complete();
418418
}
419419

420+
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __copy_by_words(const vector& __v);
421+
420422
template <class _Iterator, class _Sentinel>
421423
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void __assign_with_sentinel(_Iterator __first, _Sentinel __last);
422424

@@ -716,18 +718,24 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>::vector(const vector& __v
716718
}
717719
}
718720

721+
// This function copies entire storage words instead of individual bits for improved performance.
722+
template <class _Allocator>
723+
_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI void vector<bool, _Allocator>::__copy_by_words(const vector& __v) {
724+
if (__v.__size_) {
725+
if (__v.__size_ > capacity()) {
726+
__vdeallocate();
727+
__vallocate(__v.__size_);
728+
}
729+
std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
730+
}
731+
__size_ = __v.__size_;
732+
}
733+
719734
template <class _Allocator>
720735
_LIBCPP_CONSTEXPR_SINCE_CXX20 vector<bool, _Allocator>& vector<bool, _Allocator>::operator=(const vector& __v) {
721736
if (this != std::addressof(__v)) {
722737
__copy_assign_alloc(__v);
723-
if (__v.__size_) {
724-
if (__v.__size_ > capacity()) {
725-
__vdeallocate();
726-
__vallocate(__v.__size_);
727-
}
728-
std::copy(__v.__begin_, __v.__begin_ + __external_cap_to_internal(__v.__size_), __begin_);
729-
}
730-
__size_ = __v.__size_;
738+
__copy_by_words(__v);
731739
}
732740
return *this;
733741
}
@@ -775,7 +783,7 @@ vector<bool, _Allocator>::operator=(vector&& __v)
775783
template <class _Allocator>
776784
_LIBCPP_CONSTEXPR_SINCE_CXX20 void vector<bool, _Allocator>::__move_assign(vector& __c, false_type) {
777785
if (__alloc_ != __c.__alloc_)
778-
assign(__c.begin(), __c.end());
786+
__copy_by_words(__c);
779787
else
780788
__move_assign(__c, true_type());
781789
}

‎libcxx/test/benchmarks/containers/ContainerBenchmarks.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,20 @@ void BM_Assignment(benchmark::State& st, Container) {
5151
}
5252
}
5353

54+
template <class Container, class Allocator>
55+
void BM_Move_Assignment(benchmark::State& st, Container, Allocator) {
56+
auto size = st.range(0);
57+
Container c1(Allocator{1});
58+
Container c2(Allocator{2});
59+
// c1.reserve(size);
60+
c2.resize(size);
61+
for (auto _ : st) {
62+
c1 = std::move(c2);
63+
DoNotOptimizeData(c1);
64+
DoNotOptimizeData(c2);
65+
}
66+
}
67+
5468
template <std::size_t... sz, typename Container, typename GenInputs>
5569
void BM_AssignInputIterIter(benchmark::State& st, Container c, GenInputs gen) {
5670
auto v = gen(1, sz...);

‎libcxx/test/benchmarks/containers/vector_bool_operations.bench.cpp‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// UNSUPPORTED: c++03, c++11, c++14, c++17, c++20
10+
11+
#include <cstdint>
12+
#include <cstdlib>
13+
#include <cstring>
14+
#include <deque>
15+
#include <functional>
16+
#include <memory>
17+
#include <string>
18+
#include <vector>
19+
20+
#include "benchmark/benchmark.h"
21+
#include "ContainerBenchmarks.h"
22+
#include "../GenerateInput.h"
23+
#include "sized_allocator.h"
24+
#include "test_allocator.h"
25+
26+
using namespace ContainerBenchmarks;
27+
28+
BENCHMARK_CAPTURE(BM_Move_Assignment,
29+
vector_bool_uint32_t,
30+
std::vector<bool, sized_allocator<bool, std::uint32_t, std::int32_t>>{},
31+
sized_allocator<bool, std::uint32_t, std::int32_t>{})
32+
->Arg(5140480);
33+
34+
BENCHMARK_CAPTURE(BM_Move_Assignment,
35+
vector_bool_uint64_t,
36+
std::vector<bool, sized_allocator<bool, std::uint64_t, std::int64_t>>{},
37+
sized_allocator<bool, std::uint64_t, std::int64_t>{})
38+
->Arg(5140480);
39+
40+
BENCHMARK_CAPTURE(BM_Move_Assignment,
41+
vector_bool_size_t,
42+
std::vector<bool, sized_allocator<bool, std::size_t, std::ptrdiff_t>>{},
43+
sized_allocator<bool, std::size_t, std::ptrdiff_t>{})
44+
->Arg(5140480);
45+
46+
BENCHMARK_MAIN();

‎libcxx/test/std/containers/sequences/vector.bool/assign_copy.pass.cpp‎

Lines changed: 63 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -7,49 +7,77 @@
77
//===----------------------------------------------------------------------===//
88

99
// <vector>
10+
// vector<bool>
1011

1112
// vector& operator=(const vector& c);
1213

13-
#include <vector>
1414
#include <cassert>
15-
#include "test_macros.h"
16-
#include "test_allocator.h"
15+
#include <vector>
16+
1717
#include "min_allocator.h"
18+
#include "test_allocator.h"
19+
#include "test_macros.h"
1820

19-
TEST_CONSTEXPR_CXX20 bool tests()
20-
{
21-
{
22-
std::vector<bool, test_allocator<bool> > l(3, true, test_allocator<bool>(5));
23-
std::vector<bool, test_allocator<bool> > l2(l, test_allocator<bool>(3));
24-
l2 = l;
25-
assert(l2 == l);
26-
assert(l2.get_allocator() == test_allocator<bool>(3));
27-
}
28-
{
29-
std::vector<bool, other_allocator<bool> > l(3, true, other_allocator<bool>(5));
30-
std::vector<bool, other_allocator<bool> > l2(l, other_allocator<bool>(3));
31-
l2 = l;
32-
assert(l2 == l);
33-
assert(l2.get_allocator() == other_allocator<bool>(5));
34-
}
35-
#if TEST_STD_VER >= 11
36-
{
37-
std::vector<bool, min_allocator<bool> > l(3, true, min_allocator<bool>());
38-
std::vector<bool, min_allocator<bool> > l2(l, min_allocator<bool>());
39-
l2 = l;
40-
assert(l2 == l);
41-
assert(l2.get_allocator() == min_allocator<bool>());
42-
}
43-
#endif
21+
TEST_CONSTEXPR_CXX20 void test_copy_assignment(unsigned N) {
22+
//
23+
// Test with insufficient space where reallocation occurs during assignment
24+
//
25+
{ // pocca = true_type, thus copy-assign the allocator
26+
std::vector<bool, other_allocator<bool> > l(N, true, other_allocator<bool>(5));
27+
std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(3));
28+
l2 = l;
29+
assert(l2 == l);
30+
assert(l2.get_allocator() == other_allocator<bool>(5));
31+
}
32+
{
33+
std::vector<bool, test_allocator<bool> > l(N + 64, true, test_allocator<bool>(5));
34+
std::vector<bool, test_allocator<bool> > l2(10, false, test_allocator<bool>(3));
35+
l2 = l;
36+
assert(l2 == l);
37+
assert(l2.get_allocator() == test_allocator<bool>(3));
38+
}
39+
{
40+
std::vector<bool, min_allocator<bool> > l(N + 64, true, min_allocator<bool>());
41+
std::vector<bool, min_allocator<bool> > l2(N / 2, false, min_allocator<bool>());
42+
l2 = l;
43+
assert(l2 == l);
44+
assert(l2.get_allocator() == min_allocator<bool>());
45+
}
46+
47+
//
48+
// Test with sufficient size where no reallocation occurs during assignment
49+
//
50+
{
51+
std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
52+
std::vector<bool, test_allocator<bool> > l2(N + 64, false, test_allocator<bool>(3));
53+
l2 = l;
54+
assert(l2 == l);
55+
assert(l2.get_allocator() == test_allocator<bool>(3));
56+
}
57+
{ // pocca = true_type, thus copy-assign the allocator
58+
std::vector<bool, other_allocator<bool> > l(N, true, other_allocator<bool>(5));
59+
std::vector<bool, other_allocator<bool> > l2(N * 2, false, other_allocator<bool>(3));
60+
l2.reserve(5);
61+
l2 = l;
62+
assert(l2 == l);
63+
assert(l2.get_allocator() == other_allocator<bool>(5));
64+
}
65+
}
66+
67+
TEST_CONSTEXPR_CXX20 bool tests() {
68+
test_copy_assignment(9);
69+
test_copy_assignment(33);
70+
test_copy_assignment(65);
71+
test_copy_assignment(257);
72+
test_copy_assignment(1000);
4473

45-
return true;
74+
return true;
4675
}
4776

48-
int main(int, char**)
49-
{
50-
tests();
51-
#if TEST_STD_VER > 17
52-
static_assert(tests());
77+
int main(int, char**) {
78+
tests();
79+
#if TEST_STD_VER >= 20
80+
static_assert(tests());
5381
#endif
54-
return 0;
82+
return 0;
5583
}

‎libcxx/test/std/containers/sequences/vector.bool/assign_move.pass.cpp‎

Lines changed: 84 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -9,82 +9,98 @@
99
// UNSUPPORTED: c++03
1010

1111
// <vector>
12+
// vector<bool>
1213

1314
// vector& operator=(vector&& c);
1415

15-
#include <vector>
1616
#include <cassert>
17-
#include "test_macros.h"
18-
#include "test_allocator.h"
17+
#include <vector>
18+
1919
#include "min_allocator.h"
20+
#include "test_allocator.h"
21+
#include "test_macros.h"
22+
23+
TEST_CONSTEXPR_CXX20 void test_move_assignment(unsigned N) {
24+
//
25+
// Testing for O(1) ownership move
26+
//
27+
{ // Test with pocma = true_type, thus performing an ownership move.
28+
std::vector<bool, other_allocator<bool> > l(N, true, other_allocator<bool>(5));
29+
std::vector<bool, other_allocator<bool> > lo(N, true, other_allocator<bool>(5));
30+
std::vector<bool, other_allocator<bool> > l2(N + 10, false, other_allocator<bool>(42));
31+
l2 = std::move(l);
32+
assert(l2 == lo);
33+
LIBCPP_ASSERT(l.empty()); // After move, source vector is in a vliad but unspecified state. libc++ leaves it empty.
34+
assert(l2.get_allocator() == lo.get_allocator());
35+
}
36+
{ // Test with pocma = false_type but equal allocators, thus performing an ownership move.
37+
std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
38+
std::vector<bool, test_allocator<bool> > lo(N, true, test_allocator<bool>(5));
39+
std::vector<bool, test_allocator<bool> > l2(N + 10, false, test_allocator<bool>(5));
40+
l2 = std::move(l);
41+
assert(l2 == lo);
42+
LIBCPP_ASSERT(l.empty());
43+
assert(l2.get_allocator() == lo.get_allocator());
44+
}
45+
{ // Test with pocma = false_type but equal allocators, thus performing an ownership move.
46+
std::vector<bool, min_allocator<bool> > l(N, true, min_allocator<bool>{});
47+
std::vector<bool, min_allocator<bool> > lo(N, true, min_allocator<bool>{});
48+
std::vector<bool, min_allocator<bool> > l2(N + 10, false, min_allocator<bool>{});
49+
l2 = std::move(l);
50+
assert(l2 == lo);
51+
LIBCPP_ASSERT(l.empty());
52+
assert(l2.get_allocator() == lo.get_allocator());
53+
}
54+
55+
//
56+
// Testing for O(n) element-wise move
57+
//
58+
{ // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
59+
// Reallocation occurs during the element-wise move due to empty destination vector.
60+
std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
61+
std::vector<bool, test_allocator<bool> > lo(N, true, test_allocator<bool>(5));
62+
std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(42));
63+
l2 = std::move(l);
64+
assert(l2 == lo);
65+
LIBCPP_ASSERT(!l.empty());
66+
assert(l2.get_allocator() == test_allocator<bool>(42));
67+
}
68+
{ // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
69+
// Reallocation occurs during the element-wise move due to insufficient destination space.
70+
std::vector<bool, test_allocator<bool> > l(N + 64, true, test_allocator<bool>(5));
71+
std::vector<bool, test_allocator<bool> > lo(N + 64, true, test_allocator<bool>(5));
72+
std::vector<bool, test_allocator<bool> > l2(10, false, test_allocator<bool>(42));
73+
l2 = std::move(l);
74+
assert(l2 == lo);
75+
LIBCPP_ASSERT(!l.empty());
76+
assert(l2.get_allocator() == test_allocator<bool>(42));
77+
}
78+
{ // Test with pocma = false_type and unequal allocators, thus performing an element-wise move.
79+
// No reallocation occurs since source data fits within destination size.
80+
std::vector<bool, test_allocator<bool> > l(N, true, test_allocator<bool>(5));
81+
std::vector<bool, test_allocator<bool> > lo(N, true, test_allocator<bool>(5));
82+
std::vector<bool, test_allocator<bool> > l2(N * 2, false, test_allocator<bool>(42));
83+
l2 = std::move(l);
84+
assert(l2 == lo);
85+
LIBCPP_ASSERT(!l.empty());
86+
assert(l2.get_allocator() == test_allocator<bool>(42));
87+
}
88+
}
2089

21-
TEST_CONSTEXPR_CXX20 bool tests()
22-
{
23-
{
24-
std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
25-
std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
26-
for (int i = 1; i <= 3; ++i)
27-
{
28-
l.push_back(i);
29-
lo.push_back(i);
30-
}
31-
std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(5));
32-
l2 = std::move(l);
33-
assert(l2 == lo);
34-
LIBCPP_ASSERT(l.empty());
35-
assert(l2.get_allocator() == lo.get_allocator());
36-
}
37-
{
38-
std::vector<bool, test_allocator<bool> > l(test_allocator<bool>(5));
39-
std::vector<bool, test_allocator<bool> > lo(test_allocator<bool>(5));
40-
for (int i = 1; i <= 3; ++i)
41-
{
42-
l.push_back(i);
43-
lo.push_back(i);
44-
}
45-
std::vector<bool, test_allocator<bool> > l2(test_allocator<bool>(6));
46-
l2 = std::move(l);
47-
assert(l2 == lo);
48-
assert(!l.empty());
49-
assert(l2.get_allocator() == test_allocator<bool>(6));
50-
}
51-
{
52-
std::vector<bool, other_allocator<bool> > l(other_allocator<bool>(5));
53-
std::vector<bool, other_allocator<bool> > lo(other_allocator<bool>(5));
54-
for (int i = 1; i <= 3; ++i)
55-
{
56-
l.push_back(i);
57-
lo.push_back(i);
58-
}
59-
std::vector<bool, other_allocator<bool> > l2(other_allocator<bool>(6));
60-
l2 = std::move(l);
61-
assert(l2 == lo);
62-
assert(l.empty());
63-
assert(l2.get_allocator() == lo.get_allocator());
64-
}
65-
{
66-
std::vector<bool, min_allocator<bool> > l(min_allocator<bool>{});
67-
std::vector<bool, min_allocator<bool> > lo(min_allocator<bool>{});
68-
for (int i = 1; i <= 3; ++i)
69-
{
70-
l.push_back(i);
71-
lo.push_back(i);
72-
}
73-
std::vector<bool, min_allocator<bool> > l2(min_allocator<bool>{});
74-
l2 = std::move(l);
75-
assert(l2 == lo);
76-
assert(l.empty());
77-
assert(l2.get_allocator() == lo.get_allocator());
78-
}
90+
TEST_CONSTEXPR_CXX20 bool tests() {
91+
test_move_assignment(9);
92+
test_move_assignment(33);
93+
test_move_assignment(65);
94+
test_move_assignment(257);
95+
test_move_assignment(1000);
7996

80-
return true;
97+
return true;
8198
}
8299

83-
int main(int, char**)
84-
{
85-
tests();
86-
#if TEST_STD_VER > 17
87-
static_assert(tests());
100+
int main(int, char**) {
101+
tests();
102+
#if TEST_STD_VER >= 20
103+
static_assert(tests());
88104
#endif
89-
return 0;
105+
return 0;
90106
}

0 commit comments

Comments
 (0)