Skip to content

Commit cbbf303

Browse files
philnik777philnik777
authored
[libc++] Optimize __hash_table copy constructors and assignment (#151951)
``` ---------------------------------------------------------------------------------------------------------------------- Benchmark old new ---------------------------------------------------------------------------------------------------------------------- std::unordered_set<int>::ctor(const&)/0 15.4 ns 14.6 ns std::unordered_set<int>::ctor(const&)/32 686 ns 322 ns std::unordered_set<int>::ctor(const&)/1024 35839 ns 21490 ns std::unordered_set<int>::ctor(const&)/8192 385790 ns 280270 ns std::unordered_set<int>::operator=(const&) (into cleared Container)/0 15.1 ns 15.9 ns std::unordered_set<int>::operator=(const&) (into cleared Container)/32 1077 ns 333 ns std::unordered_set<int>::operator=(const&) (into cleared Container)/1024 31296 ns 9984 ns std::unordered_set<int>::operator=(const&) (into cleared Container)/8192 266776 ns 109418 ns std::unordered_set<int>::operator=(const&) (into partially populated Container)/0 15.1 ns 16.3 ns std::unordered_set<int>::operator=(const&) (into partially populated Container)/32 962 ns 320 ns std::unordered_set<int>::operator=(const&) (into partially populated Container)/1024 31713 ns 10128 ns std::unordered_set<int>::operator=(const&) (into partially populated Container)/8192 266113 ns 108525 ns std::unordered_set<int>::operator=(const&) (into populated Container)/0 0.990 ns 2.03 ns std::unordered_set<int>::operator=(const&) (into populated Container)/32 963 ns 263 ns std::unordered_set<int>::operator=(const&) (into populated Container)/1024 27600 ns 7793 ns std::unordered_set<int>::operator=(const&) (into populated Container)/8192 235295 ns 66248 ns std::unordered_set<std::string>::ctor(const&)/0 16.0 ns 15.0 ns std::unordered_set<std::string>::ctor(const&)/32 2950 ns 1277 ns std::unordered_set<std::string>::ctor(const&)/1024 246935 ns 73762 ns std::unordered_set<std::string>::ctor(const&)/8192 3310895 ns 2468608 ns std::unordered_set<std::string>::operator=(const&) (into cleared Container)/0 16.1 ns 15.8 ns std::unordered_set<std::string>::operator=(const&) (into cleared Container)/32 5856 ns 1039 ns std::unordered_set<std::string>::operator=(const&) (into cleared Container)/1024 170436 ns 74836 ns std::unordered_set<std::string>::operator=(const&) (into cleared Container)/8192 1574235 ns 1096891 ns std::unordered_set<std::string>::operator=(const&) (into partially populated Container)/0 16.0 ns 16.3 ns std::unordered_set<std::string>::operator=(const&) (into partially populated Container)/32 5571 ns 1064 ns std::unordered_set<std::string>::operator=(const&) (into partially populated Container)/1024 199220 ns 75462 ns std::unordered_set<std::string>::operator=(const&) (into partially populated Container)/8192 1552465 ns 1116094 ns std::unordered_set<std::string>::operator=(const&) (into populated Container)/0 1.70 ns 2.14 ns std::unordered_set<std::string>::operator=(const&) (into populated Container)/32 2562 ns 645 ns std::unordered_set<std::string>::operator=(const&) (into populated Container)/1024 228608 ns 39100 ns std::unordered_set<std::string>::operator=(const&) (into populated Container)/8192 2013723 ns 390401 ns ``` Fixes #77657
1 parent b9e33fd commit cbbf303

File tree

16 files changed

+2708
-1253
lines changed

16 files changed

+2708
-1253
lines changed

‎libcxx/docs/ReleaseNotes/22.rst‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ Improvements and New Features
4545

4646
- The performance of ``map::map(const map&)`` has been improved up to 2.3x
4747
- The performance of ``map::operator=(const map&)`` has been improved by up to 11x
48+
- The performance of ``unordered_set::unordered_set(const unordered_set&)`` has been improved by up to 3.3x.
49+
- The performance of ``unordered_set::operator=(const unordered_set&)`` has been improved by up to 5x.
4850

4951
Deprecations and Removals
5052
-------------------------

‎libcxx/include/__hash_table‎

Lines changed: 121 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#ifndef _LIBCPP___HASH_TABLE
1111
#define _LIBCPP___HASH_TABLE
1212

13+
#include <__algorithm/fill_n.h>
1314
#include <__algorithm/max.h>
1415
#include <__algorithm/min.h>
1516
#include <__assert>
@@ -700,6 +701,38 @@ private:
700701

701702
_LIBCPP_HIDE_FROM_ABI size_type& size() _NOEXCEPT { return __size_; }
702703

704+
_LIBCPP_HIDE_FROM_ABI void
705+
__copy_construct(__next_pointer __other_iter, __next_pointer __own_iter, size_t __current_chash) {
706+
auto __bucket_count = bucket_count();
707+
708+
for (; __other_iter; __other_iter = __other_iter->__next_) {
709+
__node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
710+
711+
size_t __new_chash = std::__constrain_hash(__new_node->__hash(), __bucket_count);
712+
if (__new_chash != __current_chash) {
713+
__bucket_list_[__new_chash] = __own_iter;
714+
__current_chash = __new_chash;
715+
}
716+
717+
__own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
718+
__own_iter = __own_iter->__next_;
719+
}
720+
}
721+
722+
_LIBCPP_HIDE_FROM_ABI void __copy_construct(__next_pointer __other_iter) {
723+
__next_pointer __own_iter = __first_node_.__ptr();
724+
{
725+
__node_holder __new_node = __construct_node_hash(__other_iter->__hash(), __other_iter->__upcast()->__get_value());
726+
__own_iter->__next_ = static_cast<__next_pointer>(__new_node.release());
727+
}
728+
729+
size_t __current_chash = std::__constrain_hash(__own_iter->__next_->__hash(), bucket_count());
730+
__bucket_list_[__current_chash] = __own_iter;
731+
__other_iter = __other_iter->__next_;
732+
__own_iter = __own_iter->__next_;
733+
__copy_construct(__other_iter, __own_iter, __current_chash);
734+
}
735+
703736
public:
704737
_LIBCPP_HIDE_FROM_ABI size_type size() const _NOEXCEPT { return __size_; }
705738

@@ -1048,16 +1081,29 @@ __hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const allocator_type& __a
10481081
__max_load_factor_(1.0f) {}
10491082

10501083
template <class _Tp, class _Hash, class _Equal, class _Alloc>
1051-
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u)
1084+
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __other)
10521085
: __bucket_list_(nullptr,
1053-
__bucket_list_deleter(allocator_traits<__pointer_allocator>::select_on_container_copy_construction(
1054-
__u.__bucket_list_.get_deleter().__alloc()),
1086+
__bucket_list_deleter(__pointer_alloc_traits::select_on_container_copy_construction(
1087+
__other.__bucket_list_.get_deleter().__alloc()),
10551088
0)),
1056-
__node_alloc_(allocator_traits<__node_allocator>::select_on_container_copy_construction(__u.__node_alloc())),
1089+
__node_alloc_(__node_traits::select_on_container_copy_construction(__other.__node_alloc())),
10571090
__size_(0),
1058-
__hasher_(__u.hash_function()),
1059-
__max_load_factor_(__u.__max_load_factor_),
1060-
__key_eq_(__u.__key_eq_) {}
1091+
__hasher_(__other.hash_function()),
1092+
__max_load_factor_(__other.__max_load_factor_),
1093+
__key_eq_(__other.__key_eq_) {
1094+
if (__other.size() == 0)
1095+
return;
1096+
1097+
auto& __bucket_list_del = __bucket_list_.get_deleter();
1098+
auto __bucket_count = __other.bucket_count();
1099+
__bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
1100+
__bucket_list_del.size() = __bucket_count;
1101+
1102+
std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);
1103+
1104+
__copy_construct(__other.__first_node_.__next_);
1105+
__size_ = __other.size();
1106+
}
10611107

10621108
template <class _Tp, class _Hash, class _Equal, class _Alloc>
10631109
__hash_table<_Tp, _Hash, _Equal, _Alloc>::__hash_table(const __hash_table& __u, const allocator_type& __a)
@@ -1131,14 +1177,75 @@ void __hash_table<_Tp, _Hash, _Equal, _Alloc>::__copy_assign_alloc(const __hash_
11311177
}
11321178

11331179
template <class _Tp, class _Hash, class _Equal, class _Alloc>
1134-
__hash_table<_Tp, _Hash, _Equal, _Alloc>& __hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __u) {
1135-
if (this != std::addressof(__u)) {
1136-
__copy_assign_alloc(__u);
1137-
hash_function() = __u.hash_function();
1138-
key_eq() = __u.key_eq();
1139-
max_load_factor() = __u.max_load_factor();
1140-
__assign_multi(__u.begin(), __u.end());
1180+
__hash_table<_Tp, _Hash, _Equal, _Alloc>&
1181+
__hash_table<_Tp, _Hash, _Equal, _Alloc>::operator=(const __hash_table& __other) {
1182+
if (this == std::addressof(__other))
1183+
return *this;
1184+
1185+
__copy_assign_alloc(__other);
1186+
hash_function() = __other.hash_function();
1187+
key_eq() = __other.key_eq();
1188+
max_load_factor() = __other.max_load_factor();
1189+
1190+
if (__other.size() == 0) {
1191+
clear();
1192+
return *this;
1193+
}
1194+
1195+
auto __bucket_count = __other.bucket_count();
1196+
if (__bucket_count != bucket_count()) {
1197+
auto& __bucket_list_del = __bucket_list_.get_deleter();
1198+
__bucket_list_.reset(__pointer_alloc_traits::allocate(__bucket_list_del.__alloc(), __bucket_count));
1199+
__bucket_list_del.size() = __bucket_count;
1200+
}
1201+
std::fill_n(__bucket_list_.get(), __bucket_count, nullptr);
1202+
1203+
if (!__first_node_.__next_) {
1204+
__copy_construct(__other.__first_node_.__next_);
1205+
__size_ = __other.size();
1206+
return *this;
11411207
}
1208+
1209+
__next_pointer __other_iter = __other.__first_node_.__next_;
1210+
__next_pointer __own_iter = __first_node_.__ptr();
1211+
{
1212+
__node_pointer __next = __own_iter->__next_->__upcast();
1213+
__assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
1214+
__next->__hash_ = __other_iter->__hash();
1215+
}
1216+
size_t __current_chash = std::__constrain_hash(__own_iter->__next_->__hash(), __bucket_count);
1217+
__bucket_list_[__current_chash] = __own_iter;
1218+
__other_iter = __other_iter->__next_;
1219+
__own_iter = __own_iter->__next_;
1220+
1221+
// Go through the nodes of the incoming hash table and copy then into the destination hash table, reusing as many
1222+
// existing nodes as posssible in the destination.
1223+
while (__other_iter && __own_iter->__next_) {
1224+
__node_pointer __next = __own_iter->__next_->__upcast();
1225+
__assign_value(__next->__get_value(), __other_iter->__upcast()->__get_value());
1226+
__next->__hash_ = __other_iter->__hash();
1227+
1228+
size_t __new_chash = std::__constrain_hash(__next->__hash_, __bucket_count);
1229+
if (__new_chash != __current_chash) {
1230+
__bucket_list_[__new_chash] = __own_iter;
1231+
__current_chash = __new_chash;
1232+
}
1233+
1234+
__other_iter = __other_iter->__next_;
1235+
__own_iter = __own_iter->__next_;
1236+
}
1237+
1238+
// At this point we either have consumed the whole incoming hash table, or we don't have any more nodes to reuse in
1239+
// the destination. Either continue with constructing new nodes, or deallocate the left over nodes.
1240+
if (__own_iter->__next_) {
1241+
__deallocate_node(__own_iter->__next_);
1242+
__own_iter->__next_ = nullptr;
1243+
} else {
1244+
__copy_construct(__other_iter, __own_iter, __current_chash);
1245+
}
1246+
1247+
__size_ = __other.size();
1248+
11421249
return *this;
11431250
}
11441251

‎libcxx/include/unordered_map‎

Lines changed: 9 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1046,10 +1046,11 @@ public:
10461046
# endif
10471047

10481048
_LIBCPP_HIDE_FROM_ABI explicit unordered_map(const allocator_type& __a);
1049-
_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u);
1049+
_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u) = default;
10501050
_LIBCPP_HIDE_FROM_ABI unordered_map(const unordered_map& __u, const allocator_type& __a);
10511051
# ifndef _LIBCPP_CXX03_LANG
1052-
_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u) _NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
1052+
_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u)
1053+
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value) = default;
10531054
_LIBCPP_HIDE_FROM_ABI unordered_map(unordered_map&& __u, const allocator_type& __a);
10541055
_LIBCPP_HIDE_FROM_ABI unordered_map(initializer_list<value_type> __il);
10551056
_LIBCPP_HIDE_FROM_ABI
@@ -1099,24 +1100,10 @@ public:
10991100
static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
11001101
}
11011102

1102-
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) {
1103-
# ifndef _LIBCPP_CXX03_LANG
1104-
__table_ = __u.__table_;
1105-
# else
1106-
if (this != std::addressof(__u)) {
1107-
__table_.clear();
1108-
__table_.hash_function() = __u.__table_.hash_function();
1109-
__table_.key_eq() = __u.__table_.key_eq();
1110-
__table_.max_load_factor() = __u.__table_.max_load_factor();
1111-
__table_.__copy_assign_alloc(__u.__table_);
1112-
insert(__u.begin(), __u.end());
1113-
}
1114-
# endif
1115-
return *this;
1116-
}
1103+
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(const unordered_map& __u) = default;
11171104
# ifndef _LIBCPP_CXX03_LANG
11181105
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(unordered_map&& __u)
1119-
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
1106+
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) = default;
11201107
_LIBCPP_HIDE_FROM_ABI unordered_map& operator=(initializer_list<value_type> __il);
11211108
# endif // _LIBCPP_CXX03_LANG
11221109

@@ -1563,12 +1550,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
15631550
insert(__first, __last);
15641551
}
15651552

1566-
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1567-
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u) : __table_(__u.__table_) {
1568-
__table_.__rehash_unique(__u.bucket_count());
1569-
insert(__u.begin(), __u.end());
1570-
}
1571-
15721553
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
15731554
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_map& __u, const allocator_type& __a)
15741555
: __table_(__u.__table_, typename __table::allocator_type(__a)) {
@@ -1578,11 +1559,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(const unordered_ma
15781559

15791560
# ifndef _LIBCPP_CXX03_LANG
15801561

1581-
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1582-
inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u)
1583-
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
1584-
: __table_(std::move(__u.__table_)) {}
1585-
15861562
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
15871563
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(unordered_map&& __u, const allocator_type& __a)
15881564
: __table_(std::move(__u.__table_), typename __table::allocator_type(__a)) {
@@ -1618,14 +1594,6 @@ unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_map(
16181594
insert(__il.begin(), __il.end());
16191595
}
16201596

1621-
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
1622-
inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
1623-
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_map&& __u)
1624-
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
1625-
__table_ = std::move(__u.__table_);
1626-
return *this;
1627-
}
1628-
16291597
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
16301598
inline unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>&
16311599
unordered_map<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {
@@ -1852,11 +1820,11 @@ public:
18521820
# endif
18531821

18541822
_LIBCPP_HIDE_FROM_ABI explicit unordered_multimap(const allocator_type& __a);
1855-
_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u);
1823+
_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u) = default;
18561824
_LIBCPP_HIDE_FROM_ABI unordered_multimap(const unordered_multimap& __u, const allocator_type& __a);
18571825
# ifndef _LIBCPP_CXX03_LANG
18581826
_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u)
1859-
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value);
1827+
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value) = default;
18601828
_LIBCPP_HIDE_FROM_ABI unordered_multimap(unordered_multimap&& __u, const allocator_type& __a);
18611829
_LIBCPP_HIDE_FROM_ABI unordered_multimap(initializer_list<value_type> __il);
18621830
_LIBCPP_HIDE_FROM_ABI unordered_multimap(
@@ -1906,24 +1874,10 @@ public:
19061874
static_assert(sizeof(std::__diagnose_unordered_container_requirements<_Key, _Hash, _Pred>(0)), "");
19071875
}
19081876

1909-
_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) {
1910-
# ifndef _LIBCPP_CXX03_LANG
1911-
__table_ = __u.__table_;
1912-
# else
1913-
if (this != std::addressof(__u)) {
1914-
__table_.clear();
1915-
__table_.hash_function() = __u.__table_.hash_function();
1916-
__table_.key_eq() = __u.__table_.key_eq();
1917-
__table_.max_load_factor() = __u.__table_.max_load_factor();
1918-
__table_.__copy_assign_alloc(__u.__table_);
1919-
insert(__u.begin(), __u.end());
1920-
}
1921-
# endif
1922-
return *this;
1923-
}
1877+
_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(const unordered_multimap& __u) = default;
19241878
# ifndef _LIBCPP_CXX03_LANG
19251879
_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(unordered_multimap&& __u)
1926-
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value);
1880+
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) = default;
19271881
_LIBCPP_HIDE_FROM_ABI unordered_multimap& operator=(initializer_list<value_type> __il);
19281882
# endif // _LIBCPP_CXX03_LANG
19291883

@@ -2315,13 +2269,6 @@ template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
23152269
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const allocator_type& __a)
23162270
: __table_(typename __table::allocator_type(__a)) {}
23172271

2318-
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2319-
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(const unordered_multimap& __u)
2320-
: __table_(__u.__table_) {
2321-
__table_.__rehash_multi(__u.bucket_count());
2322-
insert(__u.begin(), __u.end());
2323-
}
2324-
23252272
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
23262273
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
23272274
const unordered_multimap& __u, const allocator_type& __a)
@@ -2332,11 +2279,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
23322279

23332280
# ifndef _LIBCPP_CXX03_LANG
23342281

2335-
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2336-
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(unordered_multimap&& __u)
2337-
_NOEXCEPT_(is_nothrow_move_constructible<__table>::value)
2338-
: __table_(std::move(__u.__table_)) {}
2339-
23402282
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
23412283
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
23422284
unordered_multimap&& __u, const allocator_type& __a)
@@ -2373,14 +2315,6 @@ unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::unordered_multimap(
23732315
insert(__il.begin(), __il.end());
23742316
}
23752317

2376-
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
2377-
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
2378-
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(unordered_multimap&& __u)
2379-
_NOEXCEPT_(is_nothrow_move_assignable<__table>::value) {
2380-
__table_ = std::move(__u.__table_);
2381-
return *this;
2382-
}
2383-
23842318
template <class _Key, class _Tp, class _Hash, class _Pred, class _Alloc>
23852319
inline unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>&
23862320
unordered_multimap<_Key, _Tp, _Hash, _Pred, _Alloc>::operator=(initializer_list<value_type> __il) {

0 commit comments

Comments
 (0)