Skip to content

[libc++] Optimize __tree copy/move constructor/assignment with allocator #163558

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

[libc++] Optimize __tree copy/move constructor/assignment with allocator #163558

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
155 changes: 95 additions & 60 deletions libcxx/include/__tree
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -899,6 +899,18 @@ public:
}

_LIBCPP_HIDE_FROM_ABI __tree(const __tree& __t);

_LIBCPP_HIDE_FROM_ABI __tree(const __tree& __other, const allocator_type& __alloc)
: __begin_node_(__end_node()), __node_alloc_(__alloc), __size_(0), __value_comp_(__other.value_comp()) {
if (__other.size() == 0)
return;

*__root_ptr() = static_cast<__node_base_pointer>(__copy_construct_tree(__other.__root()));
__root()->__parent_ = __end_node();
__begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
__size_ = __other.size();
}

_LIBCPP_HIDE_FROM_ABI __tree& operator=(const __tree& __t);
template <class _ForwardIterator>
_LIBCPP_HIDE_FROM_ABI void __assign_unique(_ForwardIterator __first, _ForwardIterator __last);
Expand Down Expand Up @@ -1007,27 +1019,6 @@ public:
std::forward<_Args>(__args)...);
}

template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
_LIBCPP_HIDE_FROM_ABI void
__insert_unique_from_orphaned_node(const_iterator __p, __get_node_value_type_t<_Tp>&& __value) {
__emplace_hint_unique(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
}

template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
_LIBCPP_HIDE_FROM_ABI void __insert_unique_from_orphaned_node(const_iterator __p, _Tp&& __value) {
__emplace_hint_unique(__p, std::move(__value));
}

template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
_LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, value_type&& __value) {
__emplace_hint_multi(__p, const_cast<key_type&&>(__value.first), std::move(__value.second));
}

template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
_LIBCPP_HIDE_FROM_ABI void __insert_multi_from_orphaned_node(const_iterator __p, _Tp&& __value) {
__emplace_hint_multi(__p, std::move(__value));
}

template <class _InIter, class _Sent>
_LIBCPP_HIDE_FROM_ABI void __insert_range_multi(_InIter __first, _Sent __last) {
if (__first == __last)
Expand Down Expand Up @@ -1400,19 +1391,20 @@ private:
// copy the exact structure 1:1. Since this is for copy construction _only_ we know that we get a correct tree. If we
// didn't get a correct tree, the invariants of __tree are broken and we have a much bigger problem than an improperly
// balanced tree.
template <class _NodeConstructor>
#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
_LIBCPP_HIDE_FROM_ABI
#endif
__node_pointer
__copy_construct_tree(__node_pointer __src) {
__construct_from_tree(__node_pointer __src, _NodeConstructor __construct) {
if (!__src)
return nullptr;

__node_holder __new_node = __construct_node(__src->__get_value());
__node_holder __new_node = __construct(__src->__get_value());

unique_ptr<__node, __tree_deleter> __left(
__copy_construct_tree(static_cast<__node_pointer>(__src->__left_)), __node_alloc_);
__node_pointer __right = __copy_construct_tree(static_cast<__node_pointer>(__src->__right_));
__construct_from_tree(static_cast<__node_pointer>(__src->__left_), __construct), __node_alloc_);
__node_pointer __right = __construct_from_tree(static_cast<__node_pointer>(__src->__right_), __construct);

__node_pointer __new_node_ptr = __new_node.release();

Expand All @@ -1426,46 +1418,85 @@ private:
return __new_node_ptr;
}

_LIBCPP_HIDE_FROM_ABI __node_pointer __copy_construct_tree(__node_pointer __src) {
return __construct_from_tree(__src, [this](const value_type& __val) { return __construct_node(__val); });
}

template <class _ValueT = _Tp, __enable_if_t<__is_tree_value_type_v<_ValueT>, int> = 0>
_LIBCPP_HIDE_FROM_ABI __node_pointer __move_construct_tree(__node_pointer __src) {
return __construct_from_tree(__src, [this](value_type& __val) {
return __construct_node(const_cast<key_type&&>(__val.first), std::move(__val.second));
});
}

template <class _ValueT = _Tp, __enable_if_t<!__is_tree_value_type_v<_ValueT>, int> = 0>
_LIBCPP_HIDE_FROM_ABI __node_pointer __move_construct_tree(__node_pointer __src) {
return __construct_from_tree(__src, [this](value_type& __val) { return __construct_node(std::move(__val)); });
}

template <class _Assignment, class _ConstructionAlg>
// This copy assignment will always produce a correct red-black-tree assuming the incoming tree is correct, since our
// own tree is a red-black-tree and the incoming tree is a red-black-tree. The invariants of a red-black-tree are
// temporarily not met until all of the incoming red-black tree is copied.
#ifdef _LIBCPP_COMPILER_CLANG_BASED // FIXME: GCC complains about not being able to always_inline a recursive function
_LIBCPP_HIDE_FROM_ABI
#endif
__node_pointer
__copy_assign_tree(__node_pointer __dest, __node_pointer __src) {
__node_pointer __assign_from_tree(
__node_pointer __dest, __node_pointer __src, _Assignment __assign, _ConstructionAlg __continue_with_construct) {
if (!__src) {
destroy(__dest);
return nullptr;
}

__assign_value(__dest->__get_value(), __src->__get_value());
__assign(__dest->__get_value(), __src->__get_value());
__dest->__is_black_ = __src->__is_black_;

// If we already have a left node in the destination tree, reuse it and copy-assign recursively
if (__dest->__left_) {
__dest->__left_ = static_cast<__node_base_pointer>(__copy_assign_tree(
static_cast<__node_pointer>(__dest->__left_), static_cast<__node_pointer>(__src->__left_)));
__dest->__left_ = static_cast<__node_base_pointer>(__assign_from_tree(
static_cast<__node_pointer>(__dest->__left_),
static_cast<__node_pointer>(__src->__left_),
__assign,
__continue_with_construct));

// Otherwise, we must create new nodes; copy-construct from here on
} else if (__src->__left_) {
auto __new_left = __copy_construct_tree(static_cast<__node_pointer>(__src->__left_));
auto __new_left = __continue_with_construct(static_cast<__node_pointer>(__src->__left_));
__dest->__left_ = static_cast<__node_base_pointer>(__new_left);
__new_left->__parent_ = static_cast<__end_node_pointer>(__dest);
}

// Identical to the left case above, just for the right nodes
if (__dest->__right_) {
__dest->__right_ = static_cast<__node_base_pointer>(__copy_assign_tree(
static_cast<__node_pointer>(__dest->__right_), static_cast<__node_pointer>(__src->__right_)));
__dest->__right_ = static_cast<__node_base_pointer>(__assign_from_tree(
static_cast<__node_pointer>(__dest->__right_),
static_cast<__node_pointer>(__src->__right_),
__assign,
__continue_with_construct));
} else if (__src->__right_) {
auto __new_right = __copy_construct_tree(static_cast<__node_pointer>(__src->__right_));
auto __new_right = __continue_with_construct(static_cast<__node_pointer>(__src->__right_));
__dest->__right_ = static_cast<__node_base_pointer>(__new_right);
__new_right->__parent_ = static_cast<__end_node_pointer>(__dest);
}

return __dest;
}

_LIBCPP_HIDE_FROM_ABI __node_pointer __copy_assign_tree(__node_pointer __dest, __node_pointer __src) {
return __assign_from_tree(
__dest,
__src,
[](value_type& __lhs, const value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
[this](__node_pointer __nd) { return __copy_construct_tree(__nd); });
}

_LIBCPP_HIDE_FROM_ABI __node_pointer __move_assign_tree(__node_pointer __dest, __node_pointer __src) {
return __assign_from_tree(
__dest,
__src,
[](value_type& __lhs, value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
[this](__node_pointer __nd) { return __move_construct_tree(__nd); });
}
};

// Precondition: __size_ != 0
Expand Down Expand Up @@ -1606,21 +1637,26 @@ __tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t) _NOEXCEPT_(

template <class _Tp, class _Compare, class _Allocator>
__tree<_Tp, _Compare, _Allocator>::__tree(__tree&& __t, const allocator_type& __a)
: __node_alloc_(__node_allocator(__a)), __size_(0), __value_comp_(std::move(__t.value_comp())) {
: __begin_node_(__end_node()),
__node_alloc_(__node_allocator(__a)),
__size_(0),
__value_comp_(std::move(__t.value_comp())) {
if (__t.size() == 0)
return;
if (__a == __t.__alloc()) {
if (__t.__size_ == 0)
__begin_node_ = __end_node();
else {
__begin_node_ = __t.__begin_node_;
__end_node()->__left_ = __t.__end_node()->__left_;
__end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
__size_ = __t.__size_;
__t.__begin_node_ = __t.__end_node();
__t.__end_node()->__left_ = nullptr;
__t.__size_ = 0;
}
__begin_node_ = __t.__begin_node_;
__end_node()->__left_ = __t.__end_node()->__left_;
__end_node()->__left_->__parent_ = static_cast<__end_node_pointer>(__end_node());
__size_ = __t.__size_;
__t.__begin_node_ = __t.__end_node();
__t.__end_node()->__left_ = nullptr;
__t.__size_ = 0;
} else {
__begin_node_ = __end_node();
*__root_ptr() = static_cast<__node_base_pointer>(__move_construct_tree(__t.__root()));
__root()->__parent_ = __end_node();
__begin_node_ = static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_));
__size_ = __t.size();
__t.clear(); // Ensure that __t is in a valid state after moving out the keys
}
}

Expand All @@ -1645,22 +1681,21 @@ void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, true_type)

template <class _Tp, class _Compare, class _Allocator>
void __tree<_Tp, _Compare, _Allocator>::__move_assign(__tree& __t, false_type) {
if (__node_alloc() == __t.__node_alloc())
if (__node_alloc() == __t.__node_alloc()) {
__move_assign(__t, true_type());
else {
value_comp() = std::move(__t.value_comp());
const_iterator __e = end();
} else {
value_comp() = std::move(__t.value_comp());
if (__size_ != 0) {
_DetachedTreeCache __cache(this);
while (__cache.__get() != nullptr && __t.__size_ != 0) {
__assign_value(__cache.__get()->__get_value(), std::move(__t.remove(__t.begin())->__get_value()));
__node_insert_multi(__cache.__get());
__cache.__advance();
}
}
while (__t.__size_ != 0) {
__insert_multi_from_orphaned_node(__e, std::move(__t.remove(__t.begin())->__get_value()));
*__root_ptr() = static_cast<__node_base_pointer>(__move_assign_tree(__root(), __t.__root()));
} else {
*__root_ptr() = static_cast<__node_base_pointer>(__move_construct_tree(__t.__root()));
if (__root())
__root()->__parent_ = __end_node();
}
__begin_node_ =
__end_node()->__left_ ? static_cast<__end_node_pointer>(std::__tree_min(__end_node()->__left_)) : __end_node();
__size_ = __t.size();
__t.clear(); // Ensure that __t is in a valid state after moving out the keys
}
}

Expand Down
39 changes: 4 additions & 35 deletions libcxx/include/map
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -997,7 +997,7 @@ public:

_LIBCPP_HIDE_FROM_ABI map(map&& __m) = default;

_LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI map(map&& __m, const allocator_type& __a) : __tree_(std::move(__m.__tree_), __a) {}

_LIBCPP_HIDE_FROM_ABI map& operator=(map&& __m) = default;

Expand Down Expand Up @@ -1025,10 +1025,7 @@ public:

_LIBCPP_HIDE_FROM_ABI explicit map(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}

_LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __a)
: __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
insert(__m.begin(), __m.end());
}
_LIBCPP_HIDE_FROM_ABI map(const map& __m, const allocator_type& __alloc) : __tree_(__m.__tree_, __alloc) {}

_LIBCPP_HIDE_FROM_ABI ~map() { static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), ""); }

Expand Down Expand Up @@ -1428,18 +1425,6 @@ map(initializer_list<pair<_Key, _Tp>>, _Allocator)
# endif

# ifndef _LIBCPP_CXX03_LANG
template <class _Key, class _Tp, class _Compare, class _Allocator>
map<_Key, _Tp, _Compare, _Allocator>::map(map&& __m, const allocator_type& __a)
: __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
if (__a != __m.get_allocator()) {
const_iterator __e = cend();
while (!__m.empty()) {
__tree_.__insert_unique_from_orphaned_node(
__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__get_value()));
}
}
}

template <class _Key, class _Tp, class _Compare, class _Allocator>
_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
return __tree_.__emplace_unique(std::piecewise_construct, std::forward_as_tuple(__k), std::forward_as_tuple())
Expand Down Expand Up @@ -1685,7 +1670,7 @@ public:

_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m) = default;

_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI multimap(multimap&& __m, const allocator_type& __a) : __tree_(std::move(__m.__tree_), __a) {}

_LIBCPP_HIDE_FROM_ABI multimap& operator=(multimap&& __m) = default;

Expand Down Expand Up @@ -1714,10 +1699,7 @@ public:

_LIBCPP_HIDE_FROM_ABI explicit multimap(const allocator_type& __a) : __tree_(typename __base::allocator_type(__a)) {}

_LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a)
: __tree_(__m.__tree_.value_comp(), typename __base::allocator_type(__a)) {
insert(__m.begin(), __m.end());
}
_LIBCPP_HIDE_FROM_ABI multimap(const multimap& __m, const allocator_type& __a) : __tree_(__m.__tree_, __a) {}

_LIBCPP_HIDE_FROM_ABI ~multimap() {
static_assert(sizeof(std::__diagnose_non_const_comparator<_Key, _Compare>()), "");
Expand Down Expand Up @@ -1992,19 +1974,6 @@ multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
-> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
# endif

# ifndef _LIBCPP_CXX03_LANG
template <class _Key, class _Tp, class _Compare, class _Allocator>
multimap<_Key, _Tp, _Compare, _Allocator>::multimap(multimap&& __m, const allocator_type& __a)
: __tree_(std::move(__m.__tree_), typename __base::allocator_type(__a)) {
if (__a != __m.get_allocator()) {
const_iterator __e = cend();
while (!__m.empty())
__tree_.__insert_multi_from_orphaned_node(
__e.__i_, std::move(__m.__tree_.remove(__m.begin().__i_)->__get_value()));
}
}
# endif

template <class _Key, class _Tp, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {
Expand Down
40 changes: 4 additions & 36 deletions libcxx/include/set
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -673,12 +673,10 @@ public:

_LIBCPP_HIDE_FROM_ABI explicit set(const allocator_type& __a) : __tree_(__a) {}

_LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __a) : __tree_(__s.__tree_.value_comp(), __a) {
insert(__s.begin(), __s.end());
}
_LIBCPP_HIDE_FROM_ABI set(const set& __s, const allocator_type& __alloc) : __tree_(__s.__tree_, __alloc) {}

# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI set(set&& __s, const allocator_type& __alloc) : __tree_(std::move(__s.__tree_), __alloc) {}

_LIBCPP_HIDE_FROM_ABI set(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
: __tree_(__comp) {
Expand Down Expand Up @@ -948,19 +946,6 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
# endif

# ifndef _LIBCPP_CXX03_LANG

template <class _Key, class _Compare, class _Allocator>
set<_Key, _Compare, _Allocator>::set(set&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {
if (__a != __s.get_allocator()) {
const_iterator __e = cend();
while (!__s.empty())
insert(__e, std::move(__s.__tree_.remove(__s.begin())->__get_value()));
}
}

# endif // _LIBCPP_CXX03_LANG

template <class _Key, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
Expand Down Expand Up @@ -1130,13 +1115,10 @@ public:
# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s) = default;

_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a);
_LIBCPP_HIDE_FROM_ABI multiset(multiset&& __s, const allocator_type& __a) : __tree_(std::move(__s.__tree_), __a) {}
# endif // _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI explicit multiset(const allocator_type& __a) : __tree_(__a) {}
_LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a)
: __tree_(__s.__tree_.value_comp(), __a) {
insert(__s.begin(), __s.end());
}
_LIBCPP_HIDE_FROM_ABI multiset(const multiset& __s, const allocator_type& __a) : __tree_(__s.__tree_, __a) {}

# ifndef _LIBCPP_CXX03_LANG
_LIBCPP_HIDE_FROM_ABI multiset(initializer_list<value_type> __il, const value_compare& __comp = value_compare())
Expand Down Expand Up @@ -1409,20 +1391,6 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
# endif

# ifndef _LIBCPP_CXX03_LANG

template <class _Key, class _Compare, class _Allocator>
multiset<_Key, _Compare, _Allocator>::multiset(multiset&& __s, const allocator_type& __a)
: __tree_(std::move(__s.__tree_), __a) {
if (__a != __s.get_allocator()) {
const_iterator __e = cend();
while (!__s.empty())
insert(__e, std::move(__s.__tree_.remove(__s.begin())->__get_value()));
}
}

# endif // _LIBCPP_CXX03_LANG

template <class _Key, class _Compare, class _Allocator>
inline _LIBCPP_HIDE_FROM_ABI bool
operator==(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {
Expand Down
Loading
Loading