Skip to content

Commit fae64c3

Browse files
philnik777philnik777
committed
[libc++] Optimize ranges::for_each for iterating over __trees
[libc++] Optimize std::for_each for __tree iterators
1 parent 07ad928 commit fae64c3

File tree

9 files changed

+323
-12
lines changed

9 files changed

+323
-12
lines changed

‎libcxx/include/__algorithm/for_each.h‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#define _LIBCPP___ALGORITHM_FOR_EACH_H
1212

1313
#include <__algorithm/for_each_segment.h>
14+
#include <__algorithm/specialized_algorithms.h>
1415
#include <__config>
1516
#include <__functional/identity.h>
1617
#include <__iterator/segmented_iterator.h>
@@ -27,7 +28,12 @@ template <class _InputIterator, class _Sent, class _Func, class _Proj>
2728
_LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _InputIterator
2829
__for_each(_InputIterator __first, _Sent __last, _Func& __func, _Proj& __proj) {
2930
#ifndef _LIBCPP_CXX03_LANG
30-
if constexpr (is_same<_InputIterator, _Sent>::value && __is_segmented_iterator_v<_InputIterator>) {
31+
if constexpr (using _SpecialAlg =
32+
__specialized_algorithm<_Algorithm::__for_each, __iterator_pair<_InputIterator, _Sent>>;
33+
_SpecialAlg::__has_algorithm) {
34+
_SpecialAlg()(__first, __last, __func, __proj);
35+
return __last;
36+
} else if constexpr (is_same<_InputIterator, _Sent>::value && __is_segmented_iterator_v<_InputIterator>) {
3137
using __local_iterator_t = typename __segmented_iterator_traits<_InputIterator>::__local_iterator;
3238
std::__for_each_segment(__first, __last, [&](__local_iterator_t __lfirst, __local_iterator_t __llast) {
3339
std::__for_each(__lfirst, __llast, __func, __proj);

‎libcxx/include/__algorithm/ranges_for_each.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include <__algorithm/for_each.h>
1313
#include <__algorithm/for_each_n.h>
1414
#include <__algorithm/in_fun_result.h>
15+
#include <__algorithm/specialized_algorithms.h>
1516
#include <__concepts/assignable.h>
1617
#include <__config>
1718
#include <__functional/identity.h>
@@ -20,6 +21,7 @@
2021
#include <__ranges/access.h>
2122
#include <__ranges/concepts.h>
2223
#include <__ranges/dangling.h>
24+
#include <__type_traits/remove_cvref.h>
2325
#include <__utility/move.h>
2426

2527
#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
@@ -71,7 +73,13 @@ struct __for_each {
7173
indirectly_unary_invocable<projected<iterator_t<_Range>, _Proj>> _Func>
7274
_LIBCPP_HIDE_FROM_ABI constexpr for_each_result<borrowed_iterator_t<_Range>, _Func>
7375
operator()(_Range&& __range, _Func __func, _Proj __proj = {}) const {
74-
return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
76+
using _SpecialAlg = __specialized_algorithm<_Algorithm::__for_each, remove_cvref_t<_Range>>;
77+
if constexpr (_SpecialAlg::__has_algorithm) {
78+
auto [__iter, __func2] = _SpecialAlg()(__range, std::move(__func), std::move(__proj));
79+
return {std::move(__iter), std::move(__func)};
80+
} else {
81+
return __for_each_impl(ranges::begin(__range), ranges::end(__range), __func, __proj);
82+
}
7583
}
7684
};
7785

‎libcxx/include/__algorithm/specialized_algorithms.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,15 @@ _LIBCPP_BEGIN_NAMESPACE_STD
1919

2020
namespace _Algorithm {
2121
struct __fill_n {};
22+
struct __for_each {};
2223
} // namespace _Algorithm
2324

2425
template <class>
2526
struct __single_iterator;
2627

28+
template <class, class>
29+
struct __iterator_pair;
30+
2731
// This struct allows specializing algorithms for specific arguments. This is useful when we know a more efficient
2832
// algorithm implementation for e.g. library-defined iterators. _Alg is one of tags defined inside the _Algorithm
2933
// namespace above. _Ranges is an essentially arbitrary subset of the arguments to the algorithm that are used for

‎libcxx/include/__tree‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
#define _LIBCPP___TREE
1212

1313
#include <__algorithm/min.h>
14+
#include <__algorithm/specialized_algorithms.h>
1415
#include <__assert>
1516
#include <__config>
1617
#include <__fwd/pair.h>
@@ -717,6 +718,63 @@ private:
717718
friend class __tree_const_iterator;
718719
};
719720

721+
template <class _Reference, class _Break, class _NodePtr, class _Func, class _Proj>
722+
_LIBCPP_HIDE_FROM_ABI bool __tree_iterate_from_root(_Break __break, _NodePtr __root, _Func& __func, _Proj& __proj) {
723+
if (__root->__left_) {
724+
if (std::__tree_iterate_from_root<_Reference>(__break, static_cast<_NodePtr>(__root->__left_), __func, __proj))
725+
return true;
726+
}
727+
if (__break(__root))
728+
return true;
729+
__func(static_cast<_Reference>(__root->__get_value()));
730+
if (__root->__right_)
731+
return std::__tree_iterate_from_root<_Reference>(__break, static_cast<_NodePtr>(__root->__right_), __func, __proj);
732+
return false;
733+
}
734+
735+
template <class _Reference, class _NodePtr, class _EndNodePtr, class _Func, class _Proj>
736+
_LIBCPP_HIDE_FROM_ABI void
737+
__tree_iterate_from_begin(_EndNodePtr __first, _EndNodePtr __last, _Func& __func, _Proj& __proj) {
738+
while (true) {
739+
if (__first == __last)
740+
return;
741+
auto __nfirst = static_cast<_NodePtr>(__first);
742+
__func(static_cast<_Reference>(__nfirst->__get_value()));
743+
if (__nfirst->__right_) {
744+
if (std::__tree_iterate_from_root<_Reference>(
745+
[&](_NodePtr __node) -> bool { return __node == __last; },
746+
static_cast<_NodePtr>(__nfirst->__right_),
747+
__func,
748+
__proj))
749+
return;
750+
}
751+
if (std::__tree_is_left_child(__nfirst)) {
752+
__first = __nfirst->__parent_;
753+
} else {
754+
do {
755+
__first = __nfirst->__parent_;
756+
} while (!std::__tree_is_left_child(__nfirst));
757+
}
758+
}
759+
}
760+
761+
#ifndef _LIBCPP_CXX03_LANG
762+
template <class _Tp, class _NodePtr, class _DiffType>
763+
struct __specialized_algorithm<
764+
_Algorithm::__for_each,
765+
__iterator_pair<__tree_iterator<_Tp, _NodePtr, _DiffType>, __tree_iterator<_Tp, _NodePtr, _DiffType>>> {
766+
static const bool __has_algorithm = true;
767+
768+
using __iterator _LIBCPP_NODEBUG = __tree_iterator<_Tp, _NodePtr, _DiffType>;
769+
770+
template <class _Func, class _Proj>
771+
_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Func& __func, _Proj& __proj) {
772+
std::__tree_iterate_from_begin<typename __iterator::reference, _NodePtr>(
773+
__first.__ptr_, __last.__ptr_, __func, __proj);
774+
}
775+
};
776+
#endif
777+
720778
template <class _Tp, class _NodePtr, class _DiffType>
721779
class __tree_const_iterator {
722780
using _NodeTypes _LIBCPP_NODEBUG = __tree_node_types<_NodePtr>;
@@ -780,7 +838,27 @@ private:
780838

781839
template <class, class, class>
782840
friend class __tree;
841+
842+
friend struct __specialized_algorithm<_Algorithm::__for_each,
843+
__iterator_pair<__tree_const_iterator, __tree_const_iterator> >;
844+
};
845+
846+
#ifndef _LIBCPP_CXX03_LANG
847+
template <class _Tp, class _NodePtr, class _DiffType>
848+
struct __specialized_algorithm<
849+
_Algorithm::__for_each,
850+
__iterator_pair<__tree_const_iterator<_Tp, _NodePtr, _DiffType>, __tree_const_iterator<_Tp, _NodePtr, _DiffType>>> {
851+
static const bool __has_algorithm = true;
852+
853+
using __iterator = __tree_const_iterator<_Tp, _NodePtr, _DiffType>;
854+
855+
template <class _Func, class _Proj>
856+
_LIBCPP_HIDE_FROM_ABI static void operator()(__iterator __first, __iterator __last, _Func& __func, _Proj& __proj) {
857+
std::__tree_iterate_from_begin<typename __iterator::reference, _NodePtr>(
858+
__first.__ptr_, __last.__ptr_, __func, __proj);
859+
}
783860
};
861+
#endif
784862

785863
template <class _Tp, class _Compare>
786864
#ifndef _LIBCPP_CXX03_LANG
@@ -1484,7 +1562,24 @@ private:
14841562
[](value_type& __lhs, value_type& __rhs) { __assign_value(__lhs, std::move(__rhs)); },
14851563
[this](__node_pointer __nd) { return __move_construct_tree(__nd); });
14861564
}
1565+
1566+
friend struct __specialized_algorithm<_Algorithm::__for_each, __tree>;
1567+
};
1568+
1569+
#if _LIBCPP_STD_VER >= 14
1570+
template <class _Tp, class _Compare, class _Allocator>
1571+
struct __specialized_algorithm<_Algorithm::__for_each, __tree<_Tp, _Compare, _Allocator> > {
1572+
static const bool __has_algorithm = true;
1573+
1574+
using __node_pointer _LIBCPP_NODEBUG = typename __tree<_Tp, _Compare, _Allocator>::__node_pointer;
1575+
1576+
template <class _Tree, class _Func, class _Proj>
1577+
_LIBCPP_HIDE_FROM_ABI static auto operator()(_Tree&& __range, _Func __func, _Proj __proj) {
1578+
std::__tree_iterate_from_root([](__node_pointer) { return false; }, __range.__root(), __func, __proj);
1579+
return std::make_pair(__range.end(), std::move(__func));
1580+
}
14871581
};
1582+
#endif
14881583

14891584
// Precondition: __size_ != 0
14901585
template <class _Tp, class _Compare, class _Allocator>

‎libcxx/include/map‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -577,6 +577,7 @@ erase_if(multimap<Key, T, Compare, Allocator>& c, Predicate pred); // C++20
577577
# include <__algorithm/equal.h>
578578
# include <__algorithm/lexicographical_compare.h>
579579
# include <__algorithm/lexicographical_compare_three_way.h>
580+
# include <__algorithm/specialized_algorithms.h>
580581
# include <__assert>
581582
# include <__config>
582583
# include <__functional/binary_function.h>
@@ -1370,6 +1371,8 @@ private:
13701371
# ifdef _LIBCPP_CXX03_LANG
13711372
_LIBCPP_HIDE_FROM_ABI __node_holder __construct_node_with_key(const key_type& __k);
13721373
# endif
1374+
1375+
friend struct __specialized_algorithm<_Algorithm::__for_each, map>;
13731376
};
13741377

13751378
# if _LIBCPP_STD_VER >= 17
@@ -1422,6 +1425,22 @@ map(initializer_list<pair<_Key, _Tp>>, _Allocator)
14221425
-> map<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
14231426
# endif
14241427

1428+
# if _LIBCPP_STD_VER >= 14
1429+
template <class _Key, class _Tp, class _Compare, class _Allocator>
1430+
struct __specialized_algorithm<_Algorithm::__for_each, map<_Key, _Tp, _Compare, _Allocator>> {
1431+
using __map _LIBCPP_NODEBUG = map<_Key, _Tp, _Compare, _Allocator>;
1432+
1433+
static const bool __has_algorithm = true;
1434+
1435+
template <class _Map, class _Func>
1436+
_LIBCPP_HIDE_FROM_ABI static auto operator()(_Map&& __map, _Func __func) {
1437+
auto [_, __func2] = __specialized_algorithm<_Algorithm::__for_each, typename __map::__base>()(
1438+
__map.__tree_, std::move(__func));
1439+
return std::make_pair(__map.end(), std::move(__func2));
1440+
}
1441+
};
1442+
# endif
1443+
14251444
# ifndef _LIBCPP_CXX03_LANG
14261445
template <class _Key, class _Tp, class _Compare, class _Allocator>
14271446
_Tp& map<_Key, _Tp, _Compare, _Allocator>::operator[](const key_type& __k) {
@@ -1920,6 +1939,8 @@ private:
19201939

19211940
typedef __map_node_destructor<__node_allocator> _Dp;
19221941
typedef unique_ptr<__node, _Dp> __node_holder;
1942+
1943+
friend struct __specialized_algorithm<_Algorithm::__for_each, multimap>;
19231944
};
19241945

19251946
# if _LIBCPP_STD_VER >= 17
@@ -1972,6 +1993,22 @@ multimap(initializer_list<pair<_Key, _Tp>>, _Allocator)
19721993
-> multimap<remove_const_t<_Key>, _Tp, less<remove_const_t<_Key>>, _Allocator>;
19731994
# endif
19741995

1996+
# if _LIBCPP_STD_VER >= 14
1997+
template <class _Key, class _Tp, class _Compare, class _Allocator>
1998+
struct __specialized_algorithm<_Algorithm::__for_each, multimap<_Key, _Tp, _Compare, _Allocator>> {
1999+
using __map _LIBCPP_NODEBUG = multimap<_Key, _Tp, _Compare, _Allocator>;
2000+
2001+
static const bool __has_algorithm = true;
2002+
2003+
template <class _Map, class _Func>
2004+
_LIBCPP_HIDE_FROM_ABI static auto operator()(_Map&& __map, _Func __func) {
2005+
auto [_, __func2] = __specialized_algorithm<_Algorithm::__for_each, typename __map::__base>()(
2006+
__map.__tree_, std::move(__func));
2007+
return std::make_pair(__map.end(), std::move(__func2));
2008+
}
2009+
};
2010+
# endif
2011+
19752012
template <class _Key, class _Tp, class _Compare, class _Allocator>
19762013
inline _LIBCPP_HIDE_FROM_ABI bool
19772014
operator==(const multimap<_Key, _Tp, _Compare, _Allocator>& __x, const multimap<_Key, _Tp, _Compare, _Allocator>& __y) {

‎libcxx/include/set‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -518,6 +518,7 @@ erase_if(multiset<Key, Compare, Allocator>& c, Predicate pred); // C++20
518518
# include <__algorithm/equal.h>
519519
# include <__algorithm/lexicographical_compare.h>
520520
# include <__algorithm/lexicographical_compare_three_way.h>
521+
# include <__algorithm/specialized_algorithms.h>
521522
# include <__assert>
522523
# include <__config>
523524
# include <__functional/is_transparent.h>
@@ -898,6 +899,9 @@ public:
898899
return __tree_.__equal_range_multi(__k);
899900
}
900901
# endif
902+
903+
template <class, class...>
904+
friend struct __specialized_algorithm;
901905
};
902906

903907
# if _LIBCPP_STD_VER >= 17
@@ -944,6 +948,21 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
944948
set(initializer_list<_Key>, _Allocator) -> set<_Key, less<_Key>, _Allocator>;
945949
# endif
946950

951+
# if _LIBCPP_STD_VER >= 14
952+
template <class _Alg, class _Key, class _Compare, class _Allocator>
953+
struct __specialized_algorithm<_Alg, set<_Key, _Compare, _Allocator>> {
954+
using __set _LIBCPP_NODEBUG = set<_Key, _Compare, _Allocator>;
955+
956+
static const bool __has_algorithm = __specialized_algorithm<_Alg, typename __set::__base>::__has_algorithm;
957+
958+
// set's begin() and end() are identical with and without const qualification
959+
template <class... _Args>
960+
_LIBCPP_HIDE_FROM_ABI static auto operator()(const __set& __set, _Args&&... __args) {
961+
return __specialized_algorithm<_Alg, typename __set::__base>()(__set.__tree_, std::forward<_Args>(__args)...);
962+
}
963+
};
964+
# endif
965+
947966
template <class _Key, class _Compare, class _Allocator>
948967
inline _LIBCPP_HIDE_FROM_ABI bool
949968
operator==(const set<_Key, _Compare, _Allocator>& __x, const set<_Key, _Compare, _Allocator>& __y) {
@@ -1342,6 +1361,9 @@ public:
13421361
return __tree_.__equal_range_multi(__k);
13431362
}
13441363
# endif
1364+
1365+
template <class, class...>
1366+
friend struct __specialized_algorithm;
13451367
};
13461368

13471369
# if _LIBCPP_STD_VER >= 17
@@ -1389,6 +1411,21 @@ template <class _Key, class _Allocator, class = enable_if_t<__is_allocator_v<_Al
13891411
multiset(initializer_list<_Key>, _Allocator) -> multiset<_Key, less<_Key>, _Allocator>;
13901412
# endif
13911413

1414+
# if _LIBCPP_STD_VER >= 14
1415+
template <class _Alg, class _Key, class _Compare, class _Allocator>
1416+
struct __specialized_algorithm<_Alg, multiset<_Key, _Compare, _Allocator>> {
1417+
using __set _LIBCPP_NODEBUG = multiset<_Key, _Compare, _Allocator>;
1418+
1419+
static const bool __has_algorithm = __specialized_algorithm<_Alg, typename __set::__base>::__has_algorithm;
1420+
1421+
// set's begin() and end() are identical with and without const qualification
1422+
template <class... _Args>
1423+
_LIBCPP_HIDE_FROM_ABI static auto operator()(const __set& __set, _Args&&... __args) {
1424+
return __specialized_algorithm<_Alg, typename __set::__base>()(__set.__tree_, std::forward<_Args>(__args)...);
1425+
}
1426+
};
1427+
# endif
1428+
13921429
template <class _Key, class _Compare, class _Allocator>
13931430
inline _LIBCPP_HIDE_FROM_ABI bool
13941431
operator==(const multiset<_Key, _Compare, _Allocator>& __x, const multiset<_Key, _Compare, _Allocator>& __y) {

0 commit comments

Comments
 (0)