<algorithm>: Rework internal buffers for stable sorting algorithms (#5807)

Co-authored-by: Stephan T. Lavavej <[email protected]>

Author: frederick-vs-ja

stl/inc/algorithm

@@ -449,17 +449,17 @@ constexpr ptrdiff_t _Temporary_buffer_size(const _Diff _Value) noexcept {
 }
 
 template <class _Ty>
-struct _Optimistic_temporary_buffer { // temporary storage with _alloca-like attempt
+struct _Optimistic_temporary_buffer2 { // temporary storage with _alloca-like attempt
     static constexpr size_t _Optimistic_size  = 4096; // default to ~1 page
     static constexpr size_t _Optimistic_count = (_STD max) (static_cast<size_t>(1), _Optimistic_size / sizeof(_Ty));
 
     template <class _Diff>
-    explicit _Optimistic_temporary_buffer(const _Diff _Requested_size) noexcept { // get temporary storage
+    explicit _Optimistic_temporary_buffer2(const _Diff _Requested_size) noexcept { // get temporary storage
         const auto _Attempt = _Temporary_buffer_size(_Requested_size);
         // Since _Diff is a count of elements in a forward range, and forward iterators must denote objects in memory,
         // it must fit in a size_t.
         if (static_cast<size_t>(_Requested_size) <= _Optimistic_count) { // unconditionally engage stack space
-            _Data     = reinterpret_cast<_Ty*>(&_Stack_space[0]);
+            _Data     = reinterpret_cast<_Ty*>(_Stack_space);
             _Capacity = static_cast<ptrdiff_t>(_Requested_size); // in bounds due to if condition
             return;
         }
@@ -473,22 +473,22 @@ struct _Optimistic_temporary_buffer { // temporary storage with _alloca-like att
 
         // less heap space than stack space, give up and use stack instead
         _STD _Return_temporary_buffer(_Raw.first);
-        _Data     = reinterpret_cast<_Ty*>(&_Stack_space[0]);
+        _Data     = reinterpret_cast<_Ty*>(_Stack_space);
         _Capacity = _Optimistic_count;
     }
 
-    _Optimistic_temporary_buffer(const _Optimistic_temporary_buffer&)            = delete;
-    _Optimistic_temporary_buffer& operator=(const _Optimistic_temporary_buffer&) = delete;
+    _Optimistic_temporary_buffer2(const _Optimistic_temporary_buffer2&)            = delete;
+    _Optimistic_temporary_buffer2& operator=(const _Optimistic_temporary_buffer2&) = delete;
 
-    ~_Optimistic_temporary_buffer() noexcept {
+    ~_Optimistic_temporary_buffer2() noexcept {
         if (static_cast<size_t>(_Capacity) > _Optimistic_count) {
             _STD _Return_temporary_buffer(_Data);
         }
     }
 
     _Ty* _Data; // points to heap memory iff _Capacity > _Optimistic_count
     ptrdiff_t _Capacity;
-    _Aligned_storage_t<sizeof(_Ty), alignof(_Ty)> _Stack_space[_Optimistic_count];
+    alignas(_Ty) unsigned char _Stack_space[sizeof(_Ty) * _Optimistic_count];
 };
 
 #if _HAS_CXX20
@@ -7110,7 +7110,7 @@ _BidIt _Stable_partition_unchecked(_BidIt _First, _BidIt _Last, _Pr _Pred) {
     using _Diff              = _Iter_diff_t<_BidIt>;
     const _Diff _Temp_count  = _STD distance(_First, _Last); // _Total_count - 1 since we never need to store *_Last
     const _Diff _Total_count = _Temp_count + static_cast<_Diff>(1);
-    _Optimistic_temporary_buffer<_Iter_value_t<_BidIt>> _Temp_buf{_Temp_count};
+    _Optimistic_temporary_buffer2<_Iter_value_t<_BidIt>> _Temp_buf{_Temp_count};
     return _STD _Stable_partition_unchecked1(_First, _Last, _Pred, _Total_count, _Temp_buf._Data, _Temp_buf._Capacity)
         .first;
 }
@@ -7229,7 +7229,7 @@ namespace ranges {
             } while (!_STD invoke(_Pred, _STD invoke(_Proj, *_Last)));
 
             const iter_difference_t<_It> _Temp_count = _RANGES distance(_First, _Last);
-            _Optimistic_temporary_buffer<iter_value_t<_It>> _Temp_buf{_Temp_count};
+            _Optimistic_temporary_buffer2<iter_value_t<_It>> _Temp_buf{_Temp_count};
 
             // _Temp_count + 1 since we work on closed ranges
             const auto _Total_count = static_cast<iter_difference_t<_It>>(_Temp_count + 1);
@@ -8442,7 +8442,7 @@ void inplace_merge(_BidIt _First, _BidIt _Mid, _BidIt _Last, _Pr _Pred) {
     }
 
     const _Diff _Count2 = _STD distance(_UMid, _ULast);
-    _Optimistic_temporary_buffer<_Iter_value_t<_BidIt>> _Temp_buf{(_STD min) (_Count1, _Count2)};
+    _Optimistic_temporary_buffer2<_Iter_value_t<_BidIt>> _Temp_buf{(_STD min) (_Count1, _Count2)};
     _STD _Buffered_inplace_merge_unchecked_impl(
         _UFirst, _UMid, _ULast, _Count1, _Count2, _Temp_buf._Data, _Temp_buf._Capacity, _STD _Pass_fn(_Pred));
 }
@@ -8793,7 +8793,7 @@ namespace ranges {
             }
 
             const iter_difference_t<_It> _Count2 = _RANGES distance(_Mid, _Last);
-            _Optimistic_temporary_buffer<iter_value_t<_It>> _Temp_buf{(_STD min) (_Count1, _Count2)};
+            _Optimistic_temporary_buffer2<iter_value_t<_It>> _Temp_buf{(_STD min) (_Count1, _Count2)};
             if (_Count1 <= _Count2 && _Count1 <= _Temp_buf._Capacity) {
                 _RANGES _Inplace_merge_buffer_left(_STD move(_First), _STD move(_Mid), _STD move(_Last),
                     _Temp_buf._Data, _Temp_buf._Capacity, _Pred, _Proj);
@@ -9403,7 +9403,7 @@ void stable_sort(const _BidIt _First, const _BidIt _Last, _Pr _Pred) {
         return;
     }
 
-    _Optimistic_temporary_buffer<_Iter_value_t<_BidIt>> _Temp_buf{_Count - _Count / 2};
+    _Optimistic_temporary_buffer2<_Iter_value_t<_BidIt>> _Temp_buf{_Count - _Count / 2};
     _STD _Stable_sort_unchecked(_UFirst, _ULast, _Count, _Temp_buf._Data, _Temp_buf._Capacity, _STD _Pass_fn(_Pred));
 }
 
@@ -9470,7 +9470,7 @@ namespace ranges {
                 return;
             }
 
-            _Optimistic_temporary_buffer<iter_value_t<_It>> _Temp_buf{_Count - _Count / 2};
+            _Optimistic_temporary_buffer2<iter_value_t<_It>> _Temp_buf{_Count - _Count / 2};
             _Stable_sort_common_buffered(
                 _STD move(_First), _STD move(_Last), _Count, _Temp_buf._Data, _Temp_buf._Capacity, _Pred, _Proj);
         }

stl/inc/execution

@@ -2785,14 +2785,14 @@ void sort(_ExPo&&, const _RanIt _First, const _RanIt _Last, _Pr _Pred) noexcept
 }
 
 template <class _Ty>
-struct _Static_partitioned_temporary_buffer2 {
-    _Optimistic_temporary_buffer<_Ty>& _Temp_buf;
+struct _Static_partitioned_temporary_buffer3 {
+    _Optimistic_temporary_buffer2<_Ty>& _Temp_buf;
     ptrdiff_t _Chunk_size;
     ptrdiff_t _Unchunked_items;
 
     template <class _Diff>
-    explicit _Static_partitioned_temporary_buffer2(
-        _Optimistic_temporary_buffer<_Ty>& _Temp_buf_raw, _Static_partition_team<_Diff>& _Team)
+    explicit _Static_partitioned_temporary_buffer3(
+        _Optimistic_temporary_buffer2<_Ty>& _Temp_buf_raw, _Static_partition_team<_Diff>& _Team)
         : _Temp_buf(_Temp_buf_raw), _Chunk_size(static_cast<ptrdiff_t>(_Temp_buf._Capacity / _Team._Chunks)),
           _Unchunked_items(static_cast<ptrdiff_t>(_Temp_buf._Capacity % _Team._Chunks)) {}
 
@@ -2899,15 +2899,15 @@ struct _Bottom_up_tree_visitor {
 };
 
 template <class _BidIt, class _Pr>
-struct _Static_partitioned_stable_sort3 {
+struct _Static_partitioned_stable_sort4 {
     using _Diff = _Iter_diff_t<_BidIt>;
     _Static_partition_team<_Diff> _Team;
     _Static_partition_range<_BidIt> _Basis;
     _Bottom_up_merge_tree _Merge_tree;
-    _Static_partitioned_temporary_buffer2<_Iter_value_t<_BidIt>> _Temp_buf;
+    _Static_partitioned_temporary_buffer3<_Iter_value_t<_BidIt>> _Temp_buf;
     _Pr _Pred;
 
-    _Static_partitioned_stable_sort3(_Optimistic_temporary_buffer<_Iter_value_t<_BidIt>>& _Temp_buf_raw,
+    _Static_partitioned_stable_sort4(_Optimistic_temporary_buffer2<_Iter_value_t<_BidIt>>& _Temp_buf_raw,
         const _Diff _Count, const size_t _Merge_tree_height_, const _BidIt _First, _Pr _Pred_)
         : _Team(_Count, static_cast<size_t>(1) << _Merge_tree_height_), _Basis{}, _Merge_tree(_Merge_tree_height_),
           _Temp_buf(_Temp_buf_raw, _Team), _Pred{_Pred_} {
@@ -3001,7 +3001,7 @@ struct _Static_partitioned_stable_sort3 {
 
     static void __stdcall _Threadpool_callback(
         __std_PTP_CALLBACK_INSTANCE, void* const _Context, __std_PTP_WORK) noexcept /* terminates */ {
-        _STD _Run_available_chunked_work(*static_cast<_Static_partitioned_stable_sort3*>(_Context));
+        _STD _Run_available_chunked_work(*static_cast<_Static_partitioned_stable_sort4*>(_Context));
     }
 };
 
@@ -3027,14 +3027,14 @@ void stable_sort(_ExPo&&, const _BidIt _First, const _BidIt _Last, _Pr _Pred) no
         _Attempt_parallelism = false;
     }
 
-    _Optimistic_temporary_buffer<_Iter_value_t<_BidIt>> _Temp_buf{_Attempt_parallelism ? _Count : _Count - _Count / 2};
+    _Optimistic_temporary_buffer2<_Iter_value_t<_BidIt>> _Temp_buf{_Attempt_parallelism ? _Count : _Count - _Count / 2};
     if constexpr (remove_reference_t<_ExPo>::_Parallelize) {
         if (_Attempt_parallelism) {
             // forward+ iterator overflow assumption for size_t cast
             const auto _Tree_height = _Get_stable_sort_tree_height(static_cast<size_t>(_Count), _Hw_threads);
             if (_Tree_height != 0) {
                 _TRY_BEGIN
-                _Static_partitioned_stable_sort3 _Operation{
+                _Static_partitioned_stable_sort4 _Operation{
                     _Temp_buf, _Count, _Tree_height, _UFirst, _STD _Pass_fn(_Pred)};
                 _STD _Run_chunked_parallel_work(_Hw_threads, _Operation);
                 return;

tests/std/test.lst

@@ -272,6 +272,7 @@ tests\GH_005472_do_not_overlap
 tests\GH_005546_containers_size_type_cast
 tests\GH_005553_regex_character_translation
 tests\GH_005768_pow_accuracy
+tests\GH_005800_stable_sort_large_alignment
 tests\LWG2381_num_get_floating_point
 tests\LWG2510_tag_classes
 tests\LWG2597_complex_branch_cut

tests/std/tests/GH_005800_stable_sort_large_alignment/env.lst

@@ -0,0 +1,4 @@
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+RUNALL_INCLUDE ..\impure_matrix.lst

tests/std/tests/GH_005800_stable_sort_large_alignment/test.cpp

@@ -0,0 +1,121 @@
+// Copyright (c) Microsoft Corporation.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#pragma warning(disable : 6262) // Function uses '16388' bytes of stack.
+
+#include <algorithm>
+#include <array>
+#include <cassert>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+
+#if _HAS_CXX17
+#include <execution>
+#endif // _HAS_CXX17
+
+using namespace std;
+
+template <size_t N>
+struct alignas(N) large_element {
+    array<unsigned char, N> elems;
+
+#if _HAS_CXX20
+    friend auto operator<=>(const large_element&, const large_element&) = default;
+#else // ^^^ _HAS_CXX20 / !_HAS_CXX20 vvv
+    friend bool operator==(const large_element& lhs, const large_element& rhs) {
+        return lhs.elems == rhs.elems;
+    }
+
+    friend bool operator!=(const large_element& lhs, const large_element& rhs) {
+        return lhs.elems != rhs.elems;
+    }
+
+    friend bool operator<(const large_element& lhs, const large_element& rhs) {
+        return lhs.elems < rhs.elems;
+    }
+
+    friend bool operator>(const large_element& lhs, const large_element& rhs) {
+        return lhs.elems > rhs.elems;
+    }
+
+    friend bool operator<=(const large_element& lhs, const large_element& rhs) {
+        return lhs.elems <= rhs.elems;
+    }
+
+    friend bool operator>=(const large_element& lhs, const large_element& rhs) {
+        return lhs.elems >= rhs.elems;
+    }
+#endif // ^^^ !_HAS_CXX20 ^^^
+};
+
+struct alignment_verifying_less {
+    template <class T, class U>
+    bool operator()(const T& t, const U& u) const {
+        assert(reinterpret_cast<uintptr_t>(&t) % alignof(T) == 0);
+        assert(reinterpret_cast<uintptr_t>(&u) % alignof(U) == 0);
+        return t < u;
+    }
+};
+
+struct alignment_verifying_truth {
+    template <class T>
+    bool operator()(const T& t) const {
+        assert(reinterpret_cast<uintptr_t>(&t) % alignof(T) == 0);
+        return true;
+    }
+};
+
+template <size_t N>
+void test() {
+    {
+        large_element<N> arr[2]{};
+
+        stable_sort(begin(arr), end(arr), alignment_verifying_less{});
+        stable_partition(begin(arr), end(arr), alignment_verifying_truth{});
+        inplace_merge(begin(arr), begin(arr), end(arr), alignment_verifying_less{});
+    }
+
+#if _HAS_CXX17
+    auto test_execution = [](const auto& execpol) {
+        large_element<N> arr[2]{};
+
+        stable_sort(execpol, begin(arr), end(arr), alignment_verifying_less{});
+        stable_partition(execpol, begin(arr), end(arr), alignment_verifying_truth{});
+        inplace_merge(execpol, begin(arr), begin(arr), end(arr), alignment_verifying_less{});
+    };
+    test_execution(execution::seq);
+    test_execution(execution::par);
+    test_execution(execution::par_unseq);
+#if _HAS_CXX20
+    test_execution(execution::unseq);
+#endif // _HAS_CXX20
+#endif // _HAS_CXX17
+
+#if _HAS_CXX20
+    {
+        large_element<N> arr[2]{};
+
+        ranges::stable_sort(arr, alignment_verifying_less{});
+        ranges::stable_partition(arr, alignment_verifying_truth{});
+        ranges::inplace_merge(arr, ranges::begin(arr), alignment_verifying_less{});
+    }
+#endif // _HAS_CXX20
+}
+
+int main() {
+    test<1>();
+    test<2>();
+    test<4>();
+    test<8>();
+    test<16>();
+    test<32>();
+    test<64>();
+    test<128>();
+    test<256>();
+    test<512>();
+    test<1024>();
+    test<2048>();
+    test<4096>();
+    test<8192>();
+}