(Semi-)vectorize includes (#5590)

Co-authored-by: Stephan T. Lavavej <[email protected]>

Author: AlexGuteniev

benchmarks/src/includes.cpp

@@ -111,6 +111,16 @@ void common_args(auto bm) {
     }
 }
 
+BENCHMARK(bm_includes<uint8_t, alg_type::std_fn>)->Apply(common_args);
+BENCHMARK(bm_includes<uint16_t, alg_type::std_fn>)->Apply(common_args);
+BENCHMARK(bm_includes<uint32_t, alg_type::std_fn>)->Apply(common_args);
+BENCHMARK(bm_includes<uint64_t, alg_type::std_fn>)->Apply(common_args);
+
+BENCHMARK(bm_includes<uint8_t, alg_type::rng>)->Apply(common_args);
+BENCHMARK(bm_includes<uint16_t, alg_type::rng>)->Apply(common_args);
+BENCHMARK(bm_includes<uint32_t, alg_type::rng>)->Apply(common_args);
+BENCHMARK(bm_includes<uint64_t, alg_type::rng>)->Apply(common_args);
+
 BENCHMARK(bm_includes<int8_t, alg_type::std_fn>)->Apply(common_args);
 BENCHMARK(bm_includes<int16_t, alg_type::std_fn>)->Apply(common_args);
 BENCHMARK(bm_includes<int32_t, alg_type::std_fn>)->Apply(common_args);

stl/inc/algorithm

@@ -86,6 +86,23 @@ const void* __stdcall __std_is_sorted_until_8u(const void* _First, const void* _
 const void* __stdcall __std_is_sorted_until_f(const void* _First, const void* _Last, bool _Greater) noexcept;
 const void* __stdcall __std_is_sorted_until_d(const void* _First, const void* _Last, bool _Greater) noexcept;
 
+__declspec(noalias) bool __stdcall __std_includes_less_1i(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_1u(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_2i(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_2u(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_4i(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_4u(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_8i(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+__declspec(noalias) bool __stdcall __std_includes_less_8u(
+    const void* _First1, const void* _Last1, const void* _First2, const void* _Last2) noexcept;
+
 // TRANSITION, DevCom-10610477
 __declspec(noalias) void __stdcall __std_replace_4(
     void* _First, void* _Last, uint32_t _Old_val, uint32_t _New_val) noexcept;
@@ -256,6 +273,40 @@ _Ty* _Is_sorted_until_vectorized(_Ty* const _First, _Ty* const _Last, const bool
     }
 }
 
+template <class _Ty>
+bool _Includes_vectorized(
+    const _Ty* const _First1, const _Ty* const _Last1, const _Ty* const _First2, const _Ty* const _Last2) noexcept {
+    constexpr bool _Signed = is_signed_v<_Ty>;
+
+    if constexpr (sizeof(_Ty) == 1) {
+        if constexpr (_Signed) {
+            return ::__std_includes_less_1i(_First1, _Last1, _First2, _Last2);
+        } else {
+            return ::__std_includes_less_1u(_First1, _Last1, _First2, _Last2);
+        }
+    } else if constexpr (sizeof(_Ty) == 2) {
+        if constexpr (_Signed) {
+            return ::__std_includes_less_2i(_First1, _Last1, _First2, _Last2);
+        } else {
+            return ::__std_includes_less_2u(_First1, _Last1, _First2, _Last2);
+        }
+    } else if constexpr (sizeof(_Ty) == 4) {
+        if constexpr (_Signed) {
+            return ::__std_includes_less_4i(_First1, _Last1, _First2, _Last2);
+        } else {
+            return ::__std_includes_less_4u(_First1, _Last1, _First2, _Last2);
+        }
+    } else if constexpr (sizeof(_Ty) == 8) {
+        if constexpr (_Signed) {
+            return ::__std_includes_less_8i(_First1, _Last1, _First2, _Last2);
+        } else {
+            return ::__std_includes_less_8u(_First1, _Last1, _First2, _Last2);
+        }
+    } else {
+        _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size
+    }
+}
+
 template <class _Ty, class _TVal1, class _TVal2>
 __declspec(noalias) void _Replace_vectorized(
     _Ty* const _First, _Ty* const _Last, const _TVal1 _Old_val, const _TVal2 _New_val) noexcept {
@@ -384,6 +435,13 @@ constexpr bool _Output_iterator_for_vector_alg_is_safe() {
     }
 }
 
+// Can we activate the vector algorithms for includes?
+template <class _Iter1, class _Iter2, class _Elem = _Iter_value_t<_Iter1>>
+constexpr bool _Vector_alg_includes_iterators_safe =
+    _Iterators_are_contiguous<_Iter1, _Iter2> // Iterators must be contiguous so we can get raw pointers.
+    && !_Iterator_is_volatile<_Iter1> && !_Iterator_is_volatile<_Iter2> // Iterators must not be volatile.
+    && is_same_v<_Elem, _Iter_value_t<_Iter2>> // Iterators have the same value type.
+    && disjunction_v<is_integral<_Elem>, is_pointer<_Elem>>; // Integral or pointer type.
 _STD_END
 #endif // _USE_STD_VECTOR_ALGORITHMS
 
@@ -10244,6 +10302,15 @@ _NODISCARD _CONSTEXPR20 bool includes(_InIt1 _First1, _InIt1 _Last1, _InIt2 _Fir
         return false;
     }
 
+#if _USE_STD_VECTOR_ALGORITHMS
+    if constexpr (_Vector_alg_includes_iterators_safe<_InIt1, _InIt2> && _Is_predicate_less<_InIt1, _Pr>) {
+        if (!_STD _Is_constant_evaluated()) {
+            return _STD _Includes_vectorized(_STD _To_address(_First1), _STD _To_address(_Last1),
+                _STD _To_address(_First2), _STD _To_address(_Last2));
+        }
+    }
+#endif // _USE_STD_VECTOR_ALGORITHMS
+
     for (;;) {
         if (_DEBUG_LT_PRED(_Pred, *_UFirst1, *_UFirst2)) {
             ++_UFirst1;
@@ -10333,6 +10400,20 @@ namespace ranges {
                 return false;
             }
 
+#if _USE_STD_VECTOR_ALGORITHMS
+            if constexpr (_Vector_alg_includes_iterators_safe<_It1, _It2> && _Is_predicate_less<_It1, _Pr>
+                          && sized_sentinel_for<_Se1, _It1> && sized_sentinel_for<_Se2, _It2>
+                          && is_same_v<_Pj1, identity> && is_same_v<_Pj2, identity>) {
+                if (!_STD is_constant_evaluated()) {
+                    const auto _First1_ptr = _STD to_address(_First1);
+                    const auto _First2_ptr = _STD to_address(_First2);
+                    const auto _Last1_ptr  = _First1_ptr + static_cast<ptrdiff_t>(_Last1 - _First1);
+                    const auto _Last2_ptr  = _First2_ptr + static_cast<ptrdiff_t>(_Last2 - _First2);
+                    return _STD _Includes_vectorized(_First1_ptr, _Last1_ptr, _First2_ptr, _Last2_ptr);
+                }
+            }
+#endif // _USE_STD_VECTOR_ALGORITHMS
+
             for (;;) {
                 if (_STD invoke(_Pred, _STD invoke(_Proj1, *_First1), _STD invoke(_Proj2, *_First2))) {
                     ++_First1;