Vectorize mismatch for clang-cl for odd element sizes (#5591)

Co-authored-by: Stephan T. Lavavej <[email protected]>

Author: AlexGuteniev

benchmarks/src/mismatch.cpp

@@ -19,13 +19,24 @@ enum class op {
     lexi,
 };
 
-template <class T, op Op>
+struct color {
+    uint16_t h;
+    uint16_t s;
+    uint16_t l;
+
+    bool operator==(const color&) const = default;
+};
+
+constexpr color c1{30000, 40000, 20000};
+constexpr color c2{30000, 40000, 30000};
+
+template <class T, op Op, T MatchVal = T{'.'}, T MismatchVal = T{'x'}>
 void bm(benchmark::State& state) {
-    vector<T, not_highly_aligned_allocator<T>> a(static_cast<size_t>(state.range(0)), T{'.'});
-    vector<T, not_highly_aligned_allocator<T>> b(static_cast<size_t>(state.range(0)), T{'.'});
+    vector<T, not_highly_aligned_allocator<T>> a(static_cast<size_t>(state.range(0)), MatchVal);
+    vector<T, not_highly_aligned_allocator<T>> b(static_cast<size_t>(state.range(0)), MatchVal);
 
     if (state.range(1) != no_pos) {
-        b.at(static_cast<size_t>(state.range(1))) = 'x';
+        b.at(static_cast<size_t>(state.range(1))) = MismatchVal;
     }
 
     for (auto _ : state) {
@@ -45,6 +56,7 @@ BENCHMARK(bm<uint8_t, op::mismatch>)->Apply(common_args);
 BENCHMARK(bm<uint16_t, op::mismatch>)->Apply(common_args);
 BENCHMARK(bm<uint32_t, op::mismatch>)->Apply(common_args);
 BENCHMARK(bm<uint64_t, op::mismatch>)->Apply(common_args);
+BENCHMARK(bm<color, op::mismatch, c1, c2>)->Apply(common_args);
 
 BENCHMARK(bm<uint8_t, op::lexi>)->Apply(common_args); // still optimized without vector algorithms using memcmp
 BENCHMARK(bm<int8_t, op::lexi>)->Apply(common_args); // optimized with vector algorithms only

stl/inc/algorithm

@@ -885,7 +885,7 @@ _NODISCARD _CONSTEXPR20 pair<_InIt1, _InIt2> mismatch(_InIt1 _First1, const _InI
     const auto _ULast1 = _STD _Get_unwrapped(_Last1);
     auto _UFirst2      = _STD _Get_unwrapped_n(_First2, _STD _Idl_distance<_InIt1>(_UFirst1, _ULast1));
 #if _USE_STD_VECTOR_ALGORITHMS
-    if constexpr (_Vector_alg_in_search_is_safe<decltype(_UFirst1), decltype(_UFirst2), _Pr>) {
+    if constexpr (_Equal_memcmp_is_safe<decltype(_UFirst1), decltype(_UFirst2), _Pr>) {
         if (!_STD _Is_constant_evaluated()) {
             constexpr size_t _Elem_size = sizeof(_Iter_value_t<_InIt1>);
 
@@ -949,7 +949,7 @@ _NODISCARD _CONSTEXPR20 pair<_InIt1, _InIt2> mismatch(
         const auto _Count = static_cast<_Iter_diff_t<_InIt1>>((_STD min) (_Count1, _Count2));
         _ULast1           = _UFirst1 + _Count;
 #if _USE_STD_VECTOR_ALGORITHMS
-        if constexpr (_Vector_alg_in_search_is_safe<decltype(_UFirst1), decltype(_UFirst2), _Pr>) {
+        if constexpr (_Equal_memcmp_is_safe<decltype(_UFirst1), decltype(_UFirst2), _Pr>) {
             if (!_STD _Is_constant_evaluated()) {
                 constexpr size_t _Elem_size = sizeof(_Iter_value_t<_InIt1>);
 

stl/inc/xutility

@@ -447,16 +447,14 @@ auto _Max_vectorized(_Ty* const _First, _Ty* const _Last) noexcept {
 
 template <size_t _Element_size>
 size_t _Mismatch_vectorized(const void* const _First1, const void* const _First2, const size_t _Count) noexcept {
-    if constexpr (_Element_size == 1) {
-        return __std_mismatch_1(_First1, _First2, _Count);
-    } else if constexpr (_Element_size == 2) {
-        return __std_mismatch_2(_First1, _First2, _Count);
-    } else if constexpr (_Element_size == 4) {
-        return __std_mismatch_4(_First1, _First2, _Count);
-    } else if constexpr (_Element_size == 8) {
-        return __std_mismatch_8(_First1, _First2, _Count);
+    if constexpr (_Element_size % 8 == 0) {
+        return __std_mismatch_8(_First1, _First2, _Count * (_Element_size / 8)) / (_Element_size / 8);
+    } else if constexpr (_Element_size % 4 == 0) {
+        return __std_mismatch_4(_First1, _First2, _Count * (_Element_size / 4)) / (_Element_size / 4);
+    } else if constexpr (_Element_size % 2 == 0) {
+        return __std_mismatch_2(_First1, _First2, _Count * (_Element_size / 2)) / (_Element_size / 2);
     } else {
-        _STL_INTERNAL_STATIC_ASSERT(false); // unexpected size
+        return __std_mismatch_1(_First1, _First2, _Count * _Element_size) / _Element_size;
     }
 }
 _STD_END
@@ -5772,7 +5770,7 @@ namespace ranges {
         _It1 _First1, _It2 _First2, iter_difference_t<_It1> _Count, _Pr _Pred, _Pj1 _Proj1, _Pj2 _Proj2) {
         _STL_INTERNAL_CHECK(_Count >= 0);
 #if _USE_STD_VECTOR_ALGORITHMS
-        if constexpr (_Vector_alg_in_search_is_safe<_It1, _It2, _Pr> && is_same_v<_Pj1, identity>
+        if constexpr (_Equal_memcmp_is_safe<_It1, _It2, _Pr> && is_same_v<_Pj1, identity>
                       && is_same_v<_Pj2, identity>) {
             if (!_STD is_constant_evaluated()) {
                 constexpr size_t _Elem_size = sizeof(iter_value_t<_It1>);

tests/std/tests/VSO_0000000_vector_algorithms_mismatch_and_lex_compare/test.cpp

@@ -64,20 +64,28 @@ auto last_known_good_lex_compare_3way(pair<FwdIt, FwdIt> expected_mismatch, FwdI
 #endif // _HAS_CXX20
 
 template <class T>
-void test_case_mismatch_and_lex_compare_family(const vector<T>& a, const vector<T>& b) {
+auto test_case_mismatch_only(const vector<T>& a, const vector<T>& b) {
     auto expected_mismatch = last_known_good_mismatch(a.begin(), a.end(), b.begin(), b.end());
     auto actual_mismatch   = mismatch(a.begin(), a.end(), b.begin(), b.end());
     assert(expected_mismatch == actual_mismatch);
 
-    auto expected_lex = last_known_good_lex_compare(expected_mismatch, a.end(), b.end());
-    auto actual_lex   = lexicographical_compare(a.begin(), a.end(), b.begin(), b.end());
-    assert(expected_lex == actual_lex);
-
 #if _HAS_CXX20
     auto ranges_actual_mismatch = ranges::mismatch(a, b);
     assert(get<0>(expected_mismatch) == ranges_actual_mismatch.in1);
     assert(get<1>(expected_mismatch) == ranges_actual_mismatch.in2);
+#endif // _HAS_CXX20
+    return expected_mismatch;
+}
+
+template <class T>
+void test_case_mismatch_and_lex_compare_family(const vector<T>& a, const vector<T>& b) {
+    auto expected_mismatch = test_case_mismatch_only(a, b);
+
+    auto expected_lex = last_known_good_lex_compare(expected_mismatch, a.end(), b.end());
+    auto actual_lex   = lexicographical_compare(a.begin(), a.end(), b.begin(), b.end());
+    assert(expected_lex == actual_lex);
 
+#if _HAS_CXX20
     auto ranges_actual_lex = ranges::lexicographical_compare(a, b);
     assert(expected_lex == ranges_actual_lex);
 
@@ -130,6 +138,65 @@ void test_mismatch_and_lex_compare_family(mt19937_64& gen) {
     }
 }
 
+#if _HAS_CXX20
+template <class T>
+struct triplet {
+    T x;
+    T y;
+    T z;
+
+    bool operator==(const triplet&) const = default;
+};
+
+template <class T>
+void test_mismatch_only_triplets(mt19937_64& gen) {
+    constexpr size_t shrinkCount   = 4;
+    constexpr size_t mismatchCount = 10;
+    using TD                       = conditional_t<sizeof(T) == 1, int, T>;
+    uniform_int_distribution<TD> dis('a', 'z');
+    vector<triplet<T>> input_a;
+    vector<triplet<T>> input_b;
+    input_a.reserve(dataCount);
+    input_b.reserve(dataCount);
+
+    for (;;) {
+        // equal
+        test_case_mismatch_only(input_a, input_b);
+
+        // different sizes
+        for (size_t i = 0; i != shrinkCount && !input_b.empty(); ++i) {
+            input_b.pop_back();
+            test_case_mismatch_only(input_a, input_b);
+            test_case_mismatch_only(input_b, input_a);
+        }
+
+        // actual mismatch (or maybe not, depending on random)
+        if (!input_b.empty()) {
+            uniform_int_distribution<size_t> mismatch_dis(0, input_a.size() - 1);
+
+            for (size_t attempts = 0; attempts < mismatchCount; ++attempts) {
+                const size_t possible_mismatch_pos = mismatch_dis(gen);
+                input_a[possible_mismatch_pos].x   = static_cast<T>(dis(gen));
+                input_a[possible_mismatch_pos].y   = static_cast<T>(dis(gen));
+                input_a[possible_mismatch_pos].z   = static_cast<T>(dis(gen));
+                test_case_mismatch_only(input_a, input_b);
+                test_case_mismatch_only(input_b, input_a);
+            }
+        }
+
+        if (input_a.size() == dataCount) {
+            break;
+        }
+
+        input_a.emplace_back();
+        input_a.back().x = static_cast<T>(dis(gen));
+        input_a.back().y = static_cast<T>(dis(gen));
+        input_a.back().z = static_cast<T>(dis(gen));
+        input_b          = input_a;
+    }
+}
+#endif // _HAS_CXX20
+
 template <class C1, class C2>
 void test_mismatch_and_lex_compare_family_containers() {
     C1 a{'m', 'e', 'o', 'w', ' ', 'C', 'A', 'T', 'S'};
@@ -245,6 +312,18 @@ void test_vector_algorithms(mt19937_64& gen) {
     test_mismatch_and_lex_compare_family<long long>(gen);
     test_mismatch_and_lex_compare_family<unsigned long long>(gen);
 
+#if _HAS_CXX20
+    test_mismatch_only_triplets<char>(gen);
+    test_mismatch_only_triplets<signed char>(gen);
+    test_mismatch_only_triplets<unsigned char>(gen);
+    test_mismatch_only_triplets<short>(gen);
+    test_mismatch_only_triplets<unsigned short>(gen);
+    test_mismatch_only_triplets<int>(gen);
+    test_mismatch_only_triplets<unsigned int>(gen);
+    test_mismatch_only_triplets<long long>(gen);
+    test_mismatch_only_triplets<unsigned long long>(gen);
+#endif // _HAS_CXX20
+
     test_mismatch_and_lex_compare_family_containers<vector<char>, vector<signed char>>();
     test_mismatch_and_lex_compare_family_containers<vector<char>, vector<unsigned char>>();
     test_mismatch_and_lex_compare_family_containers<vector<wchar_t>, vector<char>>();