update fix

BiteTheDDDDt · BiteTheDDDDt · commit 79dd3c119876 · 2026-01-06T22:38:31.000+08:00
format

update

format

fix

update fix

fix
diff --git a/be/src/pipeline/exec/join/process_hash_table_probe_impl.h b/be/src/pipeline/exec/join/process_hash_table_probe_impl.h
@@ -114,8 +114,8 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(vectorized::Mut
             _build_column_has_null[i] = false;
             if (_right_output_slot_flags[i] && column.is_nullable()) {
                 const auto& nullable = assert_cast<const vectorized::ColumnNullable&>(column);
-                _build_column_has_null[i] = !simd::contain_byte(
-                        nullable.get_null_map_data().data() + 1, nullable.size() - 1, 1);
+                _build_column_has_null[i] = !simd::contain_one(
+                        nullable.get_null_map_data().data() + 1, nullable.size() - 1);
             }
         }
     }
@@ -368,8 +368,7 @@ Status ProcessHashTableProbe<JoinOpType>::finalize_block_with_filter(
         }
         const auto& column_filter =
                 assert_cast<const vectorized::ColumnUInt8*>(filter_ptr.get())->get_data();
-        bool need_filter =
-                simd::count_zero_num((int8_t*)column_filter.data(), column_filter.size()) != 0;
+        bool need_filter = simd::contain_zero(column_filter.data(), column_filter.size());
         if (need_filter) {
             row_indexs.filter(column_filter);
         }
diff --git a/be/src/pipeline/exec/nested_loop_join_probe_operator.h b/be/src/pipeline/exec/nested_loop_join_probe_operator.h
@@ -97,9 +97,7 @@ class NestedLoopJoinProbeLocalState final
                     }
                     if (!_cur_probe_row_visited_flags[i]) {
                         _cur_probe_row_visited_flags[i] =
-                                simd::contain_byte<uint8_t>(filter.data() + offset, end - offset, 1)
-                                        ? 1
-                                        : 0;
+                                simd::contain_one(filter.data() + offset, end - offset);
                     }
                     end = offset;
                 }
diff --git a/be/src/util/simd/bits.h b/be/src/util/simd/bits.h
@@ -261,14 +261,6 @@ static size_t find_byte(const T* data, size_t start, size_t end, T byte) {
     return (T*)p - data;
 }
 
-template <typename T>
-bool contain_byte(const T* __restrict data, const size_t length, const signed char byte) {
-    if (length == 0) {
-        return false;
-    }
-    return nullptr != std::memchr(reinterpret_cast<const void*>(data), byte, length);
-}
-
 inline size_t find_one(const std::vector<uint8_t>& vec, size_t start) {
     return find_byte<uint8_t>(vec, start, 1);
 }
@@ -283,24 +275,55 @@ inline size_t find_zero(const std::vector<uint8_t>& vec, size_t start) {
 
 inline bool contain_one(const uint8_t* __restrict data, size_t size) {
     size_t i = 0;
-#if defined(__SSE2__)
-    {
-        const __m128i zero = _mm_setzero_si128();
-        __m128i acc = zero;
-        for (; i + 15 < size; i += 16) {
-            acc = _mm_or_si128(acc, _mm_loadu_si128(reinterpret_cast<const __m128i*>(data + i)));
+#if defined(__AVX2__)
+    for (; i + 32 <= size; i += 32) {
+        __m256i chunk = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(data + i));
+        if (!_mm256_testz_si256(chunk, chunk)) {
+            return true;
+        }
+    }
+#elif defined(__SSE2__)
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 16 <= size; i += 16) {
+        __m128i chunk = _mm_loadu_si128(reinterpret_cast<const __m128i*>(data + i));
+        if (_mm_movemask_epi8(_mm_cmpeq_epi8(chunk, zero)) != 0xFFFF) {
+            return true;
         }
+    }
+#endif
+    for (; i < size; ++i) {
+        if (data[i]) {
+            return true;
+        }
+    }
+    return false;
+}
 
-        if (_mm_movemask_epi8(_mm_cmpeq_epi8(acc, zero)) != 0xFFFF) {
+inline bool contain_zero(const uint8_t* __restrict data, size_t size) {
+    size_t i = 0;
+#if defined(__AVX2__)
+    const __m256i zero = _mm256_setzero_si256();
+    for (; i + 32 <= size; i += 32) {
+        __m256i chunk = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(data + i));
+        if (_mm256_movemask_epi8(_mm256_cmpeq_epi8(chunk, zero)) != 0) {
+            return true;
+        }
+    }
+#elif defined(__SSE2__)
+    const __m128i zero = _mm_setzero_si128();
+    for (; i + 16 <= size; i += 16) {
+        __m128i chunk = _mm_loadu_si128(reinterpret_cast<const __m128i*>(data + i));
+        if (_mm_movemask_epi8(_mm_cmpeq_epi8(chunk, zero)) != 0) {
             return true;
         }
     }
 #endif
-    uint8_t tail_acc = 0;
     for (; i < size; ++i) {
-        tail_acc |= data[i];
+        if (!data[i]) {
+            return true;
+        }
     }
-    return tail_acc != 0;
+    return false;
 }
 
 } // namespace doris::simd
diff --git a/be/src/vec/columns/column_nullable.cpp b/be/src/vec/columns/column_nullable.cpp
@@ -280,7 +280,7 @@ size_t ColumnNullable::serialize_impl(char* pos, const size_t row) const {
 }
 
 void ColumnNullable::serialize(StringRef* keys, size_t num_rows) const {
-    const bool has_null = simd::contain_byte(get_null_map_data().data(), num_rows, 1);
+    const bool has_null = simd::contain_one(get_null_map_data().data(), num_rows);
     const auto* __restrict null_map =
             assert_cast<const ColumnUInt8&>(get_null_map_column()).get_data().data();
     _nested_column->serialize_with_nullable(keys, num_rows, has_null, null_map);
@@ -598,11 +598,11 @@ void ColumnNullable::sort_column(const ColumnSorter* sorter, EqualFlags& flags,
 }
 
 bool ColumnNullable::only_null() const {
-    return !simd::contain_byte(get_null_map_data().data(), size(), 0);
+    return !simd::contain_zero(get_null_map_data().data(), size());
 }
 
 bool ColumnNullable::has_null(size_t begin, size_t end) const {
-    return simd::contain_byte(get_null_map_data().data() + begin, end - begin, 1);
+    return simd::contain_one(get_null_map_data().data() + begin, end - begin);
 }
 
 bool ColumnNullable::has_null() const {
diff --git a/be/src/vec/common/hash_table/hash_key_type.h b/be/src/vec/common/hash_table/hash_key_type.h
@@ -80,6 +80,10 @@ inline HashKeyType get_hash_key_type_with_fixed(size_t size) {
 }
 
 inline HashKeyType get_hash_key_type_fixed(const std::vector<vectorized::DataTypePtr>& data_types) {
+    if (data_types.size() >= vectorized::BITSIZE) {
+        return HashKeyType::serialized;
+    }
+
     bool has_null = false;
     size_t key_byte_size = 0;
 
@@ -94,8 +98,7 @@ inline HashKeyType get_hash_key_type_fixed(const std::vector<vectorized::DataTyp
         }
     }
 
-    size_t bitmap_size = has_null ? vectorized::get_bitmap_size(data_types.size()) : 0;
-    return get_hash_key_type_with_fixed(bitmap_size + key_byte_size);
+    return get_hash_key_type_with_fixed(has_null + key_byte_size);
 }
 
 inline HashKeyType get_hash_key_type(const std::vector<vectorized::DataTypePtr>& data_types) {
diff --git a/be/src/vec/common/hash_table/hash_map_context.h b/be/src/vec/common/hash_table/hash_map_context.h
@@ -17,6 +17,7 @@
 
 #pragma once
 
+#include <algorithm>
 #include <cstdint>
 #include <type_traits>
 #include <utility>
@@ -291,10 +292,6 @@ struct MethodSerialized : public MethodBase<TData> {
     }
 };
 
-inline size_t get_bitmap_size(size_t key_number) {
-    return (key_number + BITSIZE - 1) / BITSIZE;
-}
-
 template <typename TData>
 struct MethodStringNoCache : public MethodBase<TData> {
     using Base = MethodBase<TData>;
@@ -562,7 +559,7 @@ struct MethodKeysFixed : public MethodBase<TData> {
     template <typename T>
     void pack_fixeds(size_t row_numbers, const ColumnRawPtrs& key_columns,
                      const ColumnRawPtrs& nullmap_columns, DorisVector<T>& result) {
-        size_t bitmap_size = get_bitmap_size(nullmap_columns.size());
+        size_t bitmap_size = nullmap_columns.empty() ? 0 : 1;
         if (bitmap_size) {
             // set size to 0 at first, then use resize to call default constructor on index included from [0, row_numbers) to reset all memory
             // only need to reset the memory used to bitmap
@@ -587,16 +584,12 @@ struct MethodKeysFixed : public MethodBase<TData> {
                 has_null_column[j] = simd::contain_one(data, row_numbers);
                 if (has_null_column[j]) {
                     nullmap_datas.emplace_back(data);
-                    bit_offsets.emplace_back(j % BITSIZE);
+                    bit_offsets.emplace_back(j);
                 }
             }
-            for (size_t j = 0, bucket = 0; j < nullmap_datas.size(); j += BITSIZE, bucket++) {
-                int column_batch = std::min(BITSIZE, (int)(nullmap_datas.size() - j));
-                constexpr_int_match<1, BITSIZE, PackNullmapsReducer>::run(
-                        column_batch, nullmap_datas.data() + j, bit_offsets.data() + j, row_numbers,
-                        sizeof(T), reinterpret_cast<uint8_t*>(result_data + bucket));
-            }
-
+            constexpr_int_match<1, BITSIZE, PackNullmapsReducer>::run(
+                    int(nullmap_datas.size()), nullmap_datas.data(), bit_offsets.data(),
+                    row_numbers, sizeof(T), reinterpret_cast<uint8_t*>(result_data));
             offset += bitmap_size;
         }
 
@@ -657,6 +650,7 @@ struct MethodKeysFixed : public MethodBase<TData> {
     void init_serialized_keys(const ColumnRawPtrs& key_columns, uint32_t num_rows,
                               const uint8_t* null_map = nullptr, bool is_join = false,
                               bool is_build = false, uint32_t bucket_size = 0) override {
+        CHECK(key_columns.size() <= BITSIZE);
         ColumnRawPtrs actual_columns;
         ColumnRawPtrs null_maps;
         actual_columns.reserve(key_columns.size());
@@ -694,14 +688,8 @@ struct MethodKeysFixed : public MethodBase<TData> {
 
     void insert_keys_into_columns(std::vector<typename Base::Key>& input_keys,
                                   MutableColumns& key_columns, const uint32_t num_rows) override {
-        // In any hash key value, column values to be read start just after the bitmap, if it exists.
-        size_t pos = 0;
-        for (size_t i = 0; i < key_columns.size(); ++i) {
-            if (key_columns[i]->is_nullable()) {
-                pos = get_bitmap_size(key_columns.size());
-                break;
-            }
-        }
+        size_t pos = std::ranges::any_of(key_columns,
+                                         [](const auto& col) { return col->is_nullable(); });
 
         for (size_t i = 0; i < key_columns.size(); ++i) {
             size_t size = key_sizes[i];
@@ -720,11 +708,8 @@ struct MethodKeysFixed : public MethodBase<TData> {
 
                 // The current column is nullable. Check if the value of the
                 // corresponding key is nullable. Update the null map accordingly.
-                size_t bucket = i / BITSIZE;
-                size_t offset = i % BITSIZE;
                 for (size_t j = 0; j < num_rows; j++) {
-                    nullmap[j] =
-                            (reinterpret_cast<const UInt8*>(&input_keys[j])[bucket] >> offset) & 1;
+                    nullmap[j] = (*reinterpret_cast<const UInt8*>(&input_keys[j]) >> i) & 1;
                 }
             } else {
                 // key_columns is a mutable element. However, when accessed through get_raw_data().data,
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.cpp b/be/src/vec/data_types/serde/data_type_map_serde.cpp
@@ -352,8 +352,8 @@ Status DataTypeMapSerDe::write_column_to_arrow(const IColumn& column, const Null
         if ((null_map && (*null_map)[r])) {
             RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column.get_name(),
                                              array_builder->type()->name()));
-        } else if (simd::contain_byte(keys_nullmap_data + offsets[r - 1],
-                                      offsets[r] - offsets[r - 1], 1)) {
+        } else if (simd::contain_one(keys_nullmap_data + offsets[r - 1],
+                                     offsets[r] - offsets[r - 1])) {
             // arrow do not support key is null, so we ignore the null key-value
             MutableColumnPtr key_mutable_data = nested_keys_column.clone_empty();
             MutableColumnPtr value_mutable_data = nested_values_column.clone_empty();
diff --git a/be/src/vec/functions/function_jsonb.cpp b/be/src/vec/functions/function_jsonb.cpp
@@ -727,8 +727,7 @@ class FunctionJsonbExtractPath : public IFunction {
                 VectorizedUtils::update_null_map(*result_null_map, *path_null_map, path_const);
             }
 
-            if (0 == simd::count_zero_num(reinterpret_cast<const int8_t*>(result_null_map->data()),
-                                          input_rows_count)) {
+            if (!simd::contain_zero(result_null_map->data(), input_rows_count)) {
                 return create_all_null_result();
             }
         }
diff --git a/be/test/util/simd/bits_test.cpp b/be/test/util/simd/bits_test.cpp
diff --git a/be/test/vec/columns/column_nullable_test.cpp b/be/test/vec/columns/column_nullable_test.cpp

Original file line number	Diff line number	Diff line change
`@@ -114,8 +114,8 @@ void ProcessHashTableProbe<JoinOpType>::build_side_output_column(vectorized::Mut`
`114`	`114`	`_build_column_has_null[i] = false;`
`115`	`115`	`if (_right_output_slot_flags[i] && column.is_nullable()) {`
`116`	`116`	`const auto& nullable = assert_cast<const vectorized::ColumnNullable&>(column);`
`117`		`- _build_column_has_null[i] = !simd::contain_byte(`
`118`		`- nullable.get_null_map_data().data() + 1, nullable.size() - 1, 1);`
	`117`	`+ _build_column_has_null[i] = !simd::contain_one(`
	`118`	`+ nullable.get_null_map_data().data() + 1, nullable.size() - 1);`
`119`	`119`	`}`
`120`	`120`	`}`
`121`	`121`	`}`
`@@ -368,8 +368,7 @@ Status ProcessHashTableProbe<JoinOpType>::finalize_block_with_filter(`
`368`	`368`	`}`
`369`	`369`	`const auto& column_filter =`
`370`	`370`	`assert_cast<const vectorized::ColumnUInt8*>(filter_ptr.get())->get_data();`
`371`		`- bool need_filter =`
`372`		`- simd::count_zero_num((int8_t*)column_filter.data(), column_filter.size()) != 0;`
	`371`	`+ bool need_filter = simd::contain_zero(column_filter.data(), column_filter.size());`
`373`	`372`	`if (need_filter) {`
`374`	`373`	`row_indexs.filter(column_filter);`
`375`	`374`	`}`
Original file line number	Diff line number	Diff line change
`@@ -97,9 +97,7 @@ class NestedLoopJoinProbeLocalState final`
`97`	`97`	`}`
`98`	`98`	`if (!_cur_probe_row_visited_flags[i]) {`
`99`	`99`	`_cur_probe_row_visited_flags[i] =`
`100`		`- simd::contain_byte<uint8_t>(filter.data() + offset, end - offset, 1)`
`101`		`- ? 1`
`102`		`- : 0;`
	`100`	`+ simd::contain_one(filter.data() + offset, end - offset);`
`103`	`101`	`}`
`104`	`102`	`end = offset;`
`105`	`103`	`}`
Original file line number	Diff line number	Diff line change
`@@ -280,7 +280,7 @@ size_t ColumnNullable::serialize_impl(char* pos, const size_t row) const {`
`280`	`280`	`}`
`281`	`281`
`282`	`282`	`void ColumnNullable::serialize(StringRef* keys, size_t num_rows) const {`
`283`		`- const bool has_null = simd::contain_byte(get_null_map_data().data(), num_rows, 1);`
	`283`	`+ const bool has_null = simd::contain_one(get_null_map_data().data(), num_rows);`
`284`	`284`	`const auto* __restrict null_map =`
`285`	`285`	`assert_cast<const ColumnUInt8&>(get_null_map_column()).get_data().data();`
`286`	`286`	`_nested_column->serialize_with_nullable(keys, num_rows, has_null, null_map);`
`@@ -598,11 +598,11 @@ void ColumnNullable::sort_column(const ColumnSorter* sorter, EqualFlags& flags,`
`598`	`598`	`}`
`599`	`599`
`600`	`600`	`bool ColumnNullable::only_null() const {`
`601`		`- return !simd::contain_byte(get_null_map_data().data(), size(), 0);`
	`601`	`+ return !simd::contain_zero(get_null_map_data().data(), size());`
`602`	`602`	`}`
`603`	`603`
`604`	`604`	`bool ColumnNullable::has_null(size_t begin, size_t end) const {`
`605`		`- return simd::contain_byte(get_null_map_data().data() + begin, end - begin, 1);`
	`605`	`+ return simd::contain_one(get_null_map_data().data() + begin, end - begin);`
`606`	`606`	`}`
`607`	`607`
`608`	`608`	`bool ColumnNullable::has_null() const {`
Original file line number	Diff line number	Diff line change
`@@ -80,6 +80,10 @@ inline HashKeyType get_hash_key_type_with_fixed(size_t size) {`
`80`	`80`	`}`
`81`	`81`
`82`	`82`	`inline HashKeyType get_hash_key_type_fixed(const std::vector<vectorized::DataTypePtr>& data_types) {`
	`83`	`+ if (data_types.size() >= vectorized::BITSIZE) {`
	`84`	`+ return HashKeyType::serialized;`
	`85`	`+ }`
	`86`	`+`
`83`	`87`	`bool has_null = false;`
`84`	`88`	`size_t key_byte_size = 0;`
`85`	`89`
`@@ -94,8 +98,7 @@ inline HashKeyType get_hash_key_type_fixed(const std::vector<vectorized::DataTyp`
`94`	`98`	`}`
`95`	`99`	`}`
`96`	`100`
`97`		`- size_t bitmap_size = has_null ? vectorized::get_bitmap_size(data_types.size()) : 0;`
`98`		`- return get_hash_key_type_with_fixed(bitmap_size + key_byte_size);`
	`101`	`+ return get_hash_key_type_with_fixed(has_null + key_byte_size);`
`99`	`102`	`}`
`100`	`103`
`101`	`104`	`inline HashKeyType get_hash_key_type(const std::vector<vectorized::DataTypePtr>& data_types) {`
Original file line number	Diff line number	Diff line change
`@@ -727,8 +727,7 @@ class FunctionJsonbExtractPath : public IFunction {`
`727`	`727`	`VectorizedUtils::update_null_map(result_null_map, path_null_map, path_const);`
`728`	`728`	`}`
`729`	`729`
`730`		`- if (0 == simd::count_zero_num(reinterpret_cast<const int8_t*>(result_null_map->data()),`
`731`		`- input_rows_count)) {`
	`730`	`+ if (!simd::contain_zero(result_null_map->data(), input_rows_count)) {`
`732`	`731`	`return create_all_null_result();`
`733`	`732`	`}`
`734`	`733`	`}`