Revert "[Chore](thirdparty) add crc32c-1.1.2 to thirdparty (#58462)"

BiteTheDDDDt · BiteTheDDDDt · commit 0902e17d7fb6 · 2026-01-08T23:06:38.000+08:00
This reverts commit 066b69e.
diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake
@@ -68,7 +68,6 @@ add_thirdparty(curl)
 add_thirdparty(lz4)
 add_thirdparty(thrift)
 add_thirdparty(thriftnb)
-add_thirdparty(crc32c)
 
 add_thirdparty(libevent_core LIBNAME "lib/libevent_core.a")
 add_thirdparty(libevent_openssl LIBNAME "lib/libevent_openssl.a")
diff --git a/be/src/exprs/block_bloom_filter.hpp b/be/src/exprs/block_bloom_filter.hpp
@@ -76,7 +76,7 @@ class BlockBloomFilter {
     // Same as above with convenience of hashing the key.
     void insert(const StringRef& key) noexcept {
         if (key.data) {
-            insert(HashUtil::crc32c_hash(key.data, uint32_t(key.size), _hash_seed));
+            insert(HashUtil::crc_hash(key.data, uint32_t(key.size), _hash_seed));
         }
     }
 
@@ -105,7 +105,7 @@ class BlockBloomFilter {
     // Same as above with convenience of hashing the key.
     bool find(const StringRef& key) const noexcept {
         if (key.data) {
-            return find(HashUtil::crc32c_hash(key.data, uint32_t(key.size), _hash_seed));
+            return find(HashUtil::crc_hash(key.data, uint32_t(key.size), _hash_seed));
         }
         return false;
     }
diff --git a/be/src/pipeline/dependency.h b/be/src/pipeline/dependency.h
@@ -714,7 +714,7 @@ struct SetSharedState : public BasicSharedState {
 
 enum class ExchangeType : uint8_t {
     NOOP = 0,
-    // Shuffle data by Crc32CHashPartitioner
+    // Shuffle data by Crc32HashPartitioner<LocalExchangeChannelIds>.
     HASH_SHUFFLE = 1,
     // Round-robin passthrough data blocks.
     PASSTHROUGH = 2,
diff --git a/be/src/pipeline/exec/exchange_sink_operator.cpp b/be/src/pipeline/exec/exchange_sink_operator.cpp
@@ -119,11 +119,13 @@ Status ExchangeSinkLocalState::init(RuntimeState* state, LocalSinkStateInfo& inf
 
     if (_part_type == TPartitionType::HASH_PARTITIONED) {
         _partition_count = channels.size();
-        _partitioner = std::make_unique<vectorized::Crc32CHashPartitioner>(channels.size());
+        _partitioner =
+                std::make_unique<vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>>(
+                        channels.size());
         RETURN_IF_ERROR(_partitioner->init(p._texprs));
         RETURN_IF_ERROR(_partitioner->prepare(state, p._row_desc));
         custom_profile()->add_info_string(
-                "Partitioner", fmt::format("Crc32CHashPartitioner({})", _partition_count));
+                "Partitioner", fmt::format("Crc32HashPartitioner({})", _partition_count));
     } else if (_part_type == TPartitionType::BUCKET_SHFFULE_HASH_PARTITIONED) {
         _partition_count = channels.size();
         _partitioner =
diff --git a/be/src/pipeline/local_exchange/local_exchange_sink_operator.cpp b/be/src/pipeline/local_exchange/local_exchange_sink_operator.cpp
@@ -54,7 +54,8 @@ Status LocalExchangeSinkOperatorX::init(ExchangeType type, const int num_buckets
                 _shuffle_idx_to_instance_idx[i] = i;
             }
         }
-        _partitioner.reset(new vectorized::Crc32CHashPartitioner(_num_partitions));
+        _partitioner.reset(new vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>(
+                _num_partitions));
         RETURN_IF_ERROR(_partitioner->init(_texprs));
     } else if (_type == ExchangeType::BUCKET_HASH_SHUFFLE) {
         DCHECK_GT(num_buckets, 0);
diff --git a/be/src/util/crc32c.cpp b/be/src/util/crc32c.cpp
diff --git a/be/src/util/crc32c.h b/be/src/util/crc32c.h
@@ -15,22 +15,39 @@
 // specific language governing permissions and limitations
 // under the License.
 
+// the following code are modified from RocksDB:
+// https://github.com/facebook/rocksdb/blob/master/util/crc32c.h
+
 #pragma once
 
+#include <stddef.h>
+#include <stdint.h>
+
 #include <vector>
 
 #include "util/slice.h"
 
 namespace doris {
 namespace crc32c {
 
-uint32_t Extend(uint32_t crc, const char* data, size_t n);
+// Return the crc32c of concat(A, data[0,n-1]) where init_crc is the
+// crc32c of some string A.  Extend() is often used to maintain the
+// crc32c of a stream of data.
+extern uint32_t Extend(uint32_t init_crc, const char* data, size_t n);
 
 // Return the crc32c of data[0,n-1]
-uint32_t Value(const char* data, size_t n);
+inline uint32_t Value(const char* data, size_t n) {
+    return Extend(0, data, n);
+}
 
 // Return the crc32c of data content in all slices
-uint32_t Value(const std::vector<Slice>& slices);
+inline uint32_t Value(const std::vector<Slice>& slices) {
+    uint32_t crc = 0;
+    for (auto& slice : slices) {
+        crc = Extend(crc, slice.get_data(), slice.get_size());
+    }
+    return crc;
+}
 
 } // namespace crc32c
 } // namespace doris
diff --git a/be/src/util/hash_util.hpp b/be/src/util/hash_util.hpp
@@ -29,7 +29,6 @@
 
 #include "common/compiler_util.h" // IWYU pragma: keep
 #include "util/cpu_info.h"
-#include "util/crc32c.h"
 #include "util/hash/city.h"
 #include "util/murmur_hash3.h"
 #include "util/sse_util.hpp"
@@ -49,12 +48,7 @@ class HashUtil {
         return (uint32_t)crc32(hash, (const unsigned char*)(&INT_VALUE), 4);
     }
 
-    // ATTN: crc32c's result is different with zlib_crc32 coz of different polynomial
-    // crc32c have better performance than zlib_crc32/crc_hash
-    static uint32_t crc32c_hash(const void* data, uint32_t bytes, uint32_t hash) {
-        return crc32c::Extend(hash, static_cast<const char*>(data), bytes);
-    }
-
+#if defined(__SSE4_2__) || defined(__aarch64__)
     // Compute the Crc32 hash for data using SSE4 instructions.  The input hash parameter is
     // the current hash/seed value.
     // This should only be called if SSE is supported.
@@ -64,8 +58,6 @@ class HashUtil {
     // NOTE: Any changes made to this function need to be reflected in Codegen::GetHashFn.
     // TODO: crc32 hashes with different seeds do not result in different hash functions.
     // The resulting hashes are correlated.
-    // ATTN: prefer do not use this function anymore, use crc32c_hash instead
-    // This function is retained because it is not certain whether there are compatibility issues with historical data.
     static uint32_t crc_hash(const void* data, uint32_t bytes, uint32_t hash) {
         if (!CpuInfo::is_supported(CpuInfo::SSE4_2)) {
             return zlib_crc_hash(data, bytes, hash);
@@ -124,6 +116,11 @@ class HashUtil {
 
         return converter.u64;
     }
+#else
+    static uint32_t crc_hash(const void* data, uint32_t bytes, uint32_t hash) {
+        return zlib_crc_hash(data, bytes, hash);
+    }
+#endif
 
     // refer to https://github.com/apache/commons-codec/blob/master/src/main/java/org/apache/commons/codec/digest/MurmurHash3.java
     static const uint32_t MURMUR3_32_SEED = 104729;
diff --git a/be/src/vec/columns/column.h b/be/src/vec/columns/column.h
@@ -401,14 +401,6 @@ class IColumn : public COW<IColumn> {
                                "Method update_crcs_with_value is not supported for " + get_name());
     }
 
-    virtual void update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows,
-                                           uint32_t offset,
-                                           const uint8_t* __restrict null_data = nullptr) const {
-        throw doris::Exception(
-                ErrorCode::NOT_IMPLEMENTED_ERROR,
-                "Method update_crc32cs_with_value is not supported for " + get_name());
-    }
-
     // use range for one hash value to avoid virtual function call in loop
     virtual void update_crc_with_value(size_t start, size_t end, uint32_t& hash,
                                        const uint8_t* __restrict null_data) const {
diff --git a/be/src/vec/columns/column_decimal.cpp b/be/src/vec/columns/column_decimal.cpp
@@ -20,7 +20,6 @@
 
 #include "vec/columns/column_decimal.h"
 
-#include <crc32c/crc32c.h>
 #include <fmt/format.h>
 
 #include <limits>
@@ -209,19 +208,6 @@ void ColumnDecimal<T>::update_crcs_with_value(uint32_t* __restrict hashes, Primi
     }
 }
 
-template <PrimitiveType T>
-void ColumnDecimal<T>::update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows,
-                                                 uint32_t offset,
-                                                 const uint8_t* __restrict null_data) const {
-    auto s = rows;
-    DCHECK(s == size());
-
-    for (size_t i = 0; i < s; i++) {
-        hashes[i] = crc32c_extend(hashes[i], (const uint8_t*)&data[i],
-                                  sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType));
-    }
-}
-
 template <PrimitiveType T>
 void ColumnDecimal<T>::update_xxHash_with_value(size_t start, size_t end, uint64_t& hash,
                                                 const uint8_t* __restrict null_data) const {
diff --git a/be/src/vec/columns/column_decimal.h b/be/src/vec/columns/column_decimal.h
@@ -173,9 +173,6 @@ class ColumnDecimal final : public COWHelper<IColumn, ColumnDecimal<T>> {
                                 uint32_t offset,
                                 const uint8_t* __restrict null_data) const override;
 
-    void update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows, uint32_t offset,
-                                   const uint8_t* __restrict null_data = nullptr) const override;
-
     void update_xxHash_with_value(size_t start, size_t end, uint64_t& hash,
                                   const uint8_t* __restrict null_data) const override;
     void update_crc_with_value(size_t start, size_t end, uint32_t& hash,
diff --git a/be/src/vec/columns/column_dictionary.h b/be/src/vec/columns/column_dictionary.h
@@ -345,7 +345,7 @@ class ColumnDictI32 final : public COWHelper<IColumn, ColumnDictI32> {
                 if (type == FieldType::OLAP_FIELD_TYPE_CHAR) {
                     len = strnlen(sv.data, sv.size);
                 }
-                uint32_t hash_val = HashUtil::crc32c_hash(sv.data, static_cast<uint32_t>(len), 0);
+                uint32_t hash_val = HashUtil::crc_hash(sv.data, static_cast<uint32_t>(len), 0);
                 _hash_values[code] = hash_val;
                 _compute_hash_value_flags[code] = 1;
                 return _hash_values[code];
diff --git a/be/src/vec/columns/column_nullable.cpp b/be/src/vec/columns/column_nullable.cpp
@@ -114,23 +114,6 @@ void ColumnNullable::update_crcs_with_value(uint32_t* __restrict hashes, doris::
     }
 }
 
-void ColumnNullable::update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows,
-                                               uint32_t offset,
-                                               const uint8_t* __restrict null_data) const {
-    DCHECK(null_data == nullptr);
-    auto s = rows;
-    DCHECK(s == size());
-    const auto* __restrict real_null_data =
-            assert_cast<const ColumnUInt8&>(get_null_map_column()).get_data().data();
-    if (!has_null()) {
-        _nested_column->update_crc32cs_with_value(hashes, rows, offset, nullptr);
-    } else {
-        // nullmap process is slow, replace null data to zero to avoid nullmap process
-        _nested_column->assume_mutable()->replace_column_null_data(real_null_data);
-        _nested_column->update_crc32cs_with_value(hashes, rows, offset, nullptr);
-    }
-}
-
 void ColumnNullable::update_hashes_with_value(uint64_t* __restrict hashes,
                                               const uint8_t* __restrict null_data) const {
     DCHECK(null_data == nullptr);
diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h
@@ -229,8 +229,6 @@ class ColumnNullable final : public COWHelper<IColumn, ColumnNullable> {
     void update_crcs_with_value(uint32_t* __restrict hash, PrimitiveType type, uint32_t rows,
                                 uint32_t offset,
                                 const uint8_t* __restrict null_data) const override;
-    void update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows, uint32_t offset,
-                                   const uint8_t* __restrict null_data = nullptr) const override;
     void update_hashes_with_value(uint64_t* __restrict hashes,
                                   const uint8_t* __restrict null_data) const override;
 
diff --git a/be/src/vec/columns/column_string.cpp b/be/src/vec/columns/column_string.cpp
@@ -20,8 +20,6 @@
 
 #include "vec/columns/column_string.h"
 
-#include <crc32c/crc32c.h>
-
 #include <algorithm>
 #include <boost/iterator/iterator_facade.hpp>
 #include <cstring>
@@ -320,19 +318,6 @@ void ColumnStr<T>::update_crcs_with_value(uint32_t* __restrict hashes, doris::Pr
     }
 }
 
-template <typename T>
-void ColumnStr<T>::update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows,
-                                             uint32_t offset,
-                                             const uint8_t* __restrict null_data) const {
-    auto s = rows;
-    DCHECK(s == size());
-
-    for (size_t i = 0; i < s; i++) {
-        auto data_ref = get_data_at(i);
-        hashes[i] = crc32c_extend(hashes[i], (const uint8_t*)(data_ref.data), data_ref.size);
-    }
-}
-
 template <typename T>
 ColumnPtr ColumnStr<T>::filter(const IColumn::Filter& filt, ssize_t result_size_hint) const {
     if constexpr (std::is_same_v<UInt32, T>) {
diff --git a/be/src/vec/columns/column_string.h b/be/src/vec/columns/column_string.h
@@ -427,9 +427,6 @@ class ColumnStr final : public COWHelper<IColumn, ColumnStr<T>> {
                                 uint32_t offset,
                                 const uint8_t* __restrict null_data) const override;
 
-    void update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows, uint32_t offset,
-                                   const uint8_t* __restrict null_data = nullptr) const override;
-
     void update_hashes_with_value(uint64_t* __restrict hashes,
                                   const uint8_t* __restrict null_data) const override {
         auto s = size();
diff --git a/be/src/vec/columns/column_vector.cpp b/be/src/vec/columns/column_vector.cpp
@@ -20,7 +20,6 @@
 
 #include "vec/columns/column_vector.h"
 
-#include <crc32c/crc32c.h>
 #include <fmt/format.h>
 #include <glog/logging.h>
 #include <pdqsort.h>
@@ -237,43 +236,6 @@ void ColumnVector<T>::update_crcs_with_value(uint32_t* __restrict hashes, Primit
     }
 }
 
-template <PrimitiveType T>
-void ColumnVector<T>::update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows,
-                                                uint32_t offset,
-                                                const uint8_t* __restrict null_data) const {
-    auto s = rows;
-    DCHECK(s == size());
-
-    if constexpr (is_date_or_datetime(T)) {
-        char buf[64];
-        auto date_convert_do_crc = [&](size_t i) {
-            const auto& date_val = (const VecDateTimeValue&)data[i];
-            auto len = date_val.to_buffer(buf);
-            hashes[i] = crc32c_extend(hashes[i], (const uint8_t*)buf, len);
-        };
-
-        for (size_t i = 0; i < s; i++) {
-            date_convert_do_crc(i);
-        }
-    } else {
-        for (size_t i = 0; i < s; i++) {
-            if constexpr (sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) == 1) {
-                hashes[i] = _mm_crc32_u8(hashes[i], *reinterpret_cast<const uint8_t*>(&data[i]));
-            } else if constexpr (sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) == 2) {
-                hashes[i] = _mm_crc32_u16(hashes[i], *reinterpret_cast<const uint16_t*>(&data[i]));
-            } else if constexpr (sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) == 4) {
-                hashes[i] = _mm_crc32_u32(hashes[i], *reinterpret_cast<const uint32_t*>(&data[i]));
-            } else if constexpr (sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType) == 8) {
-                hashes[i] = (uint32_t)_mm_crc32_u64(hashes[i],
-                                                    *reinterpret_cast<const uint64_t*>(&data[i]));
-            } else {
-                hashes[i] = crc32c_extend(hashes[i], (const uint8_t*)&data[i],
-                                          sizeof(typename PrimitiveTypeTraits<T>::ColumnItemType));
-            }
-        }
-    }
-}
-
 template <PrimitiveType T>
 struct ColumnVector<T>::less {
     const Self& parent;
diff --git a/be/src/vec/columns/column_vector.h b/be/src/vec/columns/column_vector.h
@@ -239,9 +239,6 @@ class ColumnVector final : public COWHelper<IColumn, ColumnVector<T>> {
                                 uint32_t offset,
                                 const uint8_t* __restrict null_data) const override;
 
-    void update_crc32cs_with_value(uint32_t* __restrict hashes, uint32_t rows, uint32_t offset,
-                                   const uint8_t* __restrict null_data = nullptr) const override;
-
     void update_hashes_with_value(uint64_t* __restrict hashes,
                                   const uint8_t* __restrict null_data) const override;
 
diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h
@@ -4598,7 +4598,7 @@ class FunctionNgramSearch : public IFunction {
 
     uint32_t sub_str_hash(const char* data, int32_t length) const {
         constexpr static uint32_t seed = 0;
-        return HashUtil::crc32c_hash(data, length, seed);
+        return HashUtil::crc_hash(data, length, seed);
     }
 
     template <bool column_const>
diff --git a/be/src/vec/runtime/partitioner.cpp b/be/src/vec/runtime/partitioner.cpp
diff --git a/be/src/vec/runtime/partitioner.h b/be/src/vec/runtime/partitioner.h
diff --git a/thirdparty/CHANGELOG.md b/thirdparty/CHANGELOG.md
diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ class BlockBloomFilter {`
`76`	`76`	`// Same as above with convenience of hashing the key.`
`77`	`77`	`void insert(const StringRef& key) noexcept {`
`78`	`78`	`if (key.data) {`
`79`		`- insert(HashUtil::crc32c_hash(key.data, uint32_t(key.size), _hash_seed));`
	`79`	`+ insert(HashUtil::crc_hash(key.data, uint32_t(key.size), _hash_seed));`
`80`	`80`	`}`
`81`	`81`	`}`
`82`	`82`
`@@ -105,7 +105,7 @@ class BlockBloomFilter {`
`105`	`105`	`// Same as above with convenience of hashing the key.`
`106`	`106`	`bool find(const StringRef& key) const noexcept {`
`107`	`107`	`if (key.data) {`
`108`		`- return find(HashUtil::crc32c_hash(key.data, uint32_t(key.size), _hash_seed));`
	`108`	`+ return find(HashUtil::crc_hash(key.data, uint32_t(key.size), _hash_seed));`
`109`	`109`	`}`
`110`	`110`	`return false;`
`111`	`111`	`}`
Original file line number	Diff line number	Diff line change
`@@ -54,7 +54,8 @@ Status LocalExchangeSinkOperatorX::init(ExchangeType type, const int num_buckets`
`54`	`54`	`_shuffle_idx_to_instance_idx[i] = i;`
`55`	`55`	`}`
`56`	`56`	`}`
`57`		`- _partitioner.reset(new vectorized::Crc32CHashPartitioner(_num_partitions));`
	`57`	`+ _partitioner.reset(new vectorized::Crc32HashPartitioner<vectorized::ShuffleChannelIds>(`
	`58`	`+ _num_partitions));`
`58`	`59`	`RETURN_IF_ERROR(_partitioner->init(_texprs));`
`59`	`60`	`} else if (_type == ExchangeType::BUCKET_HASH_SHUFFLE) {`
`60`	`61`	`DCHECK_GT(num_buckets, 0);`