feat: add support for BLAKE3-256 hash

mhx · mhx · commit 9adf667486a8 · 2025-10-30T19:04:13.000+01:00
diff --git a/.docker/Dockerfile.arch b/.docker/Dockerfile.arch
@@ -1,12 +1,11 @@
 # syntax=docker/dockerfile:1
 FROM archlinux:latest
-RUN pacman -Syu --noconfirm
-RUN pacman -S --noconfirm --needed \
+RUN pacman -Syu --noconfirm && \
+    pacman -S --noconfirm --needed \
         bash-completion \
         vim \
         devtools \
-        less
-RUN pacman -S --noconfirm --needed \
+        less \
         gcc \
         clang \
         man \
@@ -26,6 +25,7 @@ RUN pacman -S --noconfirm --needed \
         pkg-config \
         binutils \
         libarchive \
+        libblake3 \
         benchmark \
         boost \
         brotli \
diff --git a/.docker/Dockerfile.debian b/.docker/Dockerfile.debian
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1
 ARG ARCH=amd64
 FROM $ARCH/debian:testing
-RUN apt update && apt upgrade -y
-RUN apt install -y \
+RUN apt update && apt upgrade -y && \
+    apt install -y \
         bash-completion \
         wget \
         curl \
diff --git a/.docker/Dockerfile.fedora b/.docker/Dockerfile.fedora
@@ -1,12 +1,11 @@
 # syntax=docker/dockerfile:1
 ARG ARCH=amd64
 FROM $ARCH/fedora:rawhide
-RUN dnf update -y
-RUN dnf install -y \
+RUN dnf update -y && \
+    dnf install -y \
         bash-completion \
         vim \
-        less
-RUN dnf install -y \
+        less \
         gcc \
         g++ \
         clang \
@@ -27,6 +26,7 @@ RUN dnf install -y \
         pkg-config \
         binutils-devel \
         libarchive-devel \
+        blake3-devel \
         google-benchmark \
         boost-devel \
         brotli-devel \
diff --git a/.docker/Dockerfile.suse b/.docker/Dockerfile.suse
@@ -2,16 +2,11 @@
 FROM opensuse/tumbleweed
 
 # Refresh & update the base. For Tumbleweed, a normal update is fine in a fresh image.
-RUN zypper -n refresh && zypper -n update
-
-# Nice-to-haves
-RUN zypper -n in --no-recommends \
+RUN zypper -n refresh && zypper -n update &&
+    zypper -n in --no-recommends \
     bash-completion \
     vim \
-    less
-
-# Build toolchain & essentials
-RUN zypper -n in --no-recommends \
+    less \
     mold \
     gcc \
     gcc-c++ \
@@ -33,6 +28,7 @@ RUN zypper -n in --no-recommends \
     pkgconf-pkg-config \
     binutils-devel \
     libarchive-devel \
+    blake3-devel \
     benchmark-devel \
     boost-devel \
     libboost_chrono-devel \
diff --git a/.docker/check_lib_versions.py b/.docker/check_lib_versions.py
@@ -8,6 +8,7 @@
     "xiph/flac",
     "libunwind/libunwind",
     "google/benchmark",
+    "BLAKE3-team/BLAKE3",
     "boostorg/boost",
     "openssl/openssl",
     "libressl/portable",
diff --git a/.docker/static-libs-fetch.sh b/.docker/static-libs-fetch.sh
@@ -22,6 +22,7 @@ fetch_lib flac https://github.com/xiph/flac/releases/download/${FLAC_VERSION}/${
 fetch_lib libucontext https://github.com/kaniini/libucontext/archive/refs/tags/${LIBUCONTEXT_TARBALL}
 fetch_lib libunwind https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/${LIBUNWIND_TARBALL}
 fetch_lib benchmark https://github.com/google/benchmark/archive/refs/tags/v${BENCHMARK_VERSION}.tar.gz ${BENCHMARK_TARBALL}
+fetch_lib blake3 https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/${BLAKE3_VERSION}.tar.gz ${BLAKE3_TARBALL}
 fetch_lib openssl https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/${OPENSSL_TARBALL}
 fetch_lib libressl https://github.com/libressl/portable/releases/download/v${LIBRESSL_VERSION}/${LIBRESSL_TARBALL}
 fetch_lib cpptrace https://github.com/jeremy-rifkin/cpptrace/archive/refs/tags/v${CPPTRACE_VERSION}.tar.gz ${CPPTRACE_TARBALL}
diff --git a/.docker/static-libs-install.sh b/.docker/static-libs-install.sh
@@ -18,7 +18,7 @@ if [[ "$PKGS" == ":none" ]]; then
     echo "No libraries to build"
     exit 0
 elif [[ "$PKGS" == ":all" ]]; then
-    PKGS="benchmark,boost,brotli,cpptrace,double-conversion,flac,fmt,fuse,fuse3,glog,jemalloc,libarchive,libdwarf,libevent,libucontext,libunwind,libressl,lz4,mimalloc,nlohmann,openssl,parallel-hashmap,range-v3,utfcpp,xxhash,xz,zstd"
+    PKGS="benchmark,blake3,boost,brotli,cpptrace,double-conversion,flac,fmt,fuse,fuse3,glog,jemalloc,libarchive,libdwarf,libevent,libucontext,libunwind,libressl,lz4,mimalloc,nlohmann,openssl,parallel-hashmap,range-v3,utfcpp,xxhash,xz,zstd"
 fi
 
 export COMMON_CFLAGS="-ffunction-sections -fdata-sections -fmerge-all-constants"
@@ -270,6 +270,18 @@ if use_lib nlohmann; then
     cp "$HOME/pkgs/json.hpp" "$INSTALL_DIR/include/nlohmann/json.hpp"
 fi
 
+if use_lib blake3; then
+    opt_size
+    cd "$WORKDIR"
+    tar xf ${WORKROOT}/${BLAKE3_TARBALL}
+    cd BLAKE3-${BLAKE3_VERSION}/c
+    mkdir build
+    cd build
+    cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" ${CMAKE_ARGS}
+    $NINJA_PARALLEL
+    ninja install
+fi
+
 if use_lib utfcpp; then
     opt_size
     cd "$WORKDIR"
diff --git a/.docker/static-libs-versions.sh b/.docker/static-libs-versions.sh
@@ -7,6 +7,7 @@ FLAC_VERSION=1.5.0                       # 2025-02-11
 LIBUCONTEXT_VERSION=1.3.2                # 2024-10-07
 LIBUNWIND_VERSION=1.8.3                  # 2025-09-04
 BENCHMARK_VERSION=1.9.4                  # 2025-05-19
+BLAKE3_VERSION=1.8.2                     # 2025-04-21
 BOOST_VERSION=1.89.0                     # 2025-08-14
 OPENSSL_VERSION=3.6.0                    # 2025-10-01
 LIBRESSL_VERSION=4.2.1                   # 2025-10-30
@@ -38,6 +39,7 @@ FLAC_TARBALL="flac-${FLAC_VERSION}.tar.xz"
 LIBUCONTEXT_TARBALL="libucontext-${LIBUCONTEXT_VERSION}.tar.gz"
 LIBUNWIND_TARBALL="libunwind-${LIBUNWIND_VERSION}.tar.gz"
 BENCHMARK_TARBALL="benchmark-${BENCHMARK_VERSION}.tar.gz"
+BLAKE3_TARBALL="BLAKE3-${BLAKE3_VERSION}.tar.gz"
 OPENSSL_TARBALL="openssl-${OPENSSL_VERSION}.tar.gz"
 LIBRESSL_TARBALL="libressl-${LIBRESSL_VERSION}.tar.gz"
 CPPTRACE_TARBALL="cpptrace-${CPPTRACE_VERSION}.tar.gz"
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -109,6 +109,9 @@ set(RANGE_V3_PREFERRED_VERSION 0.12.0)
 set(PARALLEL_HASHMAP_REQUIRED_VERSION 1.3.8)
 set(PARALLEL_HASHMAP_PREFERRED_VERSION 2.0.0)
 
+set(BLAKE3_REQUIRED_VERSION 1.2.0)
+set(BLAKE3_PREFERRED_VERSION 1.8.2)
+
 set(BOOST_REQUIRED_VERSION 1.67.0)
 set(LIBCRYPTO_REQUIRED_VERSION 3.0.0)
 set(LIBLZ4_REQUIRED_VERSION 1.9.3)
@@ -127,11 +130,13 @@ if(DEFINED ENV{DWARFS_LOCAL_REPO_PATH})
   set(GOOGLETEST_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/googletest)
   set(RANGE_V3_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/range-v3)
   set(PARALLEL_HASHMAP_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/parallel-hashmap)
+  set(BLAKE3_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/BLAKE3)
 else()
   set(LIBFMT_GIT_REPO https://github.com/fmtlib/fmt.git)
   set(GOOGLETEST_GIT_REPO https://github.com/google/googletest.git)
   set(RANGE_V3_GIT_REPO https://github.com/ericniebler/range-v3.git)
   set(PARALLEL_HASHMAP_GIT_REPO https://github.com/greg7mdp/parallel-hashmap.git)
+  set(BLAKE3_GIT_REPO https://github.com/BLAKE3-team/BLAKE3.git)
 endif()
 
 if(APPLE)
@@ -194,6 +199,7 @@ if(WITH_TESTS)
   include(${CMAKE_SOURCE_DIR}/cmake/need_gtest.cmake)
 endif()
 
+include(${CMAKE_SOURCE_DIR}/cmake/need_blake3.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/need_fmt.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/need_range_v3.cmake)
 
diff --git a/cmake/libdwarfs.cmake b/cmake/libdwarfs.cmake
@@ -208,7 +208,7 @@ add_cpp2_thrift_library(thrift/history.thrift
 add_cpp2_thrift_library(thrift/features.thrift
                         TARGET dwarfs_features_thrift OUTPUT_PATH dwarfs)
 
-target_link_libraries(dwarfs_common PRIVATE dwarfs_folly_lite PkgConfig::LIBCRYPTO PkgConfig::XXHASH PkgConfig::ZSTD)
+target_link_libraries(dwarfs_common PRIVATE dwarfs_folly_lite PkgConfig::LIBCRYPTO PkgConfig::XXHASH PkgConfig::ZSTD BLAKE3::blake3)
 target_link_libraries(dwarfs_compressor PRIVATE dwarfs_common)
 target_link_libraries(dwarfs_decompressor PRIVATE dwarfs_common)
 target_link_libraries(dwarfs_reader PUBLIC dwarfs_common dwarfs_decompressor)
diff --git a/cmake/need_blake3.cmake b/cmake/need_blake3.cmake
@@ -0,0 +1,33 @@
+#
+# Copyright (c) Marcus Holland-Moritz
+#
+# This file is part of dwarfs.
+#
+# dwarfs is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# dwarfs is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+cmake_minimum_required(VERSION 3.28.0)
+
+find_package(blake3 ${BLAKE3_REQUIRED_VERSION} CONFIG)
+
+if(NOT blake3_FOUND)
+  FetchContent_Declare(
+    blake3
+    GIT_REPOSITORY ${BLAKE3_GIT_REPO}
+    GIT_TAG ${BLAKE3_PREFERRED_VERSION}
+    SOURCE_SUBDIR c
+    EXCLUDE_FROM_ALL
+    SYSTEM
+  )
+  FetchContent_MakeAvailable(blake3)
+endif()
diff --git a/include/dwarfs/checksum.h b/include/dwarfs/checksum.h
@@ -47,19 +47,24 @@ class checksum {
 
   struct xxh3_64_tag {};
   struct sha2_512_256_tag {};
+  struct blake3_256_tag {};
 
   static constexpr xxh3_64_tag xxh3_64{};
   static constexpr sha2_512_256_tag sha2_512_256{};
+  static constexpr blake3_256_tag blake3_256{};
 
   static bool verify(xxh3_64_tag, void const* data, size_t size,
                      void const* digest, size_t digest_size);
   static bool verify(sha2_512_256_tag, void const* data, size_t size,
                      void const* digest, size_t digest_size);
+  static bool verify(blake3_256_tag, void const* data, size_t size,
+                     void const* digest, size_t digest_size);
   static bool verify(std::string const& alg, void const* data, size_t size,
                      void const* digest, size_t digest_size);
 
   checksum(xxh3_64_tag);
   checksum(sha2_512_256_tag);
+  checksum(blake3_256_tag);
   checksum(std::string const& alg);
 
   checksum& update(void const* data, size_t size) {
diff --git a/src/checksum.cpp b/src/checksum.cpp
@@ -31,6 +31,7 @@
 #include <cassert>
 #include <cstring>
 #include <functional>
+#include <optional>
 #include <ostream>
 #include <ranges>
 #include <string_view>
@@ -39,6 +40,8 @@
 
 #include <xxhash.h>
 
+#include <blake3.h>
+
 #include <boost/algorithm/hex.hpp>
 
 #include <fmt/format.h>
@@ -55,6 +58,7 @@ using namespace std::string_view_literals;
 constexpr std::array supported_algorithms{
     "xxh3-64"sv,
     "xxh3-128"sv,
+    "blake3-256"sv,
 };
 
 constexpr std::array unsupported_algorithms{
@@ -208,8 +212,39 @@ class checksum_xxh3 : public checksum::impl {
   std::unique_ptr<XXH3_state_t, decltype(&XXH3_freeState)> state_;
 };
 
+template <size_t DigestBytes>
+class checksum_blake3 : public checksum::impl {
+ public:
+  explicit checksum_blake3() {
+    hasher_.emplace();
+    ::blake3_hasher_init(&hasher_.value());
+  }
+
+  void update(void const* data, size_t size) override {
+    ::blake3_hasher_update(&hasher_.value(), data, size);
+  }
+
+  bool finalize(void* digest) override {
+    if (!hasher_.has_value()) {
+      return false;
+    }
+    ::blake3_hasher_finalize(&hasher_.value(),
+                             reinterpret_cast<uint8_t*>(digest), DigestBytes);
+    hasher_.reset();
+    return true;
+  }
+
+  std::string hexdigest() override { return make_hexdigest(*this); }
+
+  size_t digest_size() override { return DigestBytes; }
+
+ private:
+  std::optional<blake3_hasher> hasher_;
+};
+
 using checksum_xxh3_64 = checksum_xxh3<xxh3_64_policy>;
 using checksum_xxh3_128 = checksum_xxh3<xxh3_128_policy>;
+using checksum_blake3_256 = checksum_blake3<256 / 8>;
 
 template <typename T>
 bool verify_impl(T&& alg, void const* data, size_t size, void const* digest,
@@ -251,6 +286,11 @@ bool checksum::verify(sha2_512_256_tag, void const* data, size_t size,
   return verify_impl(sha2_512_256, data, size, digest, digest_size);
 }
 
+bool checksum::verify(blake3_256_tag, void const* data, size_t size,
+                      void const* digest, size_t digest_size) {
+  return verify_impl(blake3_256, data, size, digest, digest_size);
+}
+
 bool checksum::verify(std::string const& alg, void const* data, size_t size,
                       void const* digest, size_t digest_size) {
   return verify_impl(alg, data, size, digest, digest_size);
@@ -262,11 +302,16 @@ checksum::checksum(xxh3_64_tag)
 checksum::checksum(sha2_512_256_tag)
     : impl_(std::make_unique<checksum_evp>(::EVP_sha512_256())) {}
 
+checksum::checksum(blake3_256_tag)
+    : impl_(std::make_unique<checksum_blake3_256>()) {}
+
 checksum::checksum(std::string const& alg) {
   if (alg == "xxh3-64") {
     impl_ = std::make_unique<checksum_xxh3_64>();
   } else if (alg == "xxh3-128") {
     impl_ = std::make_unique<checksum_xxh3_128>();
+  } else if (alg == "blake3-256") {
+    impl_ = std::make_unique<checksum_blake3_256>();
   } else if (auto md = ::EVP_get_digestbyname(alg.c_str())) {
     impl_ = std::make_unique<checksum_evp>(md);
   } else {
diff --git a/test/checksum_test.cpp b/test/checksum_test.cpp
diff --git a/vcpkg.json b/vcpkg.json
