feat: add support for BLAKE3-256 hash

mhx · mhx · commit 2cd327bfb9de · 2025-10-30T20:33:41.000+01:00
diff --git a/.docker/Dockerfile.arch b/.docker/Dockerfile.arch
@@ -1,12 +1,11 @@
 # syntax=docker/dockerfile:1
 FROM archlinux:latest
-RUN pacman -Syu --noconfirm
-RUN pacman -S --noconfirm --needed \
+RUN pacman -Syu --noconfirm && \
+    pacman -S --noconfirm --needed \
         bash-completion \
         vim \
         devtools \
-        less
-RUN pacman -S --noconfirm --needed \
+        less \
         gcc \
         clang \
         man \
@@ -26,6 +25,7 @@ RUN pacman -S --noconfirm --needed \
         pkg-config \
         binutils \
         libarchive \
+        libblake3 \
         benchmark \
         boost \
         brotli \
diff --git a/.docker/Dockerfile.debian b/.docker/Dockerfile.debian
@@ -1,8 +1,8 @@
 # syntax=docker/dockerfile:1
 ARG ARCH=amd64
 FROM $ARCH/debian:testing
-RUN apt update && apt upgrade -y
-RUN apt install -y \
+RUN apt update && apt upgrade -y && \
+    apt install -y \
         bash-completion \
         wget \
         curl \
diff --git a/.docker/Dockerfile.fedora b/.docker/Dockerfile.fedora
@@ -1,12 +1,11 @@
 # syntax=docker/dockerfile:1
 ARG ARCH=amd64
 FROM $ARCH/fedora:rawhide
-RUN dnf update -y
-RUN dnf install -y \
+RUN dnf update -y && \
+    dnf install -y \
         bash-completion \
         vim \
-        less
-RUN dnf install -y \
+        less \
         gcc \
         g++ \
         clang \
@@ -27,6 +26,7 @@ RUN dnf install -y \
         pkg-config \
         binutils-devel \
         libarchive-devel \
+        blake3-devel \
         google-benchmark \
         boost-devel \
         brotli-devel \
diff --git a/.docker/Dockerfile.suse b/.docker/Dockerfile.suse
@@ -2,16 +2,11 @@
 FROM opensuse/tumbleweed
 
 # Refresh & update the base. For Tumbleweed, a normal update is fine in a fresh image.
-RUN zypper -n refresh && zypper -n update
-
-# Nice-to-haves
-RUN zypper -n in --no-recommends \
+RUN zypper -n refresh && zypper -n update && \
+    zypper -n in --no-recommends \
     bash-completion \
     vim \
-    less
-
-# Build toolchain & essentials
-RUN zypper -n in --no-recommends \
+    less \
     mold \
     gcc \
     gcc-c++ \
@@ -33,6 +28,7 @@ RUN zypper -n in --no-recommends \
     pkgconf-pkg-config \
     binutils-devel \
     libarchive-devel \
+    blake3-devel \
     benchmark-devel \
     boost-devel \
     libboost_chrono-devel \
diff --git a/.docker/build-linux.sh b/.docker/build-linux.sh
@@ -6,7 +6,7 @@ export CCACHE_CONFIGPATH=/workspace/.docker/ccache.conf
 
 LOCAL_REPO_PATH=/local/repos
 mkdir -p "$LOCAL_REPO_PATH"
-LAST_UPDATE_FILE="$LOCAL_REPO_PATH/last-update"
+LAST_UPDATE_BASE="$LOCAL_REPO_PATH/last-update"
 
 WORKFLOW_LOG_DIR="/artifacts/workflow-logs/${GITHUB_RUN_ID}"
 NINJA_LOG_FILE="${WORKFLOW_LOG_DIR}/ninja-${BUILD_ARCH},${CROSS_ARCH},${BUILD_DIST},${BUILD_TYPE}.log"
@@ -19,31 +19,34 @@ log() {
     echo "$(python3 - <<<'from datetime import datetime; print(datetime.now().isoformat())')	$event" >> "$BUILD_LOG_FILE"
 }
 
-if [ -f "$LAST_UPDATE_FILE" ] && [ $(find "$LAST_UPDATE_FILE" -mmin -180) ]; then
-    echo "Skipping git repo update because it already ran in the last three hours."
-else
-    echo "Running git repo update."
-
-    log "begin:repo-update"
-
-    for repo in "fmtlib/fmt" \
-                "google/googletest" \
-                "ericniebler/range-v3" \
-                "greg7mdp/parallel-hashmap"; do
-      reponame=$(basename "$repo")
-      cd "$LOCAL_REPO_PATH"
-      if [ -d "$reponame" ]; then
-        cd "$reponame"
-        time git fetch
-      else
-        time git clone "https://github.com/$repo.git"
-      fi
-    done
+log "begin:repo-update"
 
-    log "end:repo-update"
+for repo in "BLAKE3-team/BLAKE3" \
+            "fmtlib/fmt" \
+            "google/googletest" \
+            "ericniebler/range-v3" \
+            "greg7mdp/parallel-hashmap"; do
+  reponame=$(basename "$repo")
+  cd "$LOCAL_REPO_PATH"
+  last_update_path="$LAST_UPDATE_BASE.$reponame"
 
-    touch "$LAST_UPDATE_FILE"
-fi
+  if [ -d "$reponame" ] && [ -f "$last_update_path" ] && [ $(find "$last_update_path" -mmin -180) ]; then
+    echo "[$reponame] skipping repo update because it already ran in the last three hours"
+  else
+    echo "[$reponame] running git repo update..."
+
+    if [ -d "$reponame" ]; then
+      cd "$reponame"
+      time git fetch
+    else
+      time git clone "https://github.com/$repo.git"
+    fi
+
+    touch "$last_update_path"
+  fi
+done
+
+log "end:repo-update"
 
 if [[ "$BUILD_TYPE" != "clang-release-ninja-static" ]]; then
   export DWARFS_LOCAL_REPO_PATH="$LOCAL_REPO_PATH"
diff --git a/.docker/check_lib_versions.py b/.docker/check_lib_versions.py
@@ -8,6 +8,7 @@
     "xiph/flac",
     "libunwind/libunwind",
     "google/benchmark",
+    "BLAKE3-team/BLAKE3",
     "boostorg/boost",
     "openssl/openssl",
     "libressl/portable",
diff --git a/.docker/static-libs-fetch.sh b/.docker/static-libs-fetch.sh
@@ -22,6 +22,7 @@ fetch_lib flac https://github.com/xiph/flac/releases/download/${FLAC_VERSION}/${
 fetch_lib libucontext https://github.com/kaniini/libucontext/archive/refs/tags/${LIBUCONTEXT_TARBALL}
 fetch_lib libunwind https://github.com/libunwind/libunwind/releases/download/v${LIBUNWIND_VERSION}/${LIBUNWIND_TARBALL}
 fetch_lib benchmark https://github.com/google/benchmark/archive/refs/tags/v${BENCHMARK_VERSION}.tar.gz ${BENCHMARK_TARBALL}
+fetch_lib blake3 https://github.com/BLAKE3-team/BLAKE3/archive/refs/tags/${BLAKE3_VERSION}.tar.gz ${BLAKE3_TARBALL}
 fetch_lib openssl https://github.com/openssl/openssl/releases/download/openssl-${OPENSSL_VERSION}/${OPENSSL_TARBALL}
 fetch_lib libressl https://github.com/libressl/portable/releases/download/v${LIBRESSL_VERSION}/${LIBRESSL_TARBALL}
 fetch_lib cpptrace https://github.com/jeremy-rifkin/cpptrace/archive/refs/tags/v${CPPTRACE_VERSION}.tar.gz ${CPPTRACE_TARBALL}
diff --git a/.docker/static-libs-install.sh b/.docker/static-libs-install.sh
@@ -18,7 +18,7 @@ if [[ "$PKGS" == ":none" ]]; then
     echo "No libraries to build"
     exit 0
 elif [[ "$PKGS" == ":all" ]]; then
-    PKGS="benchmark,boost,brotli,cpptrace,double-conversion,flac,fmt,fuse,fuse3,glog,jemalloc,libarchive,libdwarf,libevent,libucontext,libunwind,libressl,lz4,mimalloc,nlohmann,openssl,parallel-hashmap,range-v3,utfcpp,xxhash,xz,zstd"
+    PKGS="benchmark,blake3,boost,brotli,cpptrace,double-conversion,flac,fmt,fuse,fuse3,glog,jemalloc,libarchive,libdwarf,libevent,libucontext,libunwind,libressl,lz4,mimalloc,nlohmann,openssl,parallel-hashmap,range-v3,utfcpp,xxhash,xz,zstd"
 fi
 
 export COMMON_CFLAGS="-ffunction-sections -fdata-sections -fmerge-all-constants"
@@ -270,6 +270,18 @@ if use_lib nlohmann; then
     cp "$HOME/pkgs/json.hpp" "$INSTALL_DIR/include/nlohmann/json.hpp"
 fi
 
+if use_lib blake3; then
+    opt_size
+    cd "$WORKDIR"
+    tar xf ${WORKROOT}/${BLAKE3_TARBALL}
+    cd BLAKE3-${BLAKE3_VERSION}/c
+    mkdir build
+    cd build
+    cmake .. -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" ${CMAKE_ARGS}
+    $NINJA_PARALLEL
+    ninja install
+fi
+
 if use_lib utfcpp; then
     opt_size
     cd "$WORKDIR"
diff --git a/.docker/static-libs-versions.sh b/.docker/static-libs-versions.sh
@@ -7,6 +7,7 @@ FLAC_VERSION=1.5.0                       # 2025-02-11
 LIBUCONTEXT_VERSION=1.3.2                # 2024-10-07
 LIBUNWIND_VERSION=1.8.3                  # 2025-09-04
 BENCHMARK_VERSION=1.9.4                  # 2025-05-19
+BLAKE3_VERSION=1.8.2                     # 2025-04-21
 BOOST_VERSION=1.89.0                     # 2025-08-14
 OPENSSL_VERSION=3.6.0                    # 2025-10-01
 LIBRESSL_VERSION=4.2.1                   # 2025-10-30
@@ -38,6 +39,7 @@ FLAC_TARBALL="flac-${FLAC_VERSION}.tar.xz"
 LIBUCONTEXT_TARBALL="libucontext-${LIBUCONTEXT_VERSION}.tar.gz"
 LIBUNWIND_TARBALL="libunwind-${LIBUNWIND_VERSION}.tar.gz"
 BENCHMARK_TARBALL="benchmark-${BENCHMARK_VERSION}.tar.gz"
+BLAKE3_TARBALL="BLAKE3-${BLAKE3_VERSION}.tar.gz"
 OPENSSL_TARBALL="openssl-${OPENSSL_VERSION}.tar.gz"
 LIBRESSL_TARBALL="libressl-${LIBRESSL_VERSION}.tar.gz"
 CPPTRACE_TARBALL="cpptrace-${CPPTRACE_VERSION}.tar.gz"
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -109,6 +109,9 @@ set(RANGE_V3_PREFERRED_VERSION 0.12.0)
 set(PARALLEL_HASHMAP_REQUIRED_VERSION 1.3.8)
 set(PARALLEL_HASHMAP_PREFERRED_VERSION 2.0.0)
 
+set(BLAKE3_REQUIRED_VERSION 1.2.0)
+set(BLAKE3_PREFERRED_VERSION 1.8.2)
+
 set(BOOST_REQUIRED_VERSION 1.67.0)
 set(LIBCRYPTO_REQUIRED_VERSION 3.0.0)
 set(LIBLZ4_REQUIRED_VERSION 1.9.3)
@@ -127,11 +130,13 @@ if(DEFINED ENV{DWARFS_LOCAL_REPO_PATH})
   set(GOOGLETEST_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/googletest)
   set(RANGE_V3_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/range-v3)
   set(PARALLEL_HASHMAP_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/parallel-hashmap)
+  set(BLAKE3_GIT_REPO $ENV{DWARFS_LOCAL_REPO_PATH}/BLAKE3)
 else()
   set(LIBFMT_GIT_REPO https://github.com/fmtlib/fmt.git)
   set(GOOGLETEST_GIT_REPO https://github.com/google/googletest.git)
   set(RANGE_V3_GIT_REPO https://github.com/ericniebler/range-v3.git)
   set(PARALLEL_HASHMAP_GIT_REPO https://github.com/greg7mdp/parallel-hashmap.git)
+  set(BLAKE3_GIT_REPO https://github.com/BLAKE3-team/BLAKE3.git)
 endif()
 
 if(APPLE)
@@ -198,6 +203,7 @@ include(${CMAKE_SOURCE_DIR}/cmake/need_fmt.cmake)
 include(${CMAKE_SOURCE_DIR}/cmake/need_range_v3.cmake)
 
 if(WITH_LIBDWARFS)
+  include(${CMAKE_SOURCE_DIR}/cmake/need_blake3.cmake)
   include(${CMAKE_SOURCE_DIR}/cmake/need_phmap.cmake)
 
   find_package(Boost ${BOOST_REQUIRED_VERSION} REQUIRED CONFIG
diff --git a/cmake/dwarfs-config.cmake.in b/cmake/dwarfs-config.cmake.in
@@ -29,13 +29,18 @@ pkg_check_modules(LIBARCHIVE REQUIRED IMPORTED_TARGET libarchive>=@LIBARCHIVE_RE
 pkg_check_modules(XXHASH REQUIRED IMPORTED_TARGET libxxhash>=@XXHASH_REQUIRED_VERSION@)
 pkg_check_modules(ZSTD REQUIRED IMPORTED_TARGET libzstd>=@ZSTD_REQUIRED_VERSION@)
 
+set(LIBDWARFS_WITH_BLAKE3 @blake3_FOUND@)
 set(LIBDWARFS_WITH_CPPTRACE @cpptrace_FOUND@)
 set(LIBDWARFS_WITH_LIBLZ4 @LIBLZ4_FOUND@)
 set(LIBDWARFS_WITH_LIBLZMA @LIBLZMA_FOUND@)
 set(LIBDWARFS_WITH_LIBBROTLI (@LIBBROTLIDEC_FOUND@ AND @LIBBROTLIENC_FOUND@))
 set(LIBDWARFS_WITH_FLAC @FLAC_FOUND@)
 set(LIBDWARFS_WITH_JEMALLOC @JEMALLOC_FOUND@)
 
+if(LIBDWARFS_WITH_BLAKE3)
+  find_dependency(blake3 CONFIG REQUIRED)
+endif()
+
 if(LIBDWARFS_WITH_CPPTRACE)
   find_dependency(cpptrace CONFIG REQUIRED)
 endif()
diff --git a/cmake/libdwarfs.cmake b/cmake/libdwarfs.cmake
@@ -208,7 +208,7 @@ add_cpp2_thrift_library(thrift/history.thrift
 add_cpp2_thrift_library(thrift/features.thrift
                         TARGET dwarfs_features_thrift OUTPUT_PATH dwarfs)
 
-target_link_libraries(dwarfs_common PRIVATE dwarfs_folly_lite PkgConfig::LIBCRYPTO PkgConfig::XXHASH PkgConfig::ZSTD)
+target_link_libraries(dwarfs_common PRIVATE dwarfs_folly_lite PkgConfig::LIBCRYPTO PkgConfig::XXHASH PkgConfig::ZSTD BLAKE3::blake3)
 target_link_libraries(dwarfs_compressor PRIVATE dwarfs_common)
 target_link_libraries(dwarfs_decompressor PRIVATE dwarfs_common)
 target_link_libraries(dwarfs_reader PUBLIC dwarfs_common dwarfs_decompressor)
diff --git a/cmake/need_blake3.cmake b/cmake/need_blake3.cmake
@@ -0,0 +1,37 @@
+#
+# Copyright (c) Marcus Holland-Moritz
+#
+# This file is part of dwarfs.
+#
+# dwarfs is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# dwarfs is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# dwarfs.  If not, see <https://www.gnu.org/licenses/>.
+#
+
+cmake_minimum_required(VERSION 3.28.0)
+
+find_package(blake3 ${BLAKE3_REQUIRED_VERSION} CONFIG)
+
+if(NOT blake3_FOUND)
+  set(ORIG_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+  set(BUILD_SHARED_LIBS OFF)
+  FetchContent_Declare(
+    blake3
+    GIT_REPOSITORY ${BLAKE3_GIT_REPO}
+    GIT_TAG ${BLAKE3_PREFERRED_VERSION}
+    SOURCE_SUBDIR c
+    EXCLUDE_FROM_ALL
+    SYSTEM
+  )
+  FetchContent_MakeAvailable(blake3)
+  set_target_properties(blake3 PROPERTIES POSITION_INDEPENDENT_CODE ON)
+  set(BUILD_SHARED_LIBS ${ORIG_BUILD_SHARED_LIBS})
+endif()
diff --git a/include/dwarfs/checksum.h b/include/dwarfs/checksum.h
@@ -47,19 +47,24 @@ class checksum {
 
   struct xxh3_64_tag {};
   struct sha2_512_256_tag {};
+  struct blake3_256_tag {};
 
   static constexpr xxh3_64_tag xxh3_64{};
   static constexpr sha2_512_256_tag sha2_512_256{};
+  static constexpr blake3_256_tag blake3_256{};
 
   static bool verify(xxh3_64_tag, void const* data, size_t size,
                      void const* digest, size_t digest_size);
   static bool verify(sha2_512_256_tag, void const* data, size_t size,
                      void const* digest, size_t digest_size);
+  static bool verify(blake3_256_tag, void const* data, size_t size,
+                     void const* digest, size_t digest_size);
   static bool verify(std::string const& alg, void const* data, size_t size,
                      void const* digest, size_t digest_size);
 
   checksum(xxh3_64_tag);
   checksum(sha2_512_256_tag);
+  checksum(blake3_256_tag);
   checksum(std::string const& alg);
 
   checksum& update(void const* data, size_t size) {
diff --git a/src/checksum.cpp b/src/checksum.cpp
diff --git a/test/checksum_test.cpp b/test/checksum_test.cpp
diff --git a/vcpkg.json b/vcpkg.json
