feat(pkg/digest): support blake3 hashing

MODULE.bazel

@@ -65,6 +65,7 @@ use_repo(
     "com_github_prometheus_common",
     "com_github_sercand_kuberesolver_v5",
     "com_github_stretchr_testify",
+    "com_github_zeebo_blake3",
     "com_google_cloud_go_longrunning",
     "com_google_cloud_go_storage",
     "io_k8s_apimachinery",

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/prometheus/common v0.67.2
 	github.com/sercand/kuberesolver/v5 v5.1.1
 	github.com/stretchr/testify v1.11.1
+	github.com/zeebo/blake3 v0.2.4
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0
 	go.opentelemetry.io/contrib/propagators/b3 v1.38.0
 	go.opentelemetry.io/otel v1.38.0
@@ -120,6 +121,7 @@ require (
 	github.com/googleapis/gax-go/v2 v2.15.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.0.12 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

go.sum

@@ -208,6 +208,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
 github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -274,6 +276,12 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
+github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=

pkg/digest/BUILD.bazel

@@ -24,6 +24,7 @@ go_library(
         "@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
         "@com_github_buildbarn_go_sha256tree//:go-sha256tree",
         "@com_github_google_uuid//:uuid",
+        "@com_github_zeebo_blake3//:blake3",
         "@org_golang_google_grpc//codes",
         "@org_golang_google_grpc//status",
     ],

pkg/digest/bare_function.go

@@ -9,12 +9,14 @@ import (
 
 	remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
 	"github.com/buildbarn/go-sha256tree"
+	"github.com/zeebo/blake3"
 )
 
 // SupportedDigestFunctions is the list of digest functions supported by
 // digest.Digest, using the enumeration values that are part of the
 // Remote Execution protocol.
 var SupportedDigestFunctions = []remoteexecution.DigestFunction_Value{
+	remoteexecution.DigestFunction_BLAKE3,
 	remoteexecution.DigestFunction_MD5,
 	remoteexecution.DigestFunction_SHA1,
 	remoteexecution.DigestFunction_SHA256,
@@ -38,6 +40,13 @@ type bareFunction struct {
 }
 
 var (
+	blake3BareFunction = bareFunction{
+		enumValue: remoteexecution.DigestFunction_BLAKE3,
+		hasherFactory: func(expectedSizeBytes int64) hash.Hash {
+			return blake3.New()
+		},
+		hashBytesSize: 32,
+	}
 	md5BareFunction = bareFunction{
 		enumValue: remoteexecution.DigestFunction_MD5,
 		hasherFactory: func(expectedSizeBytes int64) hash.Hash {
@@ -100,6 +109,8 @@ func getBareFunction(digestFunction remoteexecution.DigestFunction_Value, hashSt
 		case sha512.Size * 2:
 			return &sha512BareFunction
 		}
+	case remoteexecution.DigestFunction_BLAKE3:
+		return &blake3BareFunction
 	case remoteexecution.DigestFunction_MD5:
 		return &md5BareFunction
 	case remoteexecution.DigestFunction_SHA1:

pkg/digest/digest_test.go

@@ -41,6 +41,13 @@ func TestNewDigestFromByteStreamReadPath(t *testing.T) {
 	})
 
 	t.Run("NoInstanceName", func(t *testing.T) {
+		t.Run("BLAKE3", func(t *testing.T) {
+			d, compressor, err := digest.NewDigestFromByteStreamReadPath("blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123")
+			require.NoError(t, err)
+			require.Equal(t, digest.MustNewDigest("", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123), d)
+			require.Equal(t, remoteexecution.Compressor_IDENTITY, compressor)
+		})
+
 		t.Run("MD5", func(t *testing.T) {
 			d, compressor, err := digest.NewDigestFromByteStreamReadPath("blobs/8b1a9953c4611296a827abf8c47804d7/123")
 			require.NoError(t, err)
@@ -119,6 +126,13 @@ func TestNewDigestFromByteStreamWritePath(t *testing.T) {
 	})
 
 	t.Run("NoInstanceName", func(t *testing.T) {
+		t.Run("BLAKE3", func(t *testing.T) {
+			d, compressor, err := digest.NewDigestFromByteStreamWritePath("uploads/da2f1135-326b-4956-b920-1646cdd6cb53/blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123")
+			require.NoError(t, err)
+			require.Equal(t, digest.MustNewDigest("", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123), d)
+			require.Equal(t, remoteexecution.Compressor_IDENTITY, compressor)
+		})
+
 		t.Run("MD5", func(t *testing.T) {
 			d, compressor, err := digest.NewDigestFromByteStreamWritePath("uploads/da2f1135-326b-4956-b920-1646cdd6cb53/blobs/8b1a9953c4611296a827abf8c47804d7/123")
 			require.NoError(t, err)
@@ -182,6 +196,17 @@ func TestNewDigestFromByteStreamWritePath(t *testing.T) {
 
 func TestDigestGetByteStreamReadPath(t *testing.T) {
 	t.Run("NoInstanceName", func(t *testing.T) {
+		t.Run("BLAKE3", func(t *testing.T) {
+			require.Equal(
+				t,
+				"blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123",
+				digest.MustNewDigest(
+					"",
+					remoteexecution.DigestFunction_BLAKE3,
+					"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
+					123).GetByteStreamReadPath(remoteexecution.Compressor_IDENTITY))
+		})
+
 		t.Run("MD5", func(t *testing.T) {
 			require.Equal(
 				t,
@@ -242,6 +267,17 @@ func TestDigestGetByteStreamWritePath(t *testing.T) {
 	uuid := uuid.Must(uuid.Parse("36ebab65-3c4f-4faf-818b-2eabb4cd1b02"))
 
 	t.Run("NoInstanceName", func(t *testing.T) {
+		t.Run("BLAKE3", func(t *testing.T) {
+			require.Equal(
+				t,
+				"uploads/36ebab65-3c4f-4faf-818b-2eabb4cd1b02/blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123",
+				digest.MustNewDigest(
+					"",
+					remoteexecution.DigestFunction_BLAKE3,
+					"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
+					123).GetByteStreamWritePath(uuid, remoteexecution.Compressor_IDENTITY))
+		})
+
 		t.Run("MD5", func(t *testing.T) {
 			require.Equal(
 				t,
@@ -409,6 +445,18 @@ func TestDigestGetSizeBytes(t *testing.T) {
 }
 
 func TestDigestGetKey(t *testing.T) {
+	t.Run("BLAKE3", func(t *testing.T) {
+		d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123)
+		require.Equal(
+			t,
+			"9-af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262-123",
+			d.GetKey(digest.KeyWithoutInstance))
+		require.Equal(
+			t,
+			"9-af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262-123-hello",
+			d.GetKey(digest.KeyWithInstance))
+	})
+
 	t.Run("SHA256", func(t *testing.T) {
 		d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA256, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 123)
 		require.Equal(
@@ -503,20 +551,40 @@ func TestRemoveUnsupportedDigestFunctions(t *testing.T) {
 	require.Equal(
 		t,
 		[]remoteexecution.DigestFunction_Value{
+			remoteexecution.DigestFunction_BLAKE3,
 			remoteexecution.DigestFunction_MD5,
 			remoteexecution.DigestFunction_SHA1,
 			remoteexecution.DigestFunction_SHA256,
 		},
 		digest.RemoveUnsupportedDigestFunctions([]remoteexecution.DigestFunction_Value{
+			remoteexecution.DigestFunction_BLAKE3,
 			remoteexecution.DigestFunction_MD5,
-			remoteexecution.DigestFunction_SHA256,
 			remoteexecution.DigestFunction_SHA1,
 			remoteexecution.DigestFunction_SHA1,
+			remoteexecution.DigestFunction_SHA256,
 			remoteexecution.DigestFunction_VSO,
 		}))
 }
 
 func TestDigestGetCompactBinary(t *testing.T) {
+	t.Run("BLAKE3", func(t *testing.T) {
+		d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 124982395)
+		require.Equal(
+			t,
+			[]byte{
+				// Digest function: remoteexecution.DigestFunction_BLAKE3.
+				0x09,
+				// Hash.
+				0xaf, 0x13, 0x49, 0xb9, 0xf5, 0xf9, 0xa1, 0xa6,
+				0xa0, 0x40, 0x4d, 0xea, 0x36, 0xdc, 0xc9, 0x49,
+				0x9b, 0xcb, 0x25, 0xc9, 0xad, 0xc1, 0x12, 0xb7,
+				0xcc, 0x9a, 0x93, 0xca, 0xe4, 0x1f, 0x32, 0x62,
+				// Size.
+				0xf6, 0xd1, 0x98, 0x77,
+			},
+			d.GetCompactBinary())
+	})
+
 	t.Run("SHA256", func(t *testing.T) {
 		d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA256, "18c17f53df2fcd1f8271bc1c0e55df71b1a796eaa74ff45a68900f04e3f4c7a2", 124982395)
 		require.Equal(

pkg/digest/instance_name_test.go

@@ -49,6 +49,19 @@ func TestInstanceNameGetDigestFunction(t *testing.T) {
 		testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Unknown digest function"), err)
 	})
 
+	t.Run("BLAKE3", func(t *testing.T) {
+		digestFunction, err := instanceName.GetDigestFunction(remoteexecution.DigestFunction_BLAKE3, 0)
+		require.NoError(t, err)
+
+		g := digestFunction.NewGenerator(5)
+		g.Write([]byte("Hello"))
+		require.Equal(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "fbc2b0516ee8744d293b980779178a3508850fdcfe965985782c39601b65794f", 5), g.Sum())
+
+		require.True(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123).UsesDigestFunction(digestFunction))
+		require.False(t, digest.MustNewDigest("bye", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 456).UsesDigestFunction(digestFunction))
+		require.False(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA1, "5ad9e0fd2f11ec59c95c60020c2b00afbef10e5b", 789).UsesDigestFunction(digestFunction))
+	})
+
 	t.Run("MD5", func(t *testing.T) {
 		digestFunction, err := instanceName.GetDigestFunction(remoteexecution.DigestFunction_MD5, 0)
 		require.NoError(t, err)
@@ -106,6 +119,22 @@ func TestInstanceNameGetComponents(t *testing.T) {
 func TestInstanceNameNewDigestFromCompactBinary(t *testing.T) {
 	instanceName := util.Must(digest.NewInstanceName("hello"))
 
+	t.Run("BLAKE3", func(t *testing.T) {
+		blobDigest, err := instanceName.NewDigestFromCompactBinary(bytes.NewBuffer([]byte{
+			// Digest function: remoteexecution.DigestFunction_BLAKE3.
+			0x09,
+			// Hash.
+			0xaf, 0x13, 0x49, 0xb9, 0xf5, 0xf9, 0xa1, 0xa6,
+			0xa0, 0x40, 0x4d, 0xea, 0x36, 0xdc, 0xc9, 0x49,
+			0x9b, 0xcb, 0x25, 0xc9, 0xad, 0xc1, 0x12, 0xb7,
+			0xcc, 0x9a, 0x93, 0xca, 0xe4, 0x1f, 0x32, 0x62,
+			// Size.
+			0xf6, 0xd1, 0x98, 0x77,
+		}))
+		require.NoError(t, err)
+		require.Equal(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 124982395), blobDigest)
+	})
+
 	t.Run("SHA256", func(t *testing.T) {
 		blobDigest, err := instanceName.NewDigestFromCompactBinary(bytes.NewBuffer([]byte{
 			// Digest function: remoteexecution.DigestFunction_SHA256.