Skip to content

Commit ef88a3b

Browse files
tyler-frenchtyler-french
committed
support blake3 for servers that support blake3
1 parent 2d6050f commit ef88a3b

File tree

7 files changed

+121
-1
lines changed

7 files changed

+121
-1
lines changed

‎MODULE.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ use_repo(
6565
"com_github_prometheus_common",
6666
"com_github_sercand_kuberesolver_v5",
6767
"com_github_stretchr_testify",
68+
"com_github_zeebo_blake3",
6869
"com_google_cloud_go_longrunning",
6970
"com_google_cloud_go_storage",
7071
"io_k8s_apimachinery",

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ require (
3636
github.com/prometheus/common v0.67.2
3737
github.com/sercand/kuberesolver/v5 v5.1.1
3838
github.com/stretchr/testify v1.11.1
39+
github.com/zeebo/blake3 v0.2.4
3940
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.63.0
4041
go.opentelemetry.io/contrib/propagators/b3 v1.38.0
4142
go.opentelemetry.io/otel v1.38.0
@@ -120,6 +121,7 @@ require (
120121
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
121122
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect
122123
github.com/json-iterator/go v1.1.12 // indirect
124+
github.com/klauspost/cpuid/v2 v2.0.12 // indirect
123125
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
124126
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
125127
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

‎go.sum‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
208208
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
209209
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
210210
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
211+
github.com/klauspost/cpuid/v2 v2.0.12 h1:p9dKCg8i4gmOxtv35DvrYoWqYzQrvEVdjQ762Y0OqZE=
212+
github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
211213
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
212214
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
213215
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -274,6 +276,12 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY
274276
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
275277
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
276278
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
279+
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
280+
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
281+
github.com/zeebo/blake3 v0.2.4 h1:KYQPkhpRtcqh0ssGYcKLG1JYvddkEA8QwCM/yBqhaZI=
282+
github.com/zeebo/blake3 v0.2.4/go.mod h1:7eeQ6d2iXWRGF6npfaxl2CU+xy2Fjo2gxeyZGCRUjcE=
283+
github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
284+
github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
277285
go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
278286
go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
279287
go.opentelemetry.io/contrib/detectors/gcp v1.38.0 h1:ZoYbqX7OaA/TAikspPl3ozPI6iY6LiIY9I8cUfm+pJs=

‎pkg/digest/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ go_library(
2424
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
2525
"@com_github_buildbarn_go_sha256tree//:go-sha256tree",
2626
"@com_github_google_uuid//:uuid",
27+
"@com_github_zeebo_blake3//:blake3",
2728
"@org_golang_google_grpc//codes",
2829
"@org_golang_google_grpc//status",
2930
],

‎pkg/digest/bare_function.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@ import (
99

1010
remoteexecution "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
1111
"github.com/buildbarn/go-sha256tree"
12+
"github.com/zeebo/blake3"
1213
)
1314

1415
// SupportedDigestFunctions is the list of digest functions supported by
1516
// digest.Digest, using the enumeration values that are part of the
1617
// Remote Execution protocol.
1718
var SupportedDigestFunctions = []remoteexecution.DigestFunction_Value{
19+
remoteexecution.DigestFunction_BLAKE3,
1820
remoteexecution.DigestFunction_MD5,
1921
remoteexecution.DigestFunction_SHA1,
2022
remoteexecution.DigestFunction_SHA256,
@@ -38,6 +40,13 @@ type bareFunction struct {
3840
}
3941

4042
var (
43+
blake3BareFunction = bareFunction{
44+
enumValue: remoteexecution.DigestFunction_BLAKE3,
45+
hasherFactory: func(expectedSizeBytes int64) hash.Hash {
46+
return blake3.New()
47+
},
48+
hashBytesSize: 32,
49+
}
4150
md5BareFunction = bareFunction{
4251
enumValue: remoteexecution.DigestFunction_MD5,
4352
hasherFactory: func(expectedSizeBytes int64) hash.Hash {
@@ -100,6 +109,8 @@ func getBareFunction(digestFunction remoteexecution.DigestFunction_Value, hashSt
100109
case sha512.Size * 2:
101110
return &sha512BareFunction
102111
}
112+
case remoteexecution.DigestFunction_BLAKE3:
113+
return &blake3BareFunction
103114
case remoteexecution.DigestFunction_MD5:
104115
return &md5BareFunction
105116
case remoteexecution.DigestFunction_SHA1:

‎pkg/digest/digest_test.go‎

Lines changed: 69 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ func TestNewDigestFromByteStreamReadPath(t *testing.T) {
4141
})
4242

4343
t.Run("NoInstanceName", func(t *testing.T) {
44+
t.Run("BLAKE3", func(t *testing.T) {
45+
d, compressor, err := digest.NewDigestFromByteStreamReadPath("blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123")
46+
require.NoError(t, err)
47+
require.Equal(t, digest.MustNewDigest("", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123), d)
48+
require.Equal(t, remoteexecution.Compressor_IDENTITY, compressor)
49+
})
50+
4451
t.Run("MD5", func(t *testing.T) {
4552
d, compressor, err := digest.NewDigestFromByteStreamReadPath("blobs/8b1a9953c4611296a827abf8c47804d7/123")
4653
require.NoError(t, err)
@@ -119,6 +126,13 @@ func TestNewDigestFromByteStreamWritePath(t *testing.T) {
119126
})
120127

121128
t.Run("NoInstanceName", func(t *testing.T) {
129+
t.Run("BLAKE3", func(t *testing.T) {
130+
d, compressor, err := digest.NewDigestFromByteStreamWritePath("uploads/da2f1135-326b-4956-b920-1646cdd6cb53/blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123")
131+
require.NoError(t, err)
132+
require.Equal(t, digest.MustNewDigest("", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123), d)
133+
require.Equal(t, remoteexecution.Compressor_IDENTITY, compressor)
134+
})
135+
122136
t.Run("MD5", func(t *testing.T) {
123137
d, compressor, err := digest.NewDigestFromByteStreamWritePath("uploads/da2f1135-326b-4956-b920-1646cdd6cb53/blobs/8b1a9953c4611296a827abf8c47804d7/123")
124138
require.NoError(t, err)
@@ -182,6 +196,17 @@ func TestNewDigestFromByteStreamWritePath(t *testing.T) {
182196

183197
func TestDigestGetByteStreamReadPath(t *testing.T) {
184198
t.Run("NoInstanceName", func(t *testing.T) {
199+
t.Run("BLAKE3", func(t *testing.T) {
200+
require.Equal(
201+
t,
202+
"blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123",
203+
digest.MustNewDigest(
204+
"",
205+
remoteexecution.DigestFunction_BLAKE3,
206+
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
207+
123).GetByteStreamReadPath(remoteexecution.Compressor_IDENTITY))
208+
})
209+
185210
t.Run("MD5", func(t *testing.T) {
186211
require.Equal(
187212
t,
@@ -242,6 +267,17 @@ func TestDigestGetByteStreamWritePath(t *testing.T) {
242267
uuid := uuid.Must(uuid.Parse("36ebab65-3c4f-4faf-818b-2eabb4cd1b02"))
243268

244269
t.Run("NoInstanceName", func(t *testing.T) {
270+
t.Run("BLAKE3", func(t *testing.T) {
271+
require.Equal(
272+
t,
273+
"uploads/36ebab65-3c4f-4faf-818b-2eabb4cd1b02/blobs/blake3/af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262/123",
274+
digest.MustNewDigest(
275+
"",
276+
remoteexecution.DigestFunction_BLAKE3,
277+
"af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262",
278+
123).GetByteStreamWritePath(uuid, remoteexecution.Compressor_IDENTITY))
279+
})
280+
245281
t.Run("MD5", func(t *testing.T) {
246282
require.Equal(
247283
t,
@@ -409,6 +445,18 @@ func TestDigestGetSizeBytes(t *testing.T) {
409445
}
410446

411447
func TestDigestGetKey(t *testing.T) {
448+
t.Run("BLAKE3", func(t *testing.T) {
449+
d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123)
450+
require.Equal(
451+
t,
452+
"9-af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262-123",
453+
d.GetKey(digest.KeyWithoutInstance))
454+
require.Equal(
455+
t,
456+
"9-af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262-123-hello",
457+
d.GetKey(digest.KeyWithInstance))
458+
})
459+
412460
t.Run("SHA256", func(t *testing.T) {
413461
d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA256, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", 123)
414462
require.Equal(
@@ -503,20 +551,40 @@ func TestRemoveUnsupportedDigestFunctions(t *testing.T) {
503551
require.Equal(
504552
t,
505553
[]remoteexecution.DigestFunction_Value{
554+
remoteexecution.DigestFunction_BLAKE3,
506555
remoteexecution.DigestFunction_MD5,
507556
remoteexecution.DigestFunction_SHA1,
508557
remoteexecution.DigestFunction_SHA256,
509558
},
510559
digest.RemoveUnsupportedDigestFunctions([]remoteexecution.DigestFunction_Value{
560+
remoteexecution.DigestFunction_BLAKE3,
511561
remoteexecution.DigestFunction_MD5,
512-
remoteexecution.DigestFunction_SHA256,
513562
remoteexecution.DigestFunction_SHA1,
514563
remoteexecution.DigestFunction_SHA1,
564+
remoteexecution.DigestFunction_SHA256,
515565
remoteexecution.DigestFunction_VSO,
516566
}))
517567
}
518568

519569
func TestDigestGetCompactBinary(t *testing.T) {
570+
t.Run("BLAKE3", func(t *testing.T) {
571+
d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 124982395)
572+
require.Equal(
573+
t,
574+
[]byte{
575+
// Digest function: remoteexecution.DigestFunction_BLAKE3.
576+
0x09,
577+
// Hash.
578+
0xaf, 0x13, 0x49, 0xb9, 0xf5, 0xf9, 0xa1, 0xa6,
579+
0xa0, 0x40, 0x4d, 0xea, 0x36, 0xdc, 0xc9, 0x49,
580+
0x9b, 0xcb, 0x25, 0xc9, 0xad, 0xc1, 0x12, 0xb7,
581+
0xcc, 0x9a, 0x93, 0xca, 0xe4, 0x1f, 0x32, 0x62,
582+
// Size.
583+
0xf6, 0xd1, 0x98, 0x77,
584+
},
585+
d.GetCompactBinary())
586+
})
587+
520588
t.Run("SHA256", func(t *testing.T) {
521589
d := digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA256, "18c17f53df2fcd1f8271bc1c0e55df71b1a796eaa74ff45a68900f04e3f4c7a2", 124982395)
522590
require.Equal(

‎pkg/digest/instance_name_test.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,19 @@ func TestInstanceNameGetDigestFunction(t *testing.T) {
4949
testutil.RequireEqualStatus(t, status.Error(codes.InvalidArgument, "Unknown digest function"), err)
5050
})
5151

52+
t.Run("BLAKE3", func(t *testing.T) {
53+
digestFunction, err := instanceName.GetDigestFunction(remoteexecution.DigestFunction_BLAKE3, 0)
54+
require.NoError(t, err)
55+
56+
g := digestFunction.NewGenerator(5)
57+
g.Write([]byte("Hello"))
58+
require.Equal(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "fbc2b0516ee8744d293b980779178a3508850fdcfe965985782c39601b65794f", 5), g.Sum())
59+
60+
require.True(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 123).UsesDigestFunction(digestFunction))
61+
require.False(t, digest.MustNewDigest("bye", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 456).UsesDigestFunction(digestFunction))
62+
require.False(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_SHA1, "5ad9e0fd2f11ec59c95c60020c2b00afbef10e5b", 789).UsesDigestFunction(digestFunction))
63+
})
64+
5265
t.Run("MD5", func(t *testing.T) {
5366
digestFunction, err := instanceName.GetDigestFunction(remoteexecution.DigestFunction_MD5, 0)
5467
require.NoError(t, err)
@@ -106,6 +119,22 @@ func TestInstanceNameGetComponents(t *testing.T) {
106119
func TestInstanceNameNewDigestFromCompactBinary(t *testing.T) {
107120
instanceName := util.Must(digest.NewInstanceName("hello"))
108121

122+
t.Run("BLAKE3", func(t *testing.T) {
123+
blobDigest, err := instanceName.NewDigestFromCompactBinary(bytes.NewBuffer([]byte{
124+
// Digest function: remoteexecution.DigestFunction_BLAKE3.
125+
0x09,
126+
// Hash.
127+
0xaf, 0x13, 0x49, 0xb9, 0xf5, 0xf9, 0xa1, 0xa6,
128+
0xa0, 0x40, 0x4d, 0xea, 0x36, 0xdc, 0xc9, 0x49,
129+
0x9b, 0xcb, 0x25, 0xc9, 0xad, 0xc1, 0x12, 0xb7,
130+
0xcc, 0x9a, 0x93, 0xca, 0xe4, 0x1f, 0x32, 0x62,
131+
// Size.
132+
0xf6, 0xd1, 0x98, 0x77,
133+
}))
134+
require.NoError(t, err)
135+
require.Equal(t, digest.MustNewDigest("hello", remoteexecution.DigestFunction_BLAKE3, "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", 124982395), blobDigest)
136+
})
137+
109138
t.Run("SHA256", func(t *testing.T) {
110139
blobDigest, err := instanceName.NewDigestFromCompactBinary(bytes.NewBuffer([]byte{
111140
// Digest function: remoteexecution.DigestFunction_SHA256.

0 commit comments

Comments
 (0)