Skip to content

Commit 6e0a6f1

Browse files
hiddecohiddeco
committed
storage: calculate Digest for Artifact
Signed-off-by: Hidde Beydals <[email protected]>
1 parent 964b2d3 commit 6e0a6f1

File tree

7 files changed

+349
-15
lines changed

7 files changed

+349
-15
lines changed

‎controllers/storage.go

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,17 @@ import (
3333
"time"
3434

3535
securejoin "github.com/cyphar/filepath-securejoin"
36-
3736
"github.com/fluxcd/go-git/v5/plumbing/format/gitignore"
38-
"github.com/fluxcd/pkg/lockedfile"
39-
"github.com/fluxcd/pkg/untar"
37+
digestlib "github.com/opencontainers/go-digest"
4038
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
4139
kerrors "k8s.io/apimachinery/pkg/util/errors"
4240

41+
"github.com/fluxcd/pkg/lockedfile"
4342
"github.com/fluxcd/pkg/sourceignore"
43+
"github.com/fluxcd/pkg/untar"
44+
4445
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
46+
"github.com/fluxcd/source-controller/internal/digest"
4547
sourcefs "github.com/fluxcd/source-controller/internal/fs"
4648
)
4749

@@ -358,9 +360,12 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
358360
}
359361
}()
360362

361-
h := newHash()
363+
md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
364+
if err != nil {
365+
return fmt.Errorf("failed to create digester: %w", err)
366+
}
362367
sz := &writeCounter{}
363-
mw := io.MultiWriter(h, tf, sz)
368+
mw := io.MultiWriter(md, tf, sz)
364369

365370
gw := gzip.NewWriter(mw)
366371
tw := tar.NewWriter(gw)
@@ -450,7 +455,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
450455
return err
451456
}
452457

453-
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
458+
artifact.Digest = md.Digest(digest.Canonical).String()
459+
artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
454460
artifact.LastUpdateTime = metav1.Now()
455461
artifact.Size = &sz.written
456462

@@ -472,9 +478,12 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
472478
}
473479
}()
474480

475-
h := newHash()
481+
md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
482+
if err != nil {
483+
return fmt.Errorf("failed to create digester: %w", err)
484+
}
476485
sz := &writeCounter{}
477-
mw := io.MultiWriter(h, tf, sz)
486+
mw := io.MultiWriter(md, tf, sz)
478487

479488
if _, err := io.Copy(mw, reader); err != nil {
480489
tf.Close()
@@ -492,7 +501,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
492501
return err
493502
}
494503

495-
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
504+
artifact.Digest = md.Digest(digest.Canonical).String()
505+
artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
496506
artifact.LastUpdateTime = metav1.Now()
497507
artifact.Size = &sz.written
498508

@@ -514,9 +524,12 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
514524
}
515525
}()
516526

517-
h := newHash()
527+
md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
528+
if err != nil {
529+
return fmt.Errorf("failed to create digester: %w", err)
530+
}
518531
sz := &writeCounter{}
519-
mw := io.MultiWriter(h, tf, sz)
532+
mw := io.MultiWriter(md, tf, sz)
520533

521534
if _, err := io.Copy(mw, reader); err != nil {
522535
tf.Close()
@@ -530,7 +543,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
530543
return err
531544
}
532545

533-
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
546+
artifact.Digest = md.Digest(digest.Canonical).String()
547+
artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
534548
artifact.LastUpdateTime = metav1.Now()
535549
artifact.Size = &sz.written
536550

‎go.mod

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0
1010
// The util.Walk func was never release as a tag.
1111
replace github.com/go-git/go-billy/v5 => github.com/go-git/go-billy/v5 v5.0.0-20210804024030-7ab80d7c013d
1212

13+
// Replace digest lib to master to gather access to BLAKE3.
14+
// xref: https://github.com/opencontainers/go-digest/pull/66
15+
replace github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be
16+
1317
require (
1418
cloud.google.com/go/storage v1.29.0
1519
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1
@@ -45,6 +49,8 @@ require (
4549
github.com/google/uuid v1.3.0
4650
github.com/minio/minio-go/v7 v7.0.47
4751
github.com/onsi/gomega v1.26.0
52+
github.com/opencontainers/go-digest v1.0.0
53+
github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be
4854
github.com/ory/dockertest/v3 v3.9.1
4955
github.com/otiai10/copy v1.9.0
5056
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
@@ -277,7 +283,6 @@ require (
277283
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
278284
github.com/oklog/ulid v1.3.1 // indirect
279285
github.com/olekukonko/tablewriter v0.0.5 // indirect
280-
github.com/opencontainers/go-digest v1.0.0 // indirect
281286
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
282287
github.com/opencontainers/runc v1.1.2 // indirect
283288
github.com/opentracing/opentracing-go v1.2.0 // indirect
@@ -334,6 +339,7 @@ require (
334339
github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect
335340
github.com/yvasiyarov/gorelic v0.0.7 // indirect
336341
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect
342+
github.com/zeebo/blake3 v0.1.1 // indirect
337343
github.com/zeebo/errs v1.2.2 // indirect
338344
go.etcd.io/bbolt v1.3.6 // indirect
339345
go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 // indirect

‎go.sum

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1253,8 +1253,10 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
12531253
github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q=
12541254
github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
12551255
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
1256-
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
1257-
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
1256+
github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be h1:f2PlhC9pm5sqpBZFvnAoKj+KzXRzbjFMA+TqXfJdgho=
1257+
github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
1258+
github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be h1:yJISmqboKE7zWqC2Nlg3pBkelqCblzZBoMHv2nbrUjQ=
1259+
github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be/go.mod h1:amaK2C3q0MwQTE9OgeDacYr8Qac7uKwICGry1fn3UrI=
12581260
github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
12591261
github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
12601262
github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw=
@@ -1600,8 +1602,14 @@ github.com/yvasiyarov/gorelic v0.0.7/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96Tg
16001602
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM=
16011603
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
16021604
github.com/zalando/go-keyring v0.1.0/go.mod h1:RaxNwUITJaHVdQ0VC7pELPZ3tOWn13nr0gZMZEhpVU0=
1605+
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
1606+
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
1607+
github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs=
1608+
github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4=
16031609
github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=
16041610
github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
1611+
github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0=
1612+
github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
16051613
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
16061614
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
16071615
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -1994,6 +2002,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w
19942002
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
19952003
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
19962004
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
2005+
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
19972006
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
19982007
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
19992008
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

‎internal/digest/digest.go

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
Copyright 2022 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package digest
18+
19+
import (
20+
_ "crypto/sha256"
21+
_ "crypto/sha512"
22+
"fmt"
23+
24+
"github.com/opencontainers/go-digest"
25+
_ "github.com/opencontainers/go-digest/blake3"
26+
)
27+
28+
// Canonical is the primary digest algorithm used to calculate checksums.
29+
const Canonical = digest.SHA256
30+
31+
// AlgorithmForName returns the digest algorithm for the given name, or an
32+
// error of type digest.ErrDigestUnsupported if the algorithm is unavailable.
33+
func AlgorithmForName(name string) (digest.Algorithm, error) {
34+
a := digest.Algorithm(name)
35+
if !a.Available() {
36+
return "", fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, name)
37+
}
38+
return a, nil
39+
}

‎internal/digest/digest_test.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
Copyright 2022 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package digest
18+
19+
import (
20+
"errors"
21+
"testing"
22+
23+
. "github.com/onsi/gomega"
24+
"github.com/opencontainers/go-digest"
25+
)
26+
27+
func TestAlgorithmForName(t *testing.T) {
28+
tests := []struct {
29+
name string
30+
want digest.Algorithm
31+
wantErr error
32+
}{
33+
{
34+
name: "sha256",
35+
want: digest.SHA256,
36+
},
37+
{
38+
name: "sha384",
39+
want: digest.SHA384,
40+
},
41+
{
42+
name: "sha512",
43+
want: digest.SHA512,
44+
},
45+
{
46+
name: "blake3",
47+
want: digest.BLAKE3,
48+
},
49+
{
50+
name: "sha1",
51+
want: SHA1,
52+
},
53+
{
54+
name: "not-available",
55+
wantErr: digest.ErrDigestUnsupported,
56+
},
57+
}
58+
for _, tt := range tests {
59+
t.Run(tt.name, func(t *testing.T) {
60+
g := NewWithT(t)
61+
got, err := AlgorithmForName(tt.name)
62+
if tt.wantErr != nil {
63+
g.Expect(err).To(HaveOccurred())
64+
g.Expect(errors.Is(err, tt.wantErr)).To(BeTrue())
65+
return
66+
}
67+
g.Expect(err).ToNot(HaveOccurred())
68+
g.Expect(got).To(Equal(tt.want))
69+
})
70+
}
71+
}

‎internal/digest/writer.go

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
Copyright 2022 The Flux authors
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package digest
18+
19+
import (
20+
"fmt"
21+
"io"
22+
23+
"github.com/opencontainers/go-digest"
24+
)
25+
26+
// MultiDigester is a digester that writes to multiple digesters to calculate
27+
// the checksum of different algorithms.
28+
type MultiDigester struct {
29+
d map[digest.Algorithm]digest.Digester
30+
}
31+
32+
// NewMultiDigester returns a new MultiDigester that writes to newly
33+
// initialized digesters for the given algorithms. If a provided algorithm is
34+
// not available, it returns a digest.ErrDigestUnsupported error.
35+
func NewMultiDigester(algos ...digest.Algorithm) (*MultiDigester, error) {
36+
d := make(map[digest.Algorithm]digest.Digester, len(algos))
37+
for _, a := range algos {
38+
if _, ok := d[a]; ok {
39+
continue
40+
}
41+
if !a.Available() {
42+
return nil, fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, a)
43+
}
44+
d[a] = a.Digester()
45+
}
46+
return &MultiDigester{d: d}, nil
47+
}
48+
49+
// Write writes p to all underlying digesters.
50+
func (w *MultiDigester) Write(p []byte) (n int, err error) {
51+
for _, d := range w.d {
52+
n, err = d.Hash().Write(p)
53+
if err != nil {
54+
return
55+
}
56+
if n != len(p) {
57+
err = io.ErrShortWrite
58+
return
59+
}
60+
}
61+
return len(p), nil
62+
}
63+
64+
// Digest returns the digest of the data written to the digester of the given
65+
// algorithm, or an empty digest if the algorithm is not available.
66+
func (w *MultiDigester) Digest(algo digest.Algorithm) digest.Digest {
67+
if d, ok := w.d[algo]; ok {
68+
return d.Digest()
69+
}
70+
return ""
71+
}

0 commit comments

Comments
 (0)