Skip to content

Commit 02b7d04

Browse files
lucix-awslucix-aws
authored
support crc64nvme in flex checksum (#3002)
1 parent 7995348 commit 02b7d04

File tree

7 files changed

+228
-37
lines changed

7 files changed

+228
-37
lines changed

‎.changelog/f007dcb88454412692915fadefdcf8cc.json

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
{
2+
"id": "f007dcb8-8454-4126-9291-5fadefdcf8cc",
3+
"type": "feature",
4+
"description": "Support CRC64NVME flex checksums.",
5+
"modules": [
6+
"service/internal/checksum",
7+
"service/internal/integrationtest"
8+
]
9+
}

‎feature/s3/manager/integ_upload_test.go

Lines changed: 86 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import (
1515
"fmt"
1616
"hash"
1717
"hash/crc32"
18+
"hash/crc64"
1819
"io"
1920
"log"
2021
"reflect"
@@ -29,6 +30,8 @@ import (
2930
"github.com/aws/smithy-go/middleware"
3031
)
3132

33+
const crc64NVME = 0x9a6c_9329_ac4b_c9b5
34+
3235
var integBuf12MB = make([]byte, 1024*1024*12)
3336
var integMD512MB = fmt.Sprintf("%x", md5.Sum(integBuf12MB))
3437

@@ -124,6 +127,7 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
124127
singlePartBytes := integBuf12MB[0:manager.DefaultUploadPartSize]
125128
singlePartCRC32 := base64Sum(crc32.NewIEEE(), singlePartBytes)
126129
singlePartCRC32C := base64Sum(crc32.New(crc32.MakeTable(crc32.Castagnoli)), singlePartBytes)
130+
singlePartCRC64NVME := base64Sum(crc64.New(crc64.MakeTable(crc64NVME)), singlePartBytes)
127131
singlePartSHA1 := base64Sum(sha1.New(), singlePartBytes)
128132
singlePartSHA256 := base64Sum(sha256.New(), singlePartBytes)
129133
singlePartMD5 := base64Sum(md5.New(), singlePartBytes)
@@ -132,6 +136,7 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
132136
multiPartTailBytes := integBuf12MB[manager.DefaultUploadPartSize*2:]
133137
multiPartTailCRC32 := base64Sum(crc32.NewIEEE(), multiPartTailBytes)
134138
multiPartTailCRC32C := base64Sum(crc32.New(crc32.MakeTable(crc32.Castagnoli)), multiPartTailBytes)
139+
multiPartTailCRC64NVME := base64Sum(crc64.New(crc64.MakeTable(crc64NVME)), multiPartTailBytes)
135140
multiPartTailSHA1 := base64Sum(sha1.New(), multiPartTailBytes)
136141
multiPartTailSHA256 := base64Sum(sha256.New(), multiPartTailBytes)
137142
multiPartTailETag := fmt.Sprintf("%q", hexSum(md5.New(), multiPartTailBytes))
@@ -143,20 +148,25 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
143148
multiPartSHA256 := base64SumOfSums(sha256.New(), []string{singlePartSHA256, singlePartSHA256, multiPartTailSHA256})
144149
multiPartETag := `"4e982d58b6c2ce178ae042c23f9bca6e-3"` // Not obvious how this is computed
145150

151+
// s3 wants running checksum for crc64nvme
152+
multiPartCRC64NVME := base64Sum(crc64.New(crc64.MakeTable(crc64NVME)), integBuf12MB)
153+
146154
cases := map[string]map[string]struct {
147-
algorithm s3types.ChecksumAlgorithm
148-
payload io.Reader
149-
checksumCRC32 string
150-
checksumCRC32C string
151-
checksumSHA1 string
152-
checksumSHA256 string
153-
contentMD5 string
154-
expectParts []s3types.CompletedPart
155-
expectChecksumCRC32 string
156-
expectChecksumCRC32C string
157-
expectChecksumSHA1 string
158-
expectChecksumSHA256 string
159-
expectETag string
155+
algorithm s3types.ChecksumAlgorithm
156+
payload io.Reader
157+
checksumCRC32 string
158+
checksumCRC32C string
159+
checksumCRC64NVME string
160+
checksumSHA1 string
161+
checksumSHA256 string
162+
contentMD5 string
163+
expectParts []s3types.CompletedPart
164+
expectChecksumCRC32 string
165+
expectChecksumCRC32C string
166+
expectChecksumCRC64NVME string
167+
expectChecksumSHA1 string
168+
expectChecksumSHA256 string
169+
expectETag string
160170
}{
161171
"auto single part": {
162172
"no checksum algorithm passed": {
@@ -176,6 +186,12 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
176186
expectChecksumCRC32C: singlePartCRC32C,
177187
expectETag: singlePartETag,
178188
},
189+
"CRC64NVME": {
190+
algorithm: s3types.ChecksumAlgorithmCrc64nvme,
191+
payload: bytes.NewReader(singlePartBytes),
192+
expectChecksumCRC64NVME: singlePartCRC64NVME,
193+
expectETag: singlePartETag,
194+
},
179195
"SHA1": {
180196
algorithm: s3types.ChecksumAlgorithmSha1,
181197
payload: bytes.NewReader(singlePartBytes),
@@ -202,6 +218,12 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
202218
expectChecksumCRC32C: singlePartCRC32C,
203219
expectETag: singlePartETag,
204220
},
221+
"CRC64NVME": {
222+
payload: bytes.NewReader(singlePartBytes),
223+
checksumCRC64NVME: singlePartCRC64NVME,
224+
expectChecksumCRC64NVME: singlePartCRC64NVME,
225+
expectETag: singlePartETag,
226+
},
205227
"SHA1": {
206228
payload: bytes.NewReader(singlePartBytes),
207229
checksumSHA1: singlePartSHA1,
@@ -290,6 +312,29 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
290312
expectChecksumCRC32C: multiPartCRC32C,
291313
expectETag: multiPartETag,
292314
},
315+
"CRC64NVME": {
316+
algorithm: s3types.ChecksumAlgorithmCrc64nvme,
317+
payload: bytes.NewReader(multiPartBytes),
318+
expectParts: []s3types.CompletedPart{
319+
{
320+
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
321+
ETag: aws.String(singlePartETag),
322+
PartNumber: aws.Int32(1),
323+
},
324+
{
325+
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
326+
ETag: aws.String(singlePartETag),
327+
PartNumber: aws.Int32(2),
328+
},
329+
{
330+
ChecksumCRC64NVME: aws.String(multiPartTailCRC64NVME),
331+
ETag: aws.String(multiPartTailETag),
332+
PartNumber: aws.Int32(3),
333+
},
334+
},
335+
expectChecksumCRC64NVME: multiPartCRC64NVME,
336+
expectETag: multiPartETag,
337+
},
293338
"SHA1": {
294339
algorithm: s3types.ChecksumAlgorithmSha1,
295340
payload: bytes.NewReader(multiPartBytes),
@@ -386,6 +431,30 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
386431
expectChecksumCRC32C: multiPartCRC32C,
387432
expectETag: multiPartETag,
388433
},
434+
"CRC64NVME": {
435+
algorithm: s3types.ChecksumAlgorithmCrc64nvme,
436+
payload: bytes.NewReader(multiPartBytes),
437+
checksumCRC64NVME: multiPartCRC64NVME,
438+
expectParts: []s3types.CompletedPart{
439+
{
440+
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
441+
ETag: aws.String(singlePartETag),
442+
PartNumber: aws.Int32(1),
443+
},
444+
{
445+
ChecksumCRC64NVME: aws.String(singlePartCRC64NVME),
446+
ETag: aws.String(singlePartETag),
447+
PartNumber: aws.Int32(2),
448+
},
449+
{
450+
ChecksumCRC64NVME: aws.String(multiPartTailCRC64NVME),
451+
ETag: aws.String(multiPartTailETag),
452+
PartNumber: aws.Int32(3),
453+
},
454+
},
455+
expectChecksumCRC64NVME: multiPartCRC64NVME,
456+
expectETag: multiPartETag,
457+
},
389458
"SHA1": {
390459
algorithm: s3types.ChecksumAlgorithmSha1,
391460
payload: bytes.NewReader(multiPartBytes),
@@ -449,6 +518,7 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
449518
ChecksumAlgorithm: c.algorithm,
450519
ChecksumCRC32: toStringPtr(c.checksumCRC32),
451520
ChecksumCRC32C: toStringPtr(c.checksumCRC32C),
521+
ChecksumCRC64NVME: toStringPtr(c.checksumCRC64NVME),
452522
ChecksumSHA1: toStringPtr(c.checksumSHA1),
453523
ChecksumSHA256: toStringPtr(c.checksumSHA256),
454524
ContentMD5: toStringPtr(c.contentMD5),
@@ -467,6 +537,9 @@ func TestInteg_UploadPresetChecksum(t *testing.T) {
467537
if e, a := c.expectChecksumCRC32C, aws.ToString(out.ChecksumCRC32C); e != a {
468538
t.Errorf("expect %v CRC32C checksum, got %v", e, a)
469539
}
540+
if e, a := c.expectChecksumCRC64NVME, aws.ToString(out.ChecksumCRC64NVME); e != a {
541+
t.Errorf("expect %v CRC64NVME checksum, got %v", e, a)
542+
}
470543
if e, a := c.expectChecksumSHA1, aws.ToString(out.ChecksumSHA1); e != a {
471544
t.Errorf("expect %v SHA1 checksum, got %v", e, a)
472545
}

‎feature/s3/manager/upload.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,9 @@ type UploadOutput struct {
121121
// The base64-encoded, 32-bit CRC32C checksum of the object.
122122
ChecksumCRC32C *string
123123

124+
// The base64-encoded, 64-bit CRC64NVME checksum of the object.
125+
ChecksumCRC64NVME *string
126+
124127
// The base64-encoded, 160-bit SHA-1 digest of the object.
125128
ChecksumSHA1 *string
126129

@@ -511,6 +514,7 @@ func (u *uploader) singlePart(r io.ReadSeeker, cleanup func()) (*UploadOutput, e
511514
BucketKeyEnabled: aws.ToBool(out.BucketKeyEnabled),
512515
ChecksumCRC32: out.ChecksumCRC32,
513516
ChecksumCRC32C: out.ChecksumCRC32C,
517+
ChecksumCRC64NVME: out.ChecksumCRC64NVME,
514518
ChecksumSHA1: out.ChecksumSHA1,
515519
ChecksumSHA256: out.ChecksumSHA256,
516520
ETag: out.ETag,
@@ -653,6 +657,7 @@ func (u *multiuploader) upload(firstBuf io.ReadSeeker, cleanup func()) (*UploadO
653657
BucketKeyEnabled: aws.ToBool(completeOut.BucketKeyEnabled),
654658
ChecksumCRC32: completeOut.ChecksumCRC32,
655659
ChecksumCRC32C: completeOut.ChecksumCRC32C,
660+
ChecksumCRC64NVME: completeOut.ChecksumCRC64NVME,
656661
ChecksumSHA1: completeOut.ChecksumSHA1,
657662
ChecksumSHA256: completeOut.ChecksumSHA256,
658663
ETag: completeOut.ETag,
@@ -764,6 +769,8 @@ func (u *multiuploader) initChecksumAlgorithm() {
764769
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32
765770
case u.in.ChecksumCRC32C != nil:
766771
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc32c
772+
case u.in.ChecksumCRC64NVME != nil:
773+
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmCrc64nvme
767774
case u.in.ChecksumSHA1 != nil:
768775
u.in.ChecksumAlgorithm = types.ChecksumAlgorithmSha1
769776
case u.in.ChecksumSHA256 != nil:

‎service/internal/checksum/algorithms.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"fmt"
1010
"hash"
1111
"hash/crc32"
12+
"hash/crc64"
1213
"io"
1314
"strings"
1415
"sync"
@@ -35,11 +36,15 @@ const (
3536
AlgorithmCRC64NVME Algorithm = "CRC64NVME"
3637
)
3738

39+
// inverted NVME polynomial as required by crc64.MakeTable
40+
const crc64NVME = 0x9a6c_9329_ac4b_c9b5
41+
3842
var supportedAlgorithms = []Algorithm{
3943
AlgorithmCRC32C,
4044
AlgorithmCRC32,
4145
AlgorithmSHA1,
4246
AlgorithmSHA256,
47+
AlgorithmCRC64NVME,
4348
}
4449

4550
func (a Algorithm) String() string { return string(a) }
@@ -92,6 +97,8 @@ func NewAlgorithmHash(v Algorithm) (hash.Hash, error) {
9297
return crc32.NewIEEE(), nil
9398
case AlgorithmCRC32C:
9499
return crc32.New(crc32.MakeTable(crc32.Castagnoli)), nil
100+
case AlgorithmCRC64NVME:
101+
return crc64.New(crc64.MakeTable(crc64NVME)), nil
95102
default:
96103
return nil, fmt.Errorf("unknown checksum algorithm, %v", v)
97104
}
@@ -109,6 +116,8 @@ func AlgorithmChecksumLength(v Algorithm) (int, error) {
109116
return crc32.Size, nil
110117
case AlgorithmCRC32C:
111118
return crc32.Size, nil
119+
case AlgorithmCRC64NVME:
120+
return crc64.Size, nil
112121
default:
113122
return 0, fmt.Errorf("unknown checksum algorithm, %v", v)
114123
}

‎service/internal/checksum/algorithms_test.go

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"encoding/base64"
1111
"fmt"
1212
"hash/crc32"
13+
"hash/crc64"
1314
"io"
1415
"io/ioutil"
1516
"strings"
@@ -69,6 +70,13 @@ func TestComputeChecksumReader(t *testing.T) {
6970
ExpectRead: "hello world",
7071
ExpectChecksum: "uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=",
7172
},
73+
"crc64nvme": {
74+
Input: strings.NewReader("hello world"),
75+
Algorithm: AlgorithmCRC64NVME,
76+
ExpectChecksumLen: base64.StdEncoding.EncodedLen(crc64.Size),
77+
ExpectRead: "hello world",
78+
ExpectChecksum: "jSnVw/bqjr4=",
79+
},
7280
}
7381

7482
for name, c := range cases {
@@ -392,12 +400,13 @@ func TestFilterSupportedAlgorithms(t *testing.T) {
392400
},
393401
},
394402
"mixed case": {
395-
values: []string{"Crc32", "cRc32c", "shA1", "sHA256"},
403+
values: []string{"Crc32", "cRc32c", "shA1", "sHA256", "crc64nvme"},
396404
expectAlgorithms: []Algorithm{
397405
AlgorithmCRC32,
398406
AlgorithmCRC32C,
399407
AlgorithmSHA1,
400408
AlgorithmSHA256,
409+
AlgorithmCRC64NVME,
401410
},
402411
},
403412
}
@@ -442,6 +451,10 @@ func TestAlgorithmChecksumLength(t *testing.T) {
442451
algorithm: AlgorithmSHA256,
443452
expectLength: sha256.Size,
444453
},
454+
"crc64nvme": {
455+
algorithm: AlgorithmCRC64NVME,
456+
expectLength: crc64.Size,
457+
},
445458
}
446459

447460
for name, c := range cases {

0 commit comments

Comments
 (0)