Add support for checksums

Author: Genbox

Src/ProviderTests/Multipart/MultipartTests.cs

@@ -1,4 +1,6 @@
-using Genbox.ProviderTests.Misc;
+using System.Security.Cryptography;
+using System.Text;
+using Genbox.ProviderTests.Misc;
 using Genbox.SimpleS3.Core.Abstracts;
 using Genbox.SimpleS3.Core.Common.Helpers;
 using Genbox.SimpleS3.Core.Enums;
@@ -206,7 +208,7 @@ public async Task MultipartUpload(S3Provider provider, string bucket, ISimpleCli
 
         CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, [new S3PartInfo(uploadResp1.ETag, 1), new S3PartInfo(uploadResp2.ETag, 2)]);
         Assert.Equal(200, completeResp.StatusCode);
-        Assert.NotNull(uploadResp2.ETag);
+        Assert.NotNull(completeResp.ETag);
 
         if (provider == S3Provider.AmazonS3)
         {
@@ -292,4 +294,40 @@ public async Task MultipartViaExtensions(S3Provider _, string bucket, ISimpleCli
 
         Assert.Equal(10, count);
     }
+
+    [Theory]
+    [MultipleProviders(S3Provider.AmazonS3)]
+    public async Task MultipartChecksum(S3Provider provider, string bucket, ISimpleClient client)
+    {
+        const string objectKey = nameof(MultipartChecksum);
+
+        CreateMultipartUploadResponse initResp = await client.CreateMultipartUploadAsync(bucket, objectKey, r =>
+        {
+            r.ChecksumAlgorithm = ChecksumAlgorithm.Sha1;
+            r.ChecksumType = ChecksumType.Composite;
+        });
+
+        Assert.Equal(200, initResp.StatusCode);
+        Assert.Equal(ChecksumType.Composite, initResp.ChecksumType);
+        Assert.Equal(ChecksumAlgorithm.Sha1, initResp.ChecksumAlgorithm);
+
+        byte[] data = "hello world"u8.ToArray();
+        await using MemoryStream ms = new MemoryStream(data);
+
+        byte[] checksum = SHA1.HashData(data);
+        UploadPartResponse uploadResp = await client.UploadPartAsync(bucket, objectKey, 1, initResp.UploadId, ms, r =>
+        {
+            r.ChecksumAlgorithm = ChecksumAlgorithm.Sha1;
+            r.Checksum = checksum;
+        });
+        Assert.Equal(200, uploadResp.StatusCode);
+        Assert.Equal(ChecksumAlgorithm.Sha1, uploadResp.ChecksumAlgorithm);
+        Assert.Equal(checksum, uploadResp.Checksum);
+
+        CompleteMultipartUploadResponse completeResp = await client.CompleteMultipartUploadAsync(bucket, objectKey, initResp.UploadId, [new S3PartInfo(uploadResp.ETag, 1, checksum, ChecksumAlgorithm.Sha1)]);
+        Assert.Equal(200, completeResp.StatusCode);
+        Assert.Equal(ChecksumType.Composite, completeResp.ChecksumType);
+        Assert.Equal(ChecksumAlgorithm.Sha1, completeResp.ChecksumAlgorithm);
+        Assert.NotNull(completeResp.Checksum);
+    }
 }

Src/ProviderTests/Objects/HeadObjectTests.cs

@@ -1,6 +1,8 @@
-using Genbox.HttpBuilders.Enums;
+using System.Security.Cryptography;
+using Genbox.HttpBuilders.Enums;
 using Genbox.ProviderTests.Misc;
 using Genbox.SimpleS3.Core.Abstracts;
+using Genbox.SimpleS3.Core.Enums;
 using Genbox.SimpleS3.Core.Extensions;
 using Genbox.SimpleS3.Core.Network.Responses.Objects;
 using Genbox.SimpleS3.Utility.Shared;
@@ -95,4 +97,28 @@ public async Task HeadObjectWebsiteRedirect(S3Provider _, string bucket, ISimple
         Assert.Equal(200, headResp.StatusCode);
         Assert.Equal("https://google.com", headResp.WebsiteRedirectLocation);
     }
+
+    [Theory]
+    [MultipleProviders(S3Provider.AmazonS3)]
+    public async Task HeadObjectChecksum(S3Provider _, string bucket, ISimpleClient client)
+    {
+        byte[] data = [1, 2, 3, 4];
+        byte[] checksum = SHA1.HashData(data);
+
+        PutObjectResponse putResp = await client.PutObjectDataAsync(bucket, nameof(HeadObjectChecksum), data, r =>
+        {
+            r.ChecksumAlgorithm = ChecksumAlgorithm.Sha1;
+            r.Checksum = checksum;
+        });
+
+        Assert.Equal(200, putResp.StatusCode);
+        Assert.Equal(checksum, putResp.Checksum);
+        Assert.Equal(ChecksumType.FullObject, putResp.ChecksumType);
+
+        HeadObjectResponse resp = await client.HeadObjectAsync(bucket, nameof(HeadObjectChecksum), r => r.EnableChecksum = true);
+        Assert.Equal(200, resp.StatusCode);
+        Assert.Equal(ChecksumType.FullObject, resp.ChecksumType);
+        Assert.Equal(ChecksumAlgorithm.Sha1, resp.ChecksumAlgorithm);
+        Assert.Equal(checksum, resp.Checksum);
+    }
 }

Src/ProviderTests/Objects/ListObjectVersionsTests.cs

@@ -1,4 +1,5 @@
 using System.Globalization;
+using System.Security.Cryptography;
 using Genbox.ProviderTests.Misc;
 using Genbox.SimpleS3.Core.Abstracts;
 using Genbox.SimpleS3.Core.Common.Helpers;
@@ -222,4 +223,31 @@ await CreateTempBucketAsync(provider, client, async tempBucket =>
             Assert.Equal(tempObjName, obj.ObjectKey);
         });
     }
+
+    [Theory]
+    [MultipleProviders(S3Provider.AmazonS3)]
+    public async Task ListObjectsChecksum(S3Provider provider, string _, ISimpleClient client)
+    {
+        await CreateTempBucketAsync(provider, client, async tempBucket =>
+        {
+            string tempObjName = "object-" + Guid.NewGuid();
+            byte[] data = "hello world"u8.ToArray();
+            byte[] checksum = SHA1.HashData(data);
+
+            PutObjectResponse putResp = await client.PutObjectDataAsync(tempBucket, tempObjName, data, r =>
+            {
+                r.ChecksumAlgorithm = ChecksumAlgorithm.Sha1;
+                r.Checksum = checksum;
+            });
+
+            Assert.Equal(200, putResp.StatusCode);
+
+            ListObjectVersionsResponse listResp = await client.ListObjectVersionsAsync(tempBucket);
+            Assert.Equal(200, listResp.StatusCode);
+            S3Version obj = Assert.Single(listResp.Versions);
+
+            Assert.Equal(ChecksumAlgorithm.Sha1, obj.ChecksumAlgorithm);
+            Assert.Equal(ChecksumType.FullObject, obj.ChecksumType);
+        });
+    }
 }

Src/ProviderTests/Objects/ListObjectsTests.cs

@@ -1,4 +1,6 @@
 using System.Globalization;
+using System.Security.Cryptography;
+using System.Text;
 using Genbox.ProviderTests.Misc;
 using Genbox.SimpleS3.Core.Abstracts;
 using Genbox.SimpleS3.Core.Common.Helpers;
@@ -178,4 +180,31 @@ await CreateTempBucketAsync(provider, client, async tempBucket =>
             Assert.Equal(tempObjName, obj.ObjectKey);
         });
     }
+
+    [Theory]
+    [MultipleProviders(S3Provider.AmazonS3)]
+    public async Task ListObjectsChecksum(S3Provider provider, string _, ISimpleClient client)
+    {
+        await CreateTempBucketAsync(provider, client, async tempBucket =>
+        {
+            string tempObjName = "object-" + Guid.NewGuid();
+            byte[] data = "hello world"u8.ToArray();
+            byte[] checksum = SHA1.HashData(data);
+
+            PutObjectResponse putResp = await client.PutObjectDataAsync(tempBucket, tempObjName, data, r =>
+            {
+                r.ChecksumAlgorithm = ChecksumAlgorithm.Sha1;
+                r.Checksum = checksum;
+            });
+
+            Assert.Equal(200, putResp.StatusCode);
+
+            ListObjectsResponse listResp = await client.ListObjectsAsync(tempBucket);
+            Assert.Equal(200, listResp.StatusCode);
+            S3Object obj = Assert.Single(listResp.Objects);
+
+            Assert.Equal(ChecksumAlgorithm.Sha1, obj.ChecksumAlgorithm);
+            Assert.Equal(ChecksumType.FullObject, obj.ChecksumType);
+        });
+    }
 }

Src/ProviderTests/Objects/PutObjectTests.cs

@@ -1,4 +1,5 @@
 using System.Globalization;
+using System.Security.Cryptography;
 using Genbox.HttpBuilders.Enums;
 using Genbox.ProviderTests.Misc;
 using Genbox.SimpleS3.Core.Abstracts;
@@ -339,4 +340,28 @@ public async Task PutObjectWebsiteRedirect(S3Provider _, string bucket, ISimpleC
         Assert.Equal(200, resp.StatusCode);
         Assert.Equal("https://google.com", resp.WebsiteRedirectLocation);
     }
+
+    [Theory]
+    [MultipleProviders(S3Provider.AmazonS3)]
+    public async Task PutObjectChecksum(S3Provider _, string bucket, ISimpleClient client)
+    {
+        byte[] data = [1, 2, 3, 4];
+        byte[] checksum = SHA1.HashData(data);
+
+        PutObjectResponse putResp = await client.PutObjectDataAsync(bucket, nameof(PutObjectChecksum), data, r =>
+        {
+            r.ChecksumAlgorithm = ChecksumAlgorithm.Sha1;
+            r.Checksum = checksum;
+        });
+
+        Assert.Equal(200, putResp.StatusCode);
+        Assert.Equal(checksum, putResp.Checksum);
+        Assert.Equal(ChecksumType.FullObject, putResp.ChecksumType);
+
+        GetObjectResponse resp = await client.GetObjectAsync(bucket, nameof(PutObjectChecksum), r => r.EnableChecksum = true);
+        Assert.Equal(200, resp.StatusCode);
+        Assert.Equal(ChecksumType.FullObject, resp.ChecksumType);
+        Assert.Equal(ChecksumAlgorithm.Sha1, resp.ChecksumAlgorithm);
+        Assert.Equal(checksum, resp.Checksum);
+    }
 }

Src/SimpleS3.Core.Common/Misc/AmzHeaders.cs

@@ -37,6 +37,17 @@ public static class AmzHeaders
     public const string XAmzVersionId = "x-amz-version-id";
     public const string XAmzExpiration = "x-amz-expiration";
 
+    public const string XAmzChecksumType = "x-amz-checksum-type";
+    public const string XAmzChecksumMode = "x-amz-checksum-mode";
+
+    public const string XAmzChecksumAlgorithm = "x-amz-checksum-algorithm";
+    public const string XAmzChecksum = "x-amz-checksum-"; //Needs to be combined with XAmzChecksumAlgorithm
+    public const string XAmzChecksumCrc32 = "x-amz-checksum-crc32";
+    public const string XAmzChecksumCrc32C = "x-amz-checksum-crc32c";
+    public const string XAmzChecksumCrc64Nvme = "x-amz-checksum-crc64nvme";
+    public const string XAmzChecksumSha1 = "x-amz-checksum-sha1";
+    public const string XAmzChecksumSha256 = "x-amz-checksum-sha256";
+
     #endregion
 
     #region ServerSideEncryption

Src/SimpleS3.Core/Enums/ChecksumAlgorithm.cs

@@ -0,0 +1,16 @@
+using System.ComponentModel.DataAnnotations;
+using Genbox.FastEnum;
+
+namespace Genbox.SimpleS3.Core.Enums;
+
+/// <summary>Indicates the algorithm that you want Amazon S3 to use to create the checksum for the object. For more information, see https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html</summary>
+[FastEnum]
+public enum ChecksumAlgorithm
+{
+    Unknown = 0,
+    [Display(Name = "CRC32")]Crc32,
+    [Display(Name = "CRC32C")]Crc32C,
+    [Display(Name = "SHA1")]Sha1,
+    [Display(Name = "SHA256")]Sha256,
+    [Display(Name = "CRC64NVME")]Crc64Nvme,
+}

Src/SimpleS3.Core/Enums/ChecksumType.cs

@@ -0,0 +1,13 @@
+using System.ComponentModel.DataAnnotations;
+using Genbox.FastEnum;
+
+namespace Genbox.SimpleS3.Core.Enums;
+
+/// <summary>Indicates the checksum type that you want Amazon S3 to use to calculate the object’s checksum value. For more information, see https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html</summary>
+[FastEnum]
+public enum ChecksumType
+{
+    Unknown = 0,
+    [Display(Name = "COMPOSITE")]Composite,
+    [Display(Name = "FULL_OBJECT")]FullObject
+}

Src/SimpleS3.Core/Internals/Fluent/Upload.cs

@@ -10,6 +10,7 @@
 using Genbox.SimpleS3.Core.Common.Misc;
 using Genbox.SimpleS3.Core.Common.Validation;
 using Genbox.SimpleS3.Core.Enums;
+using Genbox.SimpleS3.Core.Network.Requests.Multipart;
 using Genbox.SimpleS3.Core.Network.Requests.Objects;
 using Genbox.SimpleS3.Core.Network.Responses.Multipart;
 using Genbox.SimpleS3.Core.Network.Responses.Objects;
@@ -190,10 +191,30 @@ public IUpload RemoveLegalHold()
 
     public Task<CompleteMultipartUploadResponse> UploadMultipartAsync(Stream data, CancellationToken token = default)
     {
-        _request.Method = HttpMethodType.POST;
-        _request.Content = null;
-
-        return _multipartTransfer.MultipartUploadAsync(_request, data, token: token);
+        //Create a request and copy over all the supported properties
+        CreateMultipartUploadRequest req = new CreateMultipartUploadRequest(_request.BucketName, _request.ObjectKey);
+        req.CacheControl = _request.CacheControl;
+        req.ContentDisposition = _request.ContentDisposition;
+        req.ContentType = _request.ContentType;
+        req.ContentEncoding = _request.ContentEncoding;
+        req.SseAlgorithm = _request.SseAlgorithm;
+        req.SseKmsKeyId = _request.SseKmsKeyId;
+        req.SseCustomerAlgorithm = _request.SseCustomerAlgorithm;
+        req.SseCustomerKey = _request.SseCustomerKey;
+        req.SseCustomerKeyMd5 = _request.SseCustomerKeyMd5;
+        req.Metadata = _request.Metadata;
+        req.StorageClass = _request.StorageClass;
+        req.Tags = _request.Tags;
+        req.Acl = _request.Acl;
+        req.AclGrantRead = _request.AclGrantRead;
+        req.AclGrantReadAcp = _request.AclGrantReadAcp;
+        req.AclGrantWriteAcp = _request.AclGrantWriteAcp;
+        req.AclGrantFullControl = _request.AclGrantFullControl;
+        req.LockMode = _request.LockMode;
+        req.LockRetainUntil = _request.LockRetainUntil;
+        req.LockLegalHold = _request.LockLegalHold;
+
+        return _multipartTransfer.MultipartUploadAsync(req, data, token: token);
     }
 
     public Task<PutObjectResponse> UploadAsync(Stream? data, CancellationToken token = default)

Src/SimpleS3.Core/Internals/Marshallers/Requests/GenericRequestMapper.cs

@@ -144,5 +144,17 @@ public static void Map<T>(T req) where T : IRequest
 
         if (req is IHasExpectedBucketOwner expectedBucketOwner && expectedBucketOwner.ExpectedBucketOwner != null)
             req.SetHeader(AmzHeaders.XAmzExpectedBucketOwner, expectedBucketOwner.ExpectedBucketOwner);
+
+        if (req is IHasChecksum checksum)
+        {
+            if (checksum.ChecksumType != ChecksumType.Unknown)
+                req.SetHeader(AmzHeaders.XAmzChecksumType, checksum.ChecksumType.GetDisplayName());
+
+            if (checksum.ChecksumAlgorithm != ChecksumAlgorithm.Unknown)
+                req.SetHeader(AmzHeaders.XAmzChecksumAlgorithm, checksum.ChecksumAlgorithm.GetDisplayName());
+        }
+
+        if (req is IHasChecksumProperties checksumProps && checksumProps.ChecksumAlgorithm != ChecksumAlgorithm.Unknown && checksumProps.Checksum != null)
+            req.SetHeader(AmzHeaders.XAmzChecksum + checksumProps.ChecksumAlgorithm.ToString().ToLowerInvariant(), Convert.ToBase64String(checksumProps.Checksum));
     }
 }