Add support for UploadChunk and parallel uploads in Upload Api

Author: const-cloudinary

CloudinaryDotNet.IntegrationTests/IntegrationTestBase.cs

@@ -180,7 +180,7 @@ private void SaveTestResources(Assembly assembly)
             SaveEmbeddedToDisk(assembly, TEST_PDF, m_testPdfPath);
         }
 
-        private void SaveEmbeddedToDisk(Assembly assembly, string sourcePath, string destPath)
+        private static void SaveEmbeddedToDisk(Assembly assembly, string sourcePath, string destPath)
         {
             try
             {
@@ -206,6 +206,34 @@ private void SaveEmbeddedToDisk(Assembly assembly, string sourcePath, string des
             }
         }
 
+
+        protected List<string> SplitFile(string sourceFile, int chunkSize, string suffix = "")
+        {
+            var chunks = new List<string>();
+
+            var baseName = Path.GetFileNameWithoutExtension(sourceFile);
+            var extension = Path.GetExtension(sourceFile);
+            var buffer = new byte[chunkSize];
+            using (var fs = new FileStream(sourceFile, FileMode.Open, FileAccess.Read))
+            {
+                int read;
+                var currChunkNum = 0;
+                while ((read = fs.Read(buffer, 0, buffer.Length)) > 0)
+                {
+                    var path = $"{Path.GetDirectoryName(sourceFile)}/{baseName}.{currChunkNum}{suffix}{extension}.tmp";
+                    using (var outputFile = new FileStream(path, FileMode.Create, FileAccess.Write))
+                    {
+                        outputFile.Write(buffer, 0, read);
+                    }
+
+                    chunks.Add(path);
+                    currChunkNum++;
+                }
+            }
+
+            return chunks;
+        }
+
         /// <summary>
         /// A convenient method for uploading an image before testing.
         /// </summary>

CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
@@ -7,6 +8,7 @@
 using System.Net;
 using System.Threading.Tasks;
 using CloudinaryDotNet.Actions;
+using CloudinaryDotNet.Core;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NUnit.Framework;
@@ -260,7 +262,7 @@ public void TestModerationManual()
             Assert.AreEqual(MODERATION_MANUAL, uploadResult.Moderation[0].Kind);
             Assert.AreEqual(ModerationStatus.Pending, uploadResult.Moderation[0].Status);
 
-            var getResult = m_cloudinary.GetResource(uploadResult.PublicId);
+            var getResult = m_cloudinary.GetResourceByAssetId(uploadResult.AssetId);
 
             Assert.NotNull(getResult);
             Assert.NotNull(getResult.Moderation, getResult.Error?.Message);
@@ -561,7 +563,7 @@ public NonSeekableStream(byte[] buffer) : base(buffer) { }
         [Test, RetryWithDelay]
         public void TestUploadLargeNonSeekableStream()
         {
-            byte[] bytes = File.ReadAllBytes(m_testLargeImagePath);
+            var bytes = File.ReadAllBytes(m_testLargeImagePath);
             const string streamed = "stream_non_seekable";
 
             using (var memoryStream = new NonSeekableStream(bytes))
@@ -593,15 +595,15 @@ public void TestUploadLargeRawFiles()
         }
 
         [Test, RetryWithDelay]
-        public async Task TestUploadLargeRawFilesAsync()
+        public async Task TestUploadLargeRawFilesAsyncInParallel()
         {
             // support asynchronous uploading large raw files
             var largeFilePath = m_testLargeImagePath;
             int largeFileLength = (int)new FileInfo(largeFilePath).Length;
 
             var uploadParams = GetUploadLargeRawParams(largeFilePath);
 
-            var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);
+            var result = await m_cloudinary.UploadLargeAsync<RawUploadResult>(uploadParams, TEST_CHUNK_SIZE, 2);
 
             AssertUploadLarge(result, largeFileLength);
         }
@@ -617,6 +619,7 @@ private RawUploadParams GetUploadLargeRawParams(string path)
 
         private void AssertUploadLarge(RawUploadResult result, int fileLength)
         {
+            Assert.NotNull(result);
             Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
         }
 
@@ -655,6 +658,172 @@ public async Task TestUploadLargeAutoFilesAsync()
 
             Assert.AreEqual("image", result.ResourceType);
         }
+
+        [Test, RetryWithDelay]
+        public void TestUploadChunkSingleStream()
+        {
+            var largeFilePath = m_testLargeImagePath;
+            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
+
+            ImageUploadResult result = null;
+
+            using (var currChunk = new MemoryStream())
+            {
+                var uploadParams =  new ImageUploadParams()
+                {
+                    File = new FileDescription($"ImageFromChunks_{GetTaggedRandomValue()}", currChunk),
+                    Tags = m_apiTag
+                };
+
+                var buffer = new byte[TEST_CHUNK_SIZE];
+
+                using (var source = File.Open(largeFilePath, FileMode.Open))
+                {
+                    int read;
+                    while ((read = source.Read(buffer, 0, buffer.Length)) > 0)
+                    {
+                        currChunk.Seek(0, SeekOrigin.End);
+                        currChunk.Write(buffer, 0, read);
+
+                        // Need to specify whether the chunk is the last one in order to finish the upload.
+                        uploadParams.File.LastChunk = read != TEST_CHUNK_SIZE;
+
+                        result = m_cloudinary.UploadChunk(uploadParams);
+                    }
+                }
+            }
+
+            AssertUploadLarge(result, largeFileLength);
+            Assert.AreEqual("image", result?.ResourceType);
+        }
+
+        [Test, RetryWithDelay]
+        public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()
+        {
+            var largeFilePath = m_testLargeImagePath;
+            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
+
+            ImageUploadResult result = null;
+
+            var uploadParams =  new ImageUploadParams()
+            {
+                // File path will be ignored, since we use streams.
+                File = new FileDescription($"ImageFromMultipleChunks_{GetTaggedRandomValue()}", true),
+                Tags = m_apiTag
+            };
+
+            var buffer = new byte[TEST_CHUNK_SIZE];
+
+            using (var source = File.Open(largeFilePath, FileMode.Open))
+            {
+                int read;
+                while ((read = source.Read(buffer, 0, buffer.Length)) > 0)
+                {
+                    var currChunk = new MemoryStream(buffer);
+                    // Set current chunk
+                    uploadParams.File.AddChunk(currChunk, source.Position - read, read, read != TEST_CHUNK_SIZE);
+
+                    result = await m_cloudinary.UploadChunkAsync(uploadParams);
+                }
+            }
+
+            AssertUploadLarge(result, largeFileLength);
+            Assert.AreEqual("image", result?.ResourceType);
+        }
+
+        [Test, RetryWithDelay]
+        public void TestUploadChunkMultipleFileParts()
+        {
+            var largeFilePath = m_testLargeImagePath;
+            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
+
+            ImageUploadResult result = null;
+
+            var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple");
+
+            var uploadParams =  new ImageUploadParams()
+            {
+                File = new FileDescription($"ImageFromFileChunks_{GetTaggedRandomValue()}", true),
+                Tags = m_apiTag,
+            };
+            try
+            {
+                foreach (var chunk in fileChunks)
+                {
+                    // Set file path of the current chunk.
+                    uploadParams.File.AddChunk(chunk, fileChunks.IndexOf(chunk) == fileChunks.Count - 1);
+                    // Need to specify whether the chunk is the last one in order to finish the upload.
+
+                    result = m_cloudinary.UploadChunk(uploadParams);
+                }
+            }
+            finally
+            {
+                uploadParams.File.Dispose();
+                foreach (var chunk in fileChunks)
+                {
+                    try
+                    {
+                        File.Delete(chunk);
+                    }
+                    catch (IOException)
+                    {
+                        // nothing to do
+                    }
+                }
+            }
+
+            AssertUploadLarge(result, largeFileLength);
+            Assert.AreEqual("image", result?.ResourceType);
+        }
+
+        [Test, RetryWithDelay]
+        public void TestUploadChunkMultipleFilePartsInParallel()
+        {
+            var largeFilePath = m_testLargeImagePath;
+            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
+
+            var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple_parallel");
+
+            var uploadParams =  new RawUploadParams()
+            {
+                File = new FileDescription($"ImageFromFileChunks_{GetTaggedRandomValue()}", true),
+                Tags = m_apiTag
+            };
+
+            var resultCollection = new ConcurrentBag<RawUploadResult>();
+
+            uploadParams.File.AddChunks(fileChunks);
+
+            try
+            {
+                Parallel.For(0, fileChunks.Count, new ParallelOptions { MaxDegreeOfParallelism = 2 },chunkNum =>
+                {
+                    resultCollection.Add(m_cloudinary.UploadChunk(uploadParams));
+                });
+            }
+            finally
+            {
+                uploadParams.File.Dispose();
+                foreach (var chunk in fileChunks)
+                {
+                    try
+                    {
+                        File.Delete(chunk);
+                    }
+                    catch (IOException)
+                    {
+                        // nothing to do
+                    }
+                }
+            }
+
+            var uploadResult = resultCollection.FirstOrDefault(r => r.AssetId != null);
+
+            AssertUploadLarge(uploadResult, largeFileLength);
+            Assert.AreEqual("raw", uploadResult?.ResourceType);
+        }
+
         /// <summary>
         /// Test access control rules
         /// </summary>

CloudinaryDotNet.Tests/Asset/UrlBuilderTest.cs

@@ -485,13 +485,13 @@ public void TestExcludeEmptyTransformation()
         [Test]
         public void TestAgentPlatformHeaders()
         {
-            var request = new HttpRequestMessage { RequestUri = new Uri("http://dummy.com") };
+            var request = new HttpRequestMessage { RequestUri = new Uri("https://dummy.com") };
             m_api.UserPlatform = "Test/1.0";
 
-            m_api.PrepareRequestBody(
+            m_api.PrepareRequestBodyAsync(
                 request,
                 HttpMethod.GET,
-                new SortedDictionary<string, object>());
+                new SortedDictionary<string, object>()).GetAwaiter().GetResult();
 
             //Can't test the result, so we just verify the UserAgent parameter is sent to the server
             StringAssert.AreEqualIgnoringCase($"{m_api.UserPlatform} {ApiShared.USER_AGENT}",

CloudinaryDotNet/Actions/AssetsUpload/BasicRawUploadParams.cs

@@ -58,6 +58,11 @@ public virtual ResourceType ResourceType
         /// </summary>
         public string Signature { get; set; }
 
+        /// <summary>
+        /// Gets or sets unique upload ID.
+        /// </summary>
+        public string UniqueUploadId { get; set; }
+
         /// <summary>
         /// Validate object model.
         /// </summary>
@@ -68,7 +73,7 @@ public override void Check()
                 throw new ArgumentException("File must be specified in UploadParams!");
             }
 
-            if (!File.IsRemote && File.Stream == null && string.IsNullOrEmpty(File.FilePath))
+            if (!File.Chunked && !File.IsRemote && File.Stream == null && string.IsNullOrEmpty(File.FilePath))
             {
                 throw new ArgumentException("File is not ready!");
             }

CloudinaryDotNet/Actions/AssetsUpload/ExplicitResult.cs

@@ -25,12 +25,6 @@ public class ExplicitResult : RawUploadResult
         /// </summary>
         public List<ResponsiveBreakpointList> ResponsiveBreakpoints { get; set; }
 
-        /// <summary>
-        /// Gets or sets a status that is returned when passing 'Async' argument set to 'true' to the server.
-        /// </summary>
-        [DataMember(Name = "status")]
-        public string Status { get; set; }
-
         /// <summary>
         /// Gets or sets any requested information from executing one of the Cloudinary Add-ons on the media asset.
         /// </summary>

CloudinaryDotNet/Actions/AssetsUpload/UploadResult.cs

@@ -102,6 +102,12 @@ public long Length
         [DataMember(Name = "metadata")]
         public JToken MetadataFields { get; set; }
 
+        /// <summary>
+        /// Gets or sets a status that is returned when passing 'Async' argument set to 'true' to the server.
+        /// </summary>
+        [DataMember(Name = "status")]
+        public string Status { get; set; }
+
         /// <summary>
         /// Gets or sets upload hook execution status.
         /// </summary>

CloudinaryDotNet/Actions/AssetsUpload/VideoUploadParams.cs

@@ -8,9 +8,6 @@ public class VideoUploadParams : ImageUploadParams
         /// <summary>
         /// Gets get the type of video asset for upload.
         /// </summary>
-        public override ResourceType ResourceType
-        {
-            get { return Actions.ResourceType.Video; }
-        }
+        public override ResourceType ResourceType => ResourceType.Video;
     }
 }