Fix UploadLarge breaking large files on upload

const-cloudinary · const-cloudinary · commit c4d777b8dc26 · 2024-03-14T19:07:49.000+02:00
diff --git a/CloudinaryDotNet.IntegrationTests/IntegrationTestBase.cs b/CloudinaryDotNet.IntegrationTests/IntegrationTestBase.cs
@@ -4,6 +4,7 @@
 using System.Linq;
 using System.Net.Http;
 using System.Reflection;
+using System.Security.Cryptography;
 using System.Threading.Tasks;
 using CloudinaryDotNet.Actions;
 using NUnit.Framework;
@@ -507,6 +508,18 @@ protected string GetUniqueMetadataFieldLabel(string suffix = "")
             return label;
         }
 
+        protected static string GetFileMd5Sum(string filename)
+        {
+            using (var md5 = MD5.Create())
+            {
+                using (var stream = File.OpenRead(filename))
+                {
+                    var hash = md5.ComputeHash(stream);
+                    return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
+                }
+            }
+        }
+
         [OneTimeTearDown]
         public virtual void Cleanup()
         {
diff --git a/CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs b/CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs
@@ -40,6 +40,9 @@ public class UploadMethodsTest : IntegrationTestBase
 
         protected string m_implicitTransformationText;
 
+        protected string m_largeFileEtag;
+        protected int m_largeFileLength;
+
         public override void Initialize()
         {
             base.Initialize();
@@ -52,6 +55,9 @@ public override void Initialize()
                 m_transformationAr25, m_transformationAr69,
                 m_transformationAr30, m_transformationAr12,
                 m_eagerTransformation);
+
+            m_largeFileEtag = GetFileMd5Sum(m_testLargeImagePath);
+            m_largeFileLength = (int)new FileInfo(m_testLargeImagePath).Length;
         }
 
         [Test, RetryWithDelay]
@@ -576,7 +582,7 @@ public void TestUploadLargeNonSeekableStream()
 
                 var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);
 
-                AssertUploadLarge(result, bytes.Length);
+                AssertUploadLarge(result);
             }
         }
 
@@ -585,27 +591,25 @@ public void TestUploadLargeRawFiles()
         {
             // support uploading large raw files
             var largeFilePath = m_testLargeImagePath;
-            int largeFileLength = (int)new FileInfo(largeFilePath).Length;
 
             var uploadParams = GetUploadLargeRawParams(largeFilePath);
 
             var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);
 
-            AssertUploadLarge(result, largeFileLength);
+            AssertUploadLarge(result);
         }
 
         [Test, RetryWithDelay]
         public async Task TestUploadLargeRawFilesAsyncInParallel()
         {
             // support asynchronous uploading large raw files
             var largeFilePath = m_testLargeImagePath;
-            int largeFileLength = (int)new FileInfo(largeFilePath).Length;
 
             var uploadParams = GetUploadLargeRawParams(largeFilePath);
 
             var result = await m_cloudinary.UploadLargeAsync<RawUploadResult>(uploadParams, TEST_CHUNK_SIZE, 2);
 
-            AssertUploadLarge(result, largeFileLength);
+            AssertUploadLarge(result);
         }
 
         private RawUploadParams GetUploadLargeRawParams(string path)
@@ -617,54 +621,53 @@ private RawUploadParams GetUploadLargeRawParams(string path)
             };
         }
 
-        private void AssertUploadLarge(RawUploadResult result, int fileLength)
+        private void AssertUploadLarge(RawUploadResult result, int fileLength = 0, string etag = null)
         {
+            if (fileLength == 0)
+            {
+                fileLength = m_largeFileLength;
+            }
+            if (etag == null)
+            {
+                etag = m_largeFileEtag;
+            }
             Assert.NotNull(result);
             Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
+            Assert.AreEqual(etag, result.Etag, result.Error?.Message);
         }
 
         [Test, RetryWithDelay]
         public void TestUploadLarge()
         {
             // support uploading large image
-
-            var largeFilePath = m_testLargeImagePath;
-            int fileLength = (int)new FileInfo(largeFilePath).Length;
             var result = m_cloudinary.UploadLarge(new ImageUploadParams()
             {
-                File = new FileDescription(largeFilePath),
+                File = new FileDescription(m_testLargeImagePath),
                 Tags = m_apiTag
             }, TEST_CHUNK_SIZE);
 
-            Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
+            AssertUploadLarge(result);
         }
 
         [Test, RetryWithDelay]
         public async Task TestUploadLargeAutoFilesAsync()
         {
-            // support asynchronous uploading large raw files
-            var largeFilePath = m_testLargeImagePath;
-            int largeFileLength = (int)new FileInfo(largeFilePath).Length;
-
             var uploadParams =  new AutoUploadParams()
             {
-                File = new FileDescription(largeFilePath),
+                File = new FileDescription(m_testLargeImagePath),
                 Tags = m_apiTag
             };
 
             var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);
 
-            AssertUploadLarge(result, largeFileLength);
+            AssertUploadLarge(result);
 
             Assert.AreEqual("image", result.ResourceType);
         }
 
         [Test, RetryWithDelay]
         public void TestUploadChunkSingleStream()
         {
-            var largeFilePath = m_testLargeImagePath;
-            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
-
             ImageUploadResult result = null;
 
             using (var currChunk = new MemoryStream())
@@ -677,7 +680,7 @@ public void TestUploadChunkSingleStream()
 
                 var buffer = new byte[TEST_CHUNK_SIZE];
 
-                using (var source = File.Open(largeFilePath, FileMode.Open))
+                using (var source = File.Open(m_testLargeImagePath, FileMode.Open))
                 {
                     int read;
                     while ((read = source.Read(buffer, 0, buffer.Length)) > 0)
@@ -693,16 +696,13 @@ public void TestUploadChunkSingleStream()
                 }
             }
 
-            AssertUploadLarge(result, largeFileLength);
+            AssertUploadLarge(result);
             Assert.AreEqual("image", result?.ResourceType);
         }
 
         [Test, RetryWithDelay]
         public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()
         {
-            var largeFilePath = m_testLargeImagePath;
-            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
-
             ImageUploadResult result = null;
 
             var uploadParams =  new ImageUploadParams()
@@ -714,7 +714,7 @@ public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()
 
             var buffer = new byte[TEST_CHUNK_SIZE];
 
-            using (var source = File.Open(largeFilePath, FileMode.Open))
+            using (var source = File.Open(m_testLargeImagePath, FileMode.Open))
             {
                 int read;
                 while ((read = source.Read(buffer, 0, buffer.Length)) > 0)
@@ -727,19 +727,16 @@ public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()
                 }
             }
 
-            AssertUploadLarge(result, largeFileLength);
+            AssertUploadLarge(result);
             Assert.AreEqual("image", result?.ResourceType);
         }
 
         [Test, RetryWithDelay]
         public void TestUploadChunkMultipleFileParts()
         {
-            var largeFilePath = m_testLargeImagePath;
-            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
-
             ImageUploadResult result = null;
 
-            var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple");
+            var fileChunks = SplitFile(m_testLargeImagePath, TEST_CHUNK_SIZE, "multiple");
 
             var uploadParams =  new ImageUploadParams()
             {
@@ -773,17 +770,14 @@ public void TestUploadChunkMultipleFileParts()
                 }
             }
 
-            AssertUploadLarge(result, largeFileLength);
+            AssertUploadLarge(result);
             Assert.AreEqual("image", result?.ResourceType);
         }
 
         [Test, RetryWithDelay]
         public void TestUploadChunkMultipleFilePartsInParallel()
         {
-            var largeFilePath = m_testLargeImagePath;
-            var largeFileLength = (int)new FileInfo(largeFilePath).Length;
-
-            var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple_parallel");
+            var fileChunks = SplitFile(m_testLargeImagePath, TEST_CHUNK_SIZE, "multiple_parallel");
 
             var uploadParams =  new RawUploadParams()
             {
@@ -820,7 +814,7 @@ public void TestUploadChunkMultipleFilePartsInParallel()
 
             var uploadResult = resultCollection.FirstOrDefault(r => r.AssetId != null);
 
-            AssertUploadLarge(uploadResult, largeFileLength);
+            AssertUploadLarge(uploadResult);
             Assert.AreEqual("raw", uploadResult?.ResourceType);
         }
 
diff --git a/CloudinaryDotNet/Core/LimitedStream.cs b/CloudinaryDotNet/Core/LimitedStream.cs
@@ -12,6 +12,7 @@ internal class LimitedStream : Stream
         private readonly Stream originalStream;
         private long remainingBytes;
         private long startOffset;
+        private long currOffset;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="LimitedStream"/> class.
@@ -24,7 +25,7 @@ public LimitedStream(Stream stream, long offset, long maxBytes)
         {
             originalStream = stream ?? throw new ArgumentNullException(nameof(stream));
             remainingBytes = maxBytes;
-            startOffset = offset;
+            startOffset = currOffset = offset;
 
             if (!stream.CanSeek)
             {
@@ -64,9 +65,11 @@ public override long Position
         public override int Read(byte[] buffer, int offset, int count)
         {
             // make sure stream is not moved around.
-            originalStream.Seek(startOffset, SeekOrigin.Begin);
+            originalStream.Seek(currOffset, SeekOrigin.Begin);
 
             var bytesRead = originalStream.Read(buffer, offset, (int)Math.Min(count, remainingBytes));
+
+            currOffset += bytesRead;
             remainingBytes -= bytesRead;
 
             return bytesRead;

Original file line number	Diff line number	Diff line change
`@@ -40,6 +40,9 @@ public class UploadMethodsTest : IntegrationTestBase`
`40`	`40`
`41`	`41`	`protected string m_implicitTransformationText;`
`42`	`42`
	`43`	`+ protected string m_largeFileEtag;`
	`44`	`+ protected int m_largeFileLength;`
	`45`	`+`
`43`	`46`	`public override void Initialize()`
`44`	`47`	`{`
`45`	`48`	`base.Initialize();`
`@@ -52,6 +55,9 @@ public override void Initialize()`
`52`	`55`	`m_transformationAr25, m_transformationAr69,`
`53`	`56`	`m_transformationAr30, m_transformationAr12,`
`54`	`57`	`m_eagerTransformation);`
	`58`	`+`
	`59`	`+ m_largeFileEtag = GetFileMd5Sum(m_testLargeImagePath);`
	`60`	`+ m_largeFileLength = (int)new FileInfo(m_testLargeImagePath).Length;`
`55`	`61`	`}`
`56`	`62`
`57`	`63`	`[Test, RetryWithDelay]`
`@@ -576,7 +582,7 @@ public void TestUploadLargeNonSeekableStream()`
`576`	`582`
`577`	`583`	`var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);`
`578`	`584`
`579`		`- AssertUploadLarge(result, bytes.Length);`
	`585`	`+ AssertUploadLarge(result);`
`580`	`586`	`}`
`581`	`587`	`}`
`582`	`588`
`@@ -585,27 +591,25 @@ public void TestUploadLargeRawFiles()`
`585`	`591`	`{`
`586`	`592`	`// support uploading large raw files`
`587`	`593`	`var largeFilePath = m_testLargeImagePath;`
`588`		`- int largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`589`	`594`
`590`	`595`	`var uploadParams = GetUploadLargeRawParams(largeFilePath);`
`591`	`596`
`592`	`597`	`var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);`
`593`	`598`
`594`		`- AssertUploadLarge(result, largeFileLength);`
	`599`	`+ AssertUploadLarge(result);`
`595`	`600`	`}`
`596`	`601`
`597`	`602`	`[Test, RetryWithDelay]`
`598`	`603`	`public async Task TestUploadLargeRawFilesAsyncInParallel()`
`599`	`604`	`{`
`600`	`605`	`// support asynchronous uploading large raw files`
`601`	`606`	`var largeFilePath = m_testLargeImagePath;`
`602`		`- int largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`603`	`607`
`604`	`608`	`var uploadParams = GetUploadLargeRawParams(largeFilePath);`
`605`	`609`
`606`	`610`	`var result = await m_cloudinary.UploadLargeAsync<RawUploadResult>(uploadParams, TEST_CHUNK_SIZE, 2);`
`607`	`611`
`608`		`- AssertUploadLarge(result, largeFileLength);`
	`612`	`+ AssertUploadLarge(result);`
`609`	`613`	`}`
`610`	`614`
`611`	`615`	`private RawUploadParams GetUploadLargeRawParams(string path)`
`@@ -617,54 +621,53 @@ private RawUploadParams GetUploadLargeRawParams(string path)`
`617`	`621`	`};`
`618`	`622`	`}`
`619`	`623`
`620`		`- private void AssertUploadLarge(RawUploadResult result, int fileLength)`
	`624`	`+ private void AssertUploadLarge(RawUploadResult result, int fileLength = 0, string etag = null)`
`621`	`625`	`{`
	`626`	`+ if (fileLength == 0)`
	`627`	`+ {`
	`628`	`+ fileLength = m_largeFileLength;`
	`629`	`+ }`
	`630`	`+ if (etag == null)`
	`631`	`+ {`
	`632`	`+ etag = m_largeFileEtag;`
	`633`	`+ }`
`622`	`634`	`Assert.NotNull(result);`
`623`	`635`	`Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);`
	`636`	`+ Assert.AreEqual(etag, result.Etag, result.Error?.Message);`
`624`	`637`	`}`
`625`	`638`
`626`	`639`	`[Test, RetryWithDelay]`
`627`	`640`	`public void TestUploadLarge()`
`628`	`641`	`{`
`629`	`642`	`// support uploading large image`
`630`		`-`
`631`		`- var largeFilePath = m_testLargeImagePath;`
`632`		`- int fileLength = (int)new FileInfo(largeFilePath).Length;`
`633`	`643`	`var result = m_cloudinary.UploadLarge(new ImageUploadParams()`
`634`	`644`	`{`
`635`		`- File = new FileDescription(largeFilePath),`
	`645`	`+ File = new FileDescription(m_testLargeImagePath),`
`636`	`646`	`Tags = m_apiTag`
`637`	`647`	`}, TEST_CHUNK_SIZE);`
`638`	`648`
`639`		`- Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);`
	`649`	`+ AssertUploadLarge(result);`
`640`	`650`	`}`
`641`	`651`
`642`	`652`	`[Test, RetryWithDelay]`
`643`	`653`	`public async Task TestUploadLargeAutoFilesAsync()`
`644`	`654`	`{`
`645`		`- // support asynchronous uploading large raw files`
`646`		`- var largeFilePath = m_testLargeImagePath;`
`647`		`- int largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`648`		`-`
`649`	`655`	`var uploadParams = new AutoUploadParams()`
`650`	`656`	`{`
`651`		`- File = new FileDescription(largeFilePath),`
	`657`	`+ File = new FileDescription(m_testLargeImagePath),`
`652`	`658`	`Tags = m_apiTag`
`653`	`659`	`};`
`654`	`660`
`655`	`661`	`var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);`
`656`	`662`
`657`		`- AssertUploadLarge(result, largeFileLength);`
	`663`	`+ AssertUploadLarge(result);`
`658`	`664`
`659`	`665`	`Assert.AreEqual("image", result.ResourceType);`
`660`	`666`	`}`
`661`	`667`
`662`	`668`	`[Test, RetryWithDelay]`
`663`	`669`	`public void TestUploadChunkSingleStream()`
`664`	`670`	`{`
`665`		`- var largeFilePath = m_testLargeImagePath;`
`666`		`- var largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`667`		`-`
`668`	`671`	`ImageUploadResult result = null;`
`669`	`672`
`670`	`673`	`using (var currChunk = new MemoryStream())`
`@@ -677,7 +680,7 @@ public void TestUploadChunkSingleStream()`
`677`	`680`
`678`	`681`	`var buffer = new byte[TEST_CHUNK_SIZE];`
`679`	`682`
`680`		`- using (var source = File.Open(largeFilePath, FileMode.Open))`
	`683`	`+ using (var source = File.Open(m_testLargeImagePath, FileMode.Open))`
`681`	`684`	`{`
`682`	`685`	`int read;`
`683`	`686`	`while ((read = source.Read(buffer, 0, buffer.Length)) > 0)`
`@@ -693,16 +696,13 @@ public void TestUploadChunkSingleStream()`
`693`	`696`	`}`
`694`	`697`	`}`
`695`	`698`
`696`		`- AssertUploadLarge(result, largeFileLength);`
	`699`	`+ AssertUploadLarge(result);`
`697`	`700`	`Assert.AreEqual("image", result?.ResourceType);`
`698`	`701`	`}`
`699`	`702`
`700`	`703`	`[Test, RetryWithDelay]`
`701`	`704`	`public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()`
`702`	`705`	`{`
`703`		`- var largeFilePath = m_testLargeImagePath;`
`704`		`- var largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`705`		`-`
`706`	`706`	`ImageUploadResult result = null;`
`707`	`707`
`708`	`708`	`var uploadParams = new ImageUploadParams()`
`@@ -714,7 +714,7 @@ public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()`
`714`	`714`
`715`	`715`	`var buffer = new byte[TEST_CHUNK_SIZE];`
`716`	`716`
`717`		`- using (var source = File.Open(largeFilePath, FileMode.Open))`
	`717`	`+ using (var source = File.Open(m_testLargeImagePath, FileMode.Open))`
`718`	`718`	`{`
`719`	`719`	`int read;`
`720`	`720`	`while ((read = source.Read(buffer, 0, buffer.Length)) > 0)`
`@@ -727,19 +727,16 @@ public async Task TestUploadChunkMultipleStreamsCustomOffsetAsync()`
`727`	`727`	`}`
`728`	`728`	`}`
`729`	`729`
`730`		`- AssertUploadLarge(result, largeFileLength);`
	`730`	`+ AssertUploadLarge(result);`
`731`	`731`	`Assert.AreEqual("image", result?.ResourceType);`
`732`	`732`	`}`
`733`	`733`
`734`	`734`	`[Test, RetryWithDelay]`
`735`	`735`	`public void TestUploadChunkMultipleFileParts()`
`736`	`736`	`{`
`737`		`- var largeFilePath = m_testLargeImagePath;`
`738`		`- var largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`739`		`-`
`740`	`737`	`ImageUploadResult result = null;`
`741`	`738`
`742`		`- var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple");`
	`739`	`+ var fileChunks = SplitFile(m_testLargeImagePath, TEST_CHUNK_SIZE, "multiple");`
`743`	`740`
`744`	`741`	`var uploadParams = new ImageUploadParams()`
`745`	`742`	`{`
`@@ -773,17 +770,14 @@ public void TestUploadChunkMultipleFileParts()`
`773`	`770`	`}`
`774`	`771`	`}`
`775`	`772`
`776`		`- AssertUploadLarge(result, largeFileLength);`
	`773`	`+ AssertUploadLarge(result);`
`777`	`774`	`Assert.AreEqual("image", result?.ResourceType);`
`778`	`775`	`}`
`779`	`776`
`780`	`777`	`[Test, RetryWithDelay]`
`781`	`778`	`public void TestUploadChunkMultipleFilePartsInParallel()`
`782`	`779`	`{`
`783`		`- var largeFilePath = m_testLargeImagePath;`
`784`		`- var largeFileLength = (int)new FileInfo(largeFilePath).Length;`
`785`		`-`
`786`		`- var fileChunks = SplitFile(largeFilePath, TEST_CHUNK_SIZE, "multiple_parallel");`
	`780`	`+ var fileChunks = SplitFile(m_testLargeImagePath, TEST_CHUNK_SIZE, "multiple_parallel");`
`787`	`781`
`788`	`782`	`var uploadParams = new RawUploadParams()`
`789`	`783`	`{`
`@@ -820,7 +814,7 @@ public void TestUploadChunkMultipleFilePartsInParallel()`
`820`	`814`
`821`	`815`	`var uploadResult = resultCollection.FirstOrDefault(r => r.AssetId != null);`
`822`	`816`
`823`		`- AssertUploadLarge(uploadResult, largeFileLength);`
	`817`	`+ AssertUploadLarge(uploadResult);`
`824`	`818`	`Assert.AreEqual("raw", uploadResult?.ResourceType);`
`825`	`819`	`}`
`826`	`820`