Skip to content

Commit fc556bc

Browse files
const-cloudinaryconst-cloudinary
committed
Add support for uploading non-seekable streams
1 parent 7290a86 commit fc556bc

File tree

4 files changed

+98
-12
lines changed

4 files changed

+98
-12
lines changed

‎CloudinaryDotNet.IntegrationTests/UploadApi/UploadMethodsTest.cs‎

Lines changed: 37 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ public class UploadMethodsTest : IntegrationTestBase
2323
private const string MODERATION_WEBPURIFY = "webpurify";
2424
private const string TEST_REMOTE_IMG = "http://cloudinary.com/images/old_logo.png";
2525
private const string TEST_REMOTE_VIDEO = "http://res.cloudinary.com/demo/video/upload/v1496743637/dog.mp4";
26+
private const int TEST_CHUNK_SIZE = 5 * 1024 * 1024; // 5 MB
2627

2728
private Transformation m_implicitTransformation;
2829

@@ -546,6 +547,37 @@ public void TestUploadStream()
546547
}
547548
}
548549

550+
private class NonSeekableStream : MemoryStream
551+
{
552+
public NonSeekableStream(byte[] buffer) : base(buffer) { }
553+
554+
public override bool CanSeek => false;
555+
556+
public override long Seek(long offset, SeekOrigin loc) => throw new NotSupportedException();
557+
558+
public override long Length => throw new NotSupportedException();
559+
}
560+
561+
[Test, RetryWithDelay]
562+
public void TestUploadLargeNonSeekableStream()
563+
{
564+
byte[] bytes = File.ReadAllBytes(m_testLargeImagePath);
565+
const string streamed = "stream_non_seekable";
566+
567+
using (var memoryStream = new NonSeekableStream(bytes))
568+
{
569+
var uploadParams = new ImageUploadParams()
570+
{
571+
File = new FileDescription(streamed, memoryStream),
572+
Tags = $"{m_apiTag},{streamed}"
573+
};
574+
575+
var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);
576+
577+
AssertUploadLarge(result, bytes.Length);
578+
}
579+
}
580+
549581
[Test, RetryWithDelay]
550582
public void TestUploadLargeRawFiles()
551583
{
@@ -555,7 +587,7 @@ public void TestUploadLargeRawFiles()
555587

556588
var uploadParams = GetUploadLargeRawParams(largeFilePath);
557589

558-
var result = m_cloudinary.UploadLarge(uploadParams, 5 * 1024 * 1024);
590+
var result = m_cloudinary.UploadLarge(uploadParams, TEST_CHUNK_SIZE);
559591

560592
AssertUploadLarge(result, largeFileLength);
561593
}
@@ -569,7 +601,7 @@ public async Task TestUploadLargeRawFilesAsync()
569601

570602
var uploadParams = GetUploadLargeRawParams(largeFilePath);
571603

572-
var result = await m_cloudinary.UploadLargeAsync(uploadParams, 5 * 1024 * 1024);
604+
var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);
573605

574606
AssertUploadLarge(result, largeFileLength);
575607
}
@@ -599,7 +631,7 @@ public void TestUploadLarge()
599631
{
600632
File = new FileDescription(largeFilePath),
601633
Tags = m_apiTag
602-
}, 5 * 1024 * 1024);
634+
}, TEST_CHUNK_SIZE);
603635

604636
Assert.AreEqual(fileLength, result.Bytes, result.Error?.Message);
605637
}
@@ -617,7 +649,7 @@ public async Task TestUploadLargeAutoFilesAsync()
617649
Tags = m_apiTag
618650
};
619651

620-
var result = await m_cloudinary.UploadLargeAsync(uploadParams, 5 * 1024 * 1024);
652+
var result = await m_cloudinary.UploadLargeAsync(uploadParams, TEST_CHUNK_SIZE);
621653

622654
AssertUploadLarge(result, largeFileLength);
623655

@@ -679,7 +711,7 @@ public void TestUploadLargeVideoFromWeb()
679711
{
680712
File = new FileDescription(TEST_REMOTE_VIDEO),
681713
Tags = m_apiTag
682-
}, 5 * 1024 * 1024);
714+
}, TEST_CHUNK_SIZE);
683715

684716
Assert.AreEqual(result.StatusCode, HttpStatusCode.OK, result.Error?.Message);
685717
Assert.AreEqual(result.Format, FILE_FORMAT_MP4);

‎CloudinaryDotNet/ApiShared.Internal.cs‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,18 @@ private static bool ShouldPrepareContent(HttpMethod method, object parameters) =
342342
private static bool IsContentRange(Dictionary<string, string> extraHeaders) =>
343343
extraHeaders != null && extraHeaders.ContainsKey("Content-Range");
344344

345+
private static void UpdateContentRange(IDictionary<string, string> extraHeaders, FileDescription file)
346+
{
347+
if (!file.Eof || file.GetFileLength() > 0)
348+
{
349+
return; // no need to update the header, all good.
350+
}
351+
352+
var startOffset = file.BytesSent - file.CurrChunkSize;
353+
354+
extraHeaders["Content-Range"] = $"bytes {startOffset}-{file.BytesSent - 1}/{file.BytesSent}";
355+
}
356+
345357
private static Stream GetFileStream(FileDescription file) =>
346358
file.Stream ?? File.OpenRead(file.FilePath);
347359

@@ -385,7 +397,11 @@ private static StreamWriter SetStreamToStartAndCreateWriter(FileDescription file
385397
var memStream = new MemoryStream();
386398
var writer = new StreamWriter(memStream) { AutoFlush = true };
387399

388-
stream.Seek(file.BytesSent, SeekOrigin.Begin);
400+
if (stream.CanSeek)
401+
{
402+
stream.Seek(file.BytesSent, SeekOrigin.Begin);
403+
}
404+
389405
return writer;
390406
}
391407

@@ -446,6 +462,7 @@ private async Task<HttpContent> CreateMultipartContentAsync(
446462
// Unfortunately we don't have ByteRangeStreamContent here,
447463
// let's create another stream from the original one
448464
stream = await GetRangeFromFileAsync(file, stream, cancellationToken).ConfigureAwait(false);
465+
UpdateContentRange(extraHeaders, file);
449466
}
450467

451468
SetStreamContent(param.Key, file, stream, content);
@@ -492,6 +509,7 @@ private HttpContent CreateMultipartContent(
492509
// Unfortunately we don't have ByteRangeStreamContent here,
493510
// let's create another stream from the original one
494511
stream = GetRangeFromFile(file, stream);
512+
UpdateContentRange(extraHeaders, file);
495513
}
496514

497515
SetStreamContent(param.Key, file, stream, content);
@@ -594,14 +612,26 @@ private void PrepareRequestContent(
594612
private async Task<Stream> GetRangeFromFileAsync(FileDescription file, Stream stream, CancellationToken? cancellationToken = null)
595613
{
596614
var writer = SetStreamToStartAndCreateWriter(file, stream);
597-
file.BytesSent += await ReadBytesAsync(writer, stream, file.BufferLength, cancellationToken).ConfigureAwait(false);
615+
file.CurrChunkSize = await ReadBytesAsync(writer, stream, file.BufferLength, cancellationToken).ConfigureAwait(false);
616+
file.BytesSent += file.CurrChunkSize;
617+
if (file.CurrChunkSize < file.BufferLength)
618+
{
619+
file.Eof = true; // last chunk
620+
}
621+
598622
return WriterStreamFromBegin(writer);
599623
}
600624

601625
private Stream GetRangeFromFile(FileDescription file, Stream stream)
602626
{
603627
var writer = SetStreamToStartAndCreateWriter(file, stream);
604-
file.BytesSent += ReadBytes(writer, stream, file.BufferLength);
628+
file.CurrChunkSize = ReadBytes(writer, stream, file.BufferLength);
629+
file.BytesSent += file.CurrChunkSize;
630+
if (file.CurrChunkSize < file.BufferLength)
631+
{
632+
file.Eof = true; // last chunk
633+
}
634+
605635
return WriterStreamFromBegin(writer);
606636
}
607637

‎CloudinaryDotNet/Cloudinary.UploadApi.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -969,7 +969,10 @@ private static void UpdateContentRange(UploadLargeParams internalParams)
969969
var fileDescription = internalParams.Parameters.File;
970970
var fileLength = fileDescription.GetFileLength();
971971
var startOffset = fileDescription.BytesSent;
972-
var endOffset = startOffset + Math.Min(internalParams.BufferSize, fileLength - startOffset) - 1;
972+
var buffSize = fileLength > 0
973+
? Math.Min(internalParams.BufferSize, fileLength - startOffset)
974+
: internalParams.BufferSize;
975+
var endOffset = startOffset + buffSize - 1;
973976

974977
internalParams.Headers["Content-Range"] = $"bytes {startOffset}-{endOffset}/{fileLength}";
975978
}

‎CloudinaryDotNet/FileDescription.cs‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ public class FileDescription
1717
/// </summary>
1818
internal long BytesSent;
1919

20+
/// <summary>
21+
/// Current chunk size.
22+
/// </summary>
23+
internal long CurrChunkSize;
24+
25+
private bool isEof;
26+
2027
/// <summary>
2128
/// Initializes a new instance of the <see cref="FileDescription"/> class.
2229
/// Constructor to upload file from stream.
@@ -75,17 +82,31 @@ public FileDescription(string filePath)
7582
public bool IsRemote { get; }
7683

7784
/// <summary>
78-
/// Gets a value indicating whether the pointer is at the end of file.
85+
/// Gets or sets a value indicating whether the pointer is at the end of file.
7986
/// </summary>
80-
internal bool Eof => BytesSent == GetFileLength();
87+
internal bool Eof
88+
{
89+
get => isEof ? isEof : GetFileLength() != -1 && BytesSent == GetFileLength();
90+
set => isEof = value;
91+
}
8192

8293
/// <summary>
8394
/// Get file length.
8495
/// </summary>
8596
/// <returns>The length of file.</returns>
8697
internal long GetFileLength()
8798
{
88-
return Stream?.Length ?? new FileInfo(FilePath).Length;
99+
if (Stream == null)
100+
{
101+
return new FileInfo(FilePath).Length;
102+
}
103+
104+
if (Stream?.CanSeek ?? false)
105+
{
106+
return Stream?.Length ?? -1;
107+
}
108+
109+
return -1; // unknown length
89110
}
90111

91112
/// <summary>

0 commit comments

Comments
 (0)