Skip to content

TD-5176 file download refactor #794

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 3, 2025
Merged

TD-5176 file download refactor #794

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 5 additions & 50 deletions LearningHub.Nhs.WebUI/Services/FileService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Threading.Tasks;
using System.Threading.Tasks.Dataflow;
using Azure;
using Azure.Storage;
using Azure.Storage.Files.Shares;
using Azure.Storage.Files.Shares.Models;
using LearningHub.Nhs.Models.Resource;
Expand Down Expand Up @@ -133,10 +134,6 @@ public async Task DeleteChunkDirectory(string directoryRef, int chunks)
/// <returns>The <see cref="Task{FileDownloadResponse}"/>.</returns>
public async Task<FileDownloadResponse> DownloadFileAsync(string filePath, string fileName)
{
const int ChunkSizeInMB = 100;
const int MaxParallelDownloads = 8;
int chunkSize = ChunkSizeInMB * 1024 * 1024;

var file = await this.FindFileAsync(filePath, fileName);
if (file == null)
{
Expand All @@ -146,9 +143,9 @@ public async Task<FileDownloadResponse> DownloadFileAsync(string filePath, strin
var properties = await file.GetPropertiesAsync();
long fileSize = properties.Value.ContentLength;

// If the file size is less than 500 MB, download it directly
if (fileSize <= 500 * 1024 * 1024)
try
{
// Directly download the entire file as a stream
var response = await file.DownloadAsync();
return new FileDownloadResponse
{
Expand All @@ -157,52 +154,10 @@ public async Task<FileDownloadResponse> DownloadFileAsync(string filePath, strin
ContentLength = fileSize,
};
}

var pipe = new System.IO.Pipelines.Pipe();
var semaphore = new SemaphoreSlim(1, 1);

var downloadBlock = new ActionBlock<long>(
async offset =>
{
long rangeSize = Math.Min(chunkSize, fileSize - offset);
try
{
var response = await file.DownloadAsync(new HttpRange(offset, rangeSize));
var buffer = new byte[rangeSize];
await response.Value.Content.ReadAsync(buffer, 0, (int)rangeSize);

await semaphore.WaitAsync();
try
{
pipe.Writer.Write(buffer);
}
finally
{
semaphore.Release();
}
}
catch (Exception ex)
{
throw new Exception($"Error downloading chunk at offset {offset}: {ex.Message}");
}
},
new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxParallelDownloads, EnsureOrdered = false });

for (long offset = 0; offset < fileSize; offset += chunkSize)
catch (Exception ex)
{
downloadBlock.Post(offset);
throw new Exception($"Error downloading file: {ex.Message}");
}

downloadBlock.Complete();
await downloadBlock.Completion;
await pipe.Writer.CompleteAsync();

return new FileDownloadResponse
{
Content = pipe.Reader.AsStream(),
ContentType = properties.Value.ContentType,
ContentLength = fileSize,
};
}

/// <summary>
Expand Down
Loading