From 8e5af6906f35cbc7988543d409d109210b7d437b Mon Sep 17 00:00:00 2001 From: Oluwatobi Awe Date: Mon, 23 Dec 2024 03:25:32 +0000 Subject: [PATCH] TD-5176 Large file download fix. --- .../Interfaces/IFileService.cs | 4 +- .../Models/FileDownloadResponse.cs | 25 ++++ LearningHub.Nhs.WebUI/Services/FileService.cs | 107 +++++++++++++++--- 3 files changed, 119 insertions(+), 17 deletions(-) create mode 100644 LearningHub.Nhs.WebUI/Models/FileDownloadResponse.cs diff --git a/LearningHub.Nhs.WebUI/Interfaces/IFileService.cs b/LearningHub.Nhs.WebUI/Interfaces/IFileService.cs index c74a1681f..eb94d0185 100644 --- a/LearningHub.Nhs.WebUI/Interfaces/IFileService.cs +++ b/LearningHub.Nhs.WebUI/Interfaces/IFileService.cs @@ -5,6 +5,7 @@ using System.Threading.Tasks; using Azure.Storage.Files.Shares.Models; using LearningHub.Nhs.Models.Resource; + using LearningHub.Nhs.WebUI.Models; /// /// Defines the . @@ -25,7 +26,8 @@ public interface IFileService /// File path. /// File name. /// A representing the result of the asynchronous operation. - Task DownloadFileAsync(string filePath, string fileName); + // Task DownloadFileAsync(string filePath, string fileName); + Task DownloadFileAsync(string filePath, string fileName); /// /// The StreamFileAsync. diff --git a/LearningHub.Nhs.WebUI/Models/FileDownloadResponse.cs b/LearningHub.Nhs.WebUI/Models/FileDownloadResponse.cs new file mode 100644 index 000000000..8036e9320 --- /dev/null +++ b/LearningHub.Nhs.WebUI/Models/FileDownloadResponse.cs @@ -0,0 +1,25 @@ +namespace LearningHub.Nhs.WebUI.Models +{ + using System.IO; + + /// + /// Defines the . + /// + public class FileDownloadResponse + { + /// + /// Gets or sets the Content. + /// + public Stream Content { get; set; } + + /// + /// Gets or sets the ContentType. + /// + public string ContentType { get; set; } + + /// + /// Gets or sets the ContentType. + /// + public long ContentLength { get; set; } + } +} \ No newline at end of file diff --git a/LearningHub.Nhs.WebUI/Services/FileService.cs b/LearningHub.Nhs.WebUI/Services/FileService.cs index 4e02db124..de74a44e3 100644 --- a/LearningHub.Nhs.WebUI/Services/FileService.cs +++ b/LearningHub.Nhs.WebUI/Services/FileService.cs @@ -1,15 +1,20 @@ namespace LearningHub.Nhs.WebUI.Services { using System; + using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; + using System.Threading; using System.Threading.Tasks; + using System.Threading.Tasks.Dataflow; + using Azure; using Azure.Storage.Files.Shares; using Azure.Storage.Files.Shares.Models; using LearningHub.Nhs.Models.Resource; using LearningHub.Nhs.WebUI.Configuration; using LearningHub.Nhs.WebUI.Interfaces; + using LearningHub.Nhs.WebUI.Models; using Microsoft.AspNetCore.StaticFiles; using Microsoft.Extensions.Options; @@ -125,32 +130,79 @@ public async Task DeleteChunkDirectory(string directoryRef, int chunks) /// /// The filePath. /// The fileName. - /// The . - public async Task DownloadFileAsync(string filePath, string fileName) + /// The . + public async Task DownloadFileAsync(string filePath, string fileName) { - var directory = this.ShareClient.GetDirectoryClient(filePath); - var sourceDirectory = this.InputArchiveShareClient.GetDirectoryClient(filePath); + const int ChunkSizeInMB = 100; + const int MaxParallelDownloads = 8; + int chunkSize = ChunkSizeInMB * 1024 * 1024; - if (await directory.ExistsAsync()) + var file = await this.FindFileAsync(filePath, fileName); + if (file == null) { - var file = directory.GetFileClient(fileName); + return null; + } - if (await file.ExistsAsync()) + var properties = await file.GetPropertiesAsync(); + long fileSize = properties.Value.ContentLength; + + // If the file size is less than 500 MB, download it directly + if (fileSize <= 500 * 1024 * 1024) + { + var response = await file.DownloadAsync(); + return new FileDownloadResponse { - return await file.DownloadAsync(); - } + Content = response.Value.Content, + ContentType = properties.Value.ContentType, + ContentLength = fileSize, + }; } - else if (await sourceDirectory.ExistsAsync()) - { - var file = sourceDirectory.GetFileClient(fileName); - if (await file.ExistsAsync()) + var pipe = new System.IO.Pipelines.Pipe(); + var semaphore = new SemaphoreSlim(1, 1); + + var downloadBlock = new ActionBlock( + async offset => { - return await file.DownloadAsync(); - } + long rangeSize = Math.Min(chunkSize, fileSize - offset); + try + { + var response = await file.DownloadAsync(new HttpRange(offset, rangeSize)); + var buffer = new byte[rangeSize]; + await response.Value.Content.ReadAsync(buffer, 0, (int)rangeSize); + + await semaphore.WaitAsync(); + try + { + pipe.Writer.Write(buffer); + } + finally + { + semaphore.Release(); + } + } + catch (Exception ex) + { + throw new Exception($"Error downloading chunk at offset {offset}: {ex.Message}"); + } + }, + new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = MaxParallelDownloads, EnsureOrdered = false }); + + for (long offset = 0; offset < fileSize; offset += chunkSize) + { + downloadBlock.Post(offset); } - return null; + downloadBlock.Complete(); + await downloadBlock.Completion; + await pipe.Writer.CompleteAsync(); + + return new FileDownloadResponse + { + Content = pipe.Reader.AsStream(), + ContentType = properties.Value.ContentType, + ContentLength = fileSize, + }; } /// @@ -418,5 +470,28 @@ private async Task MoveInPutDirectoryToArchive(List allDirectoryRef) } } } + + private async Task FindFileAsync(string filePath, string fileName) + { + var directories = new[] + { + this.ShareClient.GetDirectoryClient(filePath), + this.InputArchiveShareClient.GetDirectoryClient(filePath), + }; + + foreach (var directory in directories) + { + if (await directory.ExistsAsync()) + { + var file = directory.GetFileClient(fileName); + if (await file.ExistsAsync()) + { + return file; + } + } + } + + return null; + } } } \ No newline at end of file