From e03eb2c9aec3ce43f274aa022d93e17982502f2c Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 16:43:17 -0700 Subject: [PATCH 1/7] initial commit --- .../MessageManager.cs | 2 +- .../Storage/BlobContainer.cs | 5 ++++ .../AzureStorageScenarioTests.cs | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 06ac335b1..0cd8bba4f 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -255,7 +255,7 @@ private async Task DownloadAndDecompressAsBytesAsync(Blob blob, Cancella public string GetBlobUrl(string blobName) { - return Uri.UnescapeDataString(this.blobContainer.GetBlobReference(blobName).Uri.AbsoluteUri); + return $"{this.blobContainer.GetBlobContainerUri()}/{Uri.EscapeDataString(blobName)}"; } public MessageFormatFlags GetMessageFormatFlags(MessageData messageData) diff --git a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs index 4abb9c61c..a61ac9f5b 100644 --- a/src/DurableTask.AzureStorage/Storage/BlobContainer.cs +++ b/src/DurableTask.AzureStorage/Storage/BlobContainer.cs @@ -108,6 +108,11 @@ public Task RenewLeaseAsync(string leaseId, CancellationToken cancellationToken .DecorateFailure(); } + public Uri GetBlobContainerUri() + { + return this.blobContainerClient.Uri; + } + static bool IsHnsFolder(BlobItem item) { // Check the optional "hdi_isfolder" value in the metadata to determine whether diff --git a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs index c115c127d..a66ce6695 100644 --- a/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs +++ b/test/DurableTask.AzureStorage.Tests/AzureStorageScenarioTests.cs @@ -1600,6 +1600,35 @@ await ValidateLargeMessageBlobUrlAsync( } } + /// + /// End-to-end test that validates large (>60KB) messages stored in blob storage can be retrieved successfully, + /// when the instance ID includes special characters like '|' that can affect blob URL encoding. + /// + [TestMethod] + public async Task LargeMessage_WithEscapedInstanceId_CanBeStoredAndFetchedSuccessfully() + { + // Genereates a random large message. + const int largeMessageSize = 60 * 1024; + + using (TestOrchestrationHost host = TestHelpers.GetTestOrchestrationHost(enableExtendedSessions: false)) + { + await host.StartAsync(); + + string message = this.GenerateMediumRandomStringPayload(largeMessageSize, utf8ByteSize: 1, utf16ByteSize: 2).ToString(); + + // Use an instanceId that contains special characters which must be escaped in URIs + string id = "test|123:with white spcae"; + var client = await host.StartOrchestrationAsync(typeof(Orchestrations.Echo), input:message, instanceId: id); + var status = await client.WaitForCompletionAsync(TimeSpan.FromMinutes(2)); + + Assert.AreEqual(OrchestrationStatus.Completed, status?.OrchestrationStatus); + + // Verify that the output matches the original message (the blob was successfully downloaded and not returned as a URL) + StringAssert.Contains(status.Output.ToString(), message); + await host.StopAsync(); + } + } + [TestMethod] public async Task TagsAreAvailableInOrchestrationState() { From 7a764baf0ff7847115185b8579cde99a706d18ed Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 18:36:36 -0700 Subject: [PATCH 2/7] udpate messagemanager test --- test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index e7400d5a6..6ffa740e4 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -15,6 +15,7 @@ namespace DurableTask.AzureStorage.Tests { using System; using System.Collections.Generic; + using System.Security.Policy; using DurableTask.AzureStorage.Storage; using DurableTask.Core.History; using Microsoft.VisualStudio.TestTools.UnitTesting; @@ -73,7 +74,7 @@ public void DeserializesCustomTypes() [DataRow("blob.bin")] [DataRow("@#$%!")] [DataRow("foo/bar/b@z.tar.gz")] - public void GetBlobUrlUnescaped(string blob) + public void GetBlobUrlEscaped(string blob) { var settings = new AzureStorageOrchestrationServiceSettings { @@ -82,7 +83,9 @@ public void GetBlobUrlUnescaped(string blob) const string container = "@entity12345"; var manager = new MessageManager(settings, new AzureStorageClient(settings), container); - var expected = $"http://127.0.0.1:10000/devstoreaccount1/{container}/{blob}"; + + string blobUrl = Uri.EscapeDataString(blob); + var expected = $"http://127.0.0.1:10000/devstoreaccount1/{container}/{blobUrl}"; Assert.AreEqual(expected, manager.GetBlobUrl(blob)); } From 3e138df167bfd10c6183bcf70447f32afc07636c Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 18:38:16 -0700 Subject: [PATCH 3/7] sort usings --- test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index 6ffa740e4..77830b5f7 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -13,13 +13,12 @@ #nullable enable namespace DurableTask.AzureStorage.Tests { - using System; - using System.Collections.Generic; - using System.Security.Policy; using DurableTask.AzureStorage.Storage; using DurableTask.Core.History; using Microsoft.VisualStudio.TestTools.UnitTesting; using Newtonsoft.Json; + using System; + using System.Collections.Generic; [TestClass] public class MessageManagerTests From 73078ede3f1ab66b0990f44c03f59a3b0be2c129 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 20:36:42 -0700 Subject: [PATCH 4/7] make blub url preserve slashes in blobname --- src/DurableTask.AzureStorage/MessageManager.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index 0cd8bba4f..e85d1725d 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -255,7 +255,7 @@ private async Task DownloadAndDecompressAsBytesAsync(Blob blob, Cancella public string GetBlobUrl(string blobName) { - return $"{this.blobContainer.GetBlobContainerUri()}/{Uri.EscapeDataString(blobName)}"; + return $"{this.blobContainer.GetBlobContainerUri()}/{EscapeBlobNamePreservingSlashes(blobName)}"; } public MessageFormatFlags GetMessageFormatFlags(MessageData messageData) @@ -304,6 +304,11 @@ public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance return storageOperationCount; } + + static string EscapeBlobNamePreservingSlashes(string blobName) + { + return string.Join("/", blobName.Split('/').Select(Uri.EscapeDataString)); + } } #if NETSTANDARD2_0 From e5ed7eaf554089878a7dbf5c6d83a0dd88ab8188 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 20:45:54 -0700 Subject: [PATCH 5/7] update messagemanager test --- .../MessageManagerTests.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs index 77830b5f7..008a9eac7 100644 --- a/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs +++ b/test/DurableTask.AzureStorage.Tests/MessageManagerTests.cs @@ -70,10 +70,10 @@ public void DeserializesCustomTypes() } [DataTestMethod] - [DataRow("blob.bin")] - [DataRow("@#$%!")] - [DataRow("foo/bar/b@z.tar.gz")] - public void GetBlobUrlEscaped(string blob) + [DataRow("blob.bin", "blob.bin")] + [DataRow("@#$%!", "%40%23%24%25%21")] + [DataRow("foo/bar/b@z.tar.gz", "foo/bar/b%40z.tar.gz")] + public void GetBlobUrlEscaped(string blob, string blobUrl) { var settings = new AzureStorageOrchestrationServiceSettings { @@ -83,7 +83,6 @@ public void GetBlobUrlEscaped(string blob) const string container = "@entity12345"; var manager = new MessageManager(settings, new AzureStorageClient(settings), container); - string blobUrl = Uri.EscapeDataString(blob); var expected = $"http://127.0.0.1:10000/devstoreaccount1/{container}/{blobUrl}"; Assert.AreEqual(expected, manager.GetBlobUrl(blob)); } From 459b5eec7299a47cc58e8360865271614c64f858 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 20:48:35 -0700 Subject: [PATCH 6/7] add comments --- src/DurableTask.AzureStorage/MessageManager.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index e85d1725d..fec93bcd7 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -305,6 +305,7 @@ public async Task DeleteLargeMessageBlobs(string sanitizedInstanceId, Cance return storageOperationCount; } + /// Escapes a blob name for safe inclusion in a URI while preserving path structure. static string EscapeBlobNamePreservingSlashes(string blobName) { return string.Join("/", blobName.Split('/').Select(Uri.EscapeDataString)); From 89189c2a243843e6826358a75e6b003ae80b97a8 Mon Sep 17 00:00:00 2001 From: "naiyuantian@microsoft.com" Date: Tue, 22 Apr 2025 21:03:29 -0700 Subject: [PATCH 7/7] update getbloburl --- src/DurableTask.AzureStorage/MessageManager.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/DurableTask.AzureStorage/MessageManager.cs b/src/DurableTask.AzureStorage/MessageManager.cs index fec93bcd7..a1fc796cf 100644 --- a/src/DurableTask.AzureStorage/MessageManager.cs +++ b/src/DurableTask.AzureStorage/MessageManager.cs @@ -255,7 +255,9 @@ private async Task DownloadAndDecompressAsBytesAsync(Blob blob, Cancella public string GetBlobUrl(string blobName) { - return $"{this.blobContainer.GetBlobContainerUri()}/{EscapeBlobNamePreservingSlashes(blobName)}"; + string baseUri = this.blobContainer.GetBlobContainerUri().ToString(); + + return $"{baseUri}/{EscapeBlobNamePreservingSlashes(blobName)}"; } public MessageFormatFlags GetMessageFormatFlags(MessageData messageData)