From 5c80501000e56e8b8bc4e24d27ecfc1424a3fe01 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Mon, 11 Aug 2025 13:38:17 -0700 Subject: [PATCH 01/38] save --- Directory.Packages.props | 6 ++ src/Abstractions/Abstractions.csproj | 1 + .../Converters/BlobPayloadStore.cs | 98 +++++++++++++++++++ src/Abstractions/Converters/IPayloadStore.cs | 27 +++++ .../Converters/LargePayloadDataConverter.cs | 83 ++++++++++++++++ .../Converters/LargePayloadStorageOptions.cs | 41 ++++++++ src/Abstractions/DataConverter.cs | 11 ++- .../DurableTaskClientBuilderExtensions.cs | 36 +++++++ .../DurableTaskWorkerBuilderExtensions.cs | 33 +++++++ src/Worker/Core/DurableTaskWorkerOptions.cs | 1 - 10 files changed, 333 insertions(+), 4 deletions(-) create mode 100644 src/Abstractions/Converters/BlobPayloadStore.cs create mode 100644 src/Abstractions/Converters/IPayloadStore.cs create mode 100644 src/Abstractions/Converters/LargePayloadDataConverter.cs create mode 100644 src/Abstractions/Converters/LargePayloadStorageOptions.cs diff --git a/Directory.Packages.props b/Directory.Packages.props index 8388030fa..7a24d9b7f 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -23,6 +23,7 @@ + @@ -33,6 +34,11 @@ + + + + + diff --git a/src/Abstractions/Abstractions.csproj b/src/Abstractions/Abstractions.csproj index db8be76ab..ab32b4888 100644 --- a/src/Abstractions/Abstractions.csproj +++ b/src/Abstractions/Abstractions.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Abstractions/Converters/BlobPayloadStore.cs b/src/Abstractions/Converters/BlobPayloadStore.cs new file mode 100644 index 000000000..c746a5170 --- /dev/null +++ b/src/Abstractions/Converters/BlobPayloadStore.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Globalization; +using System.IO.Compression; +using System.Text; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; + +namespace Microsoft.DurableTask.Converters; + +/// +/// Azure Blob Storage implementation of . +/// Stores payloads as blobs and returns opaque tokens in the form "dtp:v1:<container>:<blobName>". +/// +public sealed class BlobPayloadStore : IPayloadStore +{ + readonly BlobContainerClient containerClient; + readonly LargePayloadStorageOptions options; + + /// + /// Initializes a new instance of the class. + /// + /// The options for the blob payload store. + /// Thrown when is null. + /// Thrown when is null or empty. + public BlobPayloadStore(LargePayloadStorageOptions options) + { + this.options = options ?? throw new ArgumentNullException(nameof(options)); + + Check.NotNullOrEmpty(options.ConnectionString, nameof(options.ConnectionString)); + Check.NotNullOrEmpty(options.ContainerName, nameof(options.ContainerName)); + + BlobServiceClient serviceClient = new(options.ConnectionString); + this.containerClient = serviceClient.GetBlobContainerClient(options.ContainerName); + } + + /// + public async Task UploadAsync(string contentType, ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) + { + // Ensure container exists + await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false); + + // One blob per payload using GUID-based name for uniqueness + string timestamp = DateTimeOffset.UtcNow.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture); + string blobName = $"{timestamp}/{Guid.NewGuid():N}.bin"; + BlobClient blob = this.containerClient.GetBlobClient(blobName); + + byte[] payloadBuffer = payloadBytes.ToArray(); + + // Compress and upload streaming + using Stream blobStream = await blob.OpenWriteAsync(overwrite: true, cancellationToken: cancellationToken).ConfigureAwait(false); + using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); + using MemoryStream payloadStream = new(payloadBuffer, writable: false); + + await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken).ConfigureAwait(false); + await compressedBlobStream.FlushAsync(cancellationToken).ConfigureAwait(false); + await blobStream.FlushAsync(cancellationToken).ConfigureAwait(false); + + return EncodeToken(this.containerClient.Name, blobName); + } + + /// + public async Task DownloadAsync(string token, CancellationToken cancellationToken) + { + (string container, string name) = DecodeToken(token); + if (!string.Equals(container, this.containerClient.Name, StringComparison.Ordinal)) + { + throw new ArgumentException("Token container does not match configured container.", nameof(token)); + } + + BlobClient blob = this.containerClient.GetBlobClient(name); + using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + using GZipStream decompressedBlobStream = new GZipStream(result.Content, CompressionMode.Decompress); + using StreamReader reader = new(decompressedBlobStream, Encoding.UTF8); + return await reader.ReadToEndAsync(); + } + + static string EncodeToken(string container, string name) => $"dtp:v1:{container}:{name}"; + + static (string Container, string Name) DecodeToken(string token) + { + if (!token.StartsWith("dtp:v1:", StringComparison.Ordinal)) + { + throw new ArgumentException("Invalid external payload token.", nameof(token)); + } + + string rest = token.Substring("dtp:v1:".Length); + int sep = rest.IndexOf(':'); + if (sep <= 0 || sep >= rest.Length - 1) + { + throw new ArgumentException("Invalid external payload token format.", nameof(token)); + } + + return (rest.Substring(0, sep), rest.Substring(sep + 1)); + } +} diff --git a/src/Abstractions/Converters/IPayloadStore.cs b/src/Abstractions/Converters/IPayloadStore.cs new file mode 100644 index 000000000..2dc094656 --- /dev/null +++ b/src/Abstractions/Converters/IPayloadStore.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace Microsoft.DurableTask.Converters; + +/// +/// Abstraction for storing and retrieving large payloads out-of-band. +/// +public interface IPayloadStore +{ + /// + /// Uploads a payload and returns an opaque reference token that can be embedded in orchestration messages. + /// + /// The content type of the payload (e.g., application/json). + /// The payload bytes. + /// Cancellation token. + /// Opaque reference token. + Task UploadAsync(string contentType, ReadOnlyMemory payloadBytes, CancellationToken cancellationToken); + + /// + /// Downloads the payload referenced by the token. + /// + /// The opaque reference token. + /// Cancellation token. + /// Payload string. + Task DownloadAsync(string token, CancellationToken cancellationToken); +} diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Abstractions/Converters/LargePayloadDataConverter.cs new file mode 100644 index 000000000..327d17519 --- /dev/null +++ b/src/Abstractions/Converters/LargePayloadDataConverter.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Text; + +namespace Microsoft.DurableTask.Converters; + +/// +/// A DataConverter that wraps another DataConverter and externalizes payloads larger than a configured threshold. +/// It uploads large payloads to an and returns a reference token string. +/// On deserialization, it resolves tokens and feeds the underlying converter the original content. +/// +/// +/// Initializes a new instance of the class. +/// +/// The inner data converter to wrap. +/// The external payload store to use. +/// The options for the externalizing data converter. +/// Thrown when , , or is null. +public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPayloadStore payloadStore, LargePayloadStorageOptions largePayloadStorageOptions) : DataConverter +{ + const string TokenPrefix = "dtp:v1:"; // matches BlobExternalPayloadStore + + readonly DataConverter innerConverter = innerConverter ?? throw new ArgumentNullException(nameof(innerConverter)); + readonly IPayloadStore payLoadStore = payloadStore ?? throw new ArgumentNullException(nameof(payloadStore)); + readonly LargePayloadStorageOptions largePayloadStorageOptions = largePayloadStorageOptions ?? throw new ArgumentNullException(nameof(largePayloadStorageOptions)); + readonly Encoding utf8 = new UTF8Encoding(false); + + /// + public override bool UsesExternalStorage => this.largePayloadStorageOptions.Enabled || this.innerConverter.UsesExternalStorage; + + /// + /// Serializes the value to a JSON string and uploads it to the external payload store if it exceeds the configured threshold. + /// + /// The value to serialize. + /// The serialized value or the token if externalized. + public override string? Serialize(object? value) + { + if (value is null) + { + return null; + } + + string json = this.innerConverter.Serialize(value) ?? "null"; + if (!this.largePayloadStorageOptions.Enabled) + { + return json; + } + + int byteCount = this.utf8.GetByteCount(json); + if (byteCount < this.largePayloadStorageOptions.ExternalizeThresholdBytes) + { + return json; + } + + // Upload synchronously in this context by blocking on async. SDK call sites already run on threadpool. + byte[] bytes = this.utf8.GetBytes(json); + string token = this.payLoadStore.UploadAsync("application/json", bytes, CancellationToken.None).GetAwaiter().GetResult(); + return token; + } + + /// + /// Deserializes the JSON string or resolves the token to the original value. + /// + /// The JSON string or token. + /// The type to deserialize to. + /// The deserialized value. + public override object? Deserialize(string? data, Type targetType) + { + if (data is null) + { + return null; + } + + string toDeserialize = data; + if (this.largePayloadStorageOptions.Enabled && data.StartsWith(TokenPrefix, StringComparison.Ordinal)) + { + toDeserialize = this.payLoadStore.DownloadAsync(data, CancellationToken.None).GetAwaiter().GetResult(); + } + + return this.innerConverter.Deserialize(toDeserialize, targetType); + } +} diff --git a/src/Abstractions/Converters/LargePayloadStorageOptions.cs b/src/Abstractions/Converters/LargePayloadStorageOptions.cs new file mode 100644 index 000000000..19585f601 --- /dev/null +++ b/src/Abstractions/Converters/LargePayloadStorageOptions.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Intentionally no DataAnnotations to avoid extra package requirements in minimal hosts. +namespace Microsoft.DurableTask.Converters; + +/// +/// Options for externalized payload storage, used by SDKs to store large payloads out-of-band. +/// +public sealed class LargePayloadStorageOptions +{ + /// + /// Initializes a new instance of the class. + /// + /// The Azure Storage connection string to the customer's storage account. + public LargePayloadStorageOptions(string connectionString) + { + Check.NotNullOrEmpty(connectionString, nameof(connectionString)); + this.ConnectionString = connectionString; + } + + /// + /// Gets or sets a value indicating whether externalized payload storage is enabled. + /// + public bool Enabled { get; set; } = true; + + /// + /// Gets or sets the threshold in bytes at which payloads are externalized. Default is 900_000 bytes. + /// + public int ExternalizeThresholdBytes { get; set; } = 900_000; // leave headroom below 1MB + + /// + /// Gets or sets the Azure Storage connection string to the customer's storage account. Required. + /// + public string ConnectionString { get; set; } + + /// + /// Gets or sets the blob container name to use for payloads. Defaults to "durabletask-payloads". + /// + public string ContainerName { get; set; } = "durabletask-payloads"; +} diff --git a/src/Abstractions/DataConverter.cs b/src/Abstractions/DataConverter.cs index 3248761ac..6d33be6a5 100644 --- a/src/Abstractions/DataConverter.cs +++ b/src/Abstractions/DataConverter.cs @@ -10,12 +10,17 @@ namespace Microsoft.DurableTask; /// /// /// Implementations of this abstract class are free to use any serialization method. The default implementation -/// uses the JSON serializer from the System.Text.Json namespace. Currently only strings are supported as -/// the serialized representation of data. Byte array payloads and streams are not supported by this abstraction. -/// Note that these methods all accept null values, in which case the return value should also be null. +/// uses the JSON serializer from the System.Text.Json namespace. Implementations may optionally externalize +/// large payloads and return an opaque reference string that can be resolved during deserialization. +/// These methods all accept null values, in which case the return value should also be null. /// public abstract class DataConverter { + /// + /// Gets a value indicating whether this converter may return an external reference token instead of inline JSON. + /// + public virtual bool UsesExternalStorage => false; + /// /// Serializes into a text string. /// diff --git a/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs b/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs index 5c8547592..9e072a9a6 100644 --- a/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs +++ b/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using Microsoft.DurableTask.Converters; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; @@ -101,4 +102,39 @@ public static IDurableTaskClientBuilder UseDefaultVersion(this IDurableTaskClien builder.Configure(options => options.DefaultVersion = version); return builder; } + + /// + /// Enables externalized payload storage using Azure Blob Storage. + /// Registers , and wraps the + /// configured in an for this client name. + /// + /// The to configure. + /// The action to configure the . + /// The . + public static IDurableTaskClientBuilder UseExternalizedPayloads( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + builder.Services.AddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + + // Wrap DataConverter for this named client without building a ServiceProvider + builder.Services + .AddOptions(builder.Name) + .PostConfigure>((opt, store, monitor) => + { + LargePayloadStorageOptions opts = monitor.Get(builder.Name); + DataConverter inner = opt.DataConverter ?? Converters.JsonDataConverter.Default; + opt.DataConverter = new LargePayloadDataConverter(inner, store, opts); + }); + + return builder; + } } diff --git a/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs b/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs index 3f349b710..61e0d21dd 100644 --- a/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs +++ b/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs @@ -4,6 +4,7 @@ using Microsoft.DurableTask.Worker.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; +using Microsoft.DurableTask.Converters; using static Microsoft.DurableTask.Worker.DurableTaskWorkerOptions; namespace Microsoft.DurableTask.Worker; @@ -137,4 +138,36 @@ public static IDurableTaskWorkerBuilder UseOrchestrationFilter(this IDurableTask builder.Services.AddSingleton(filter); return builder; } + + /// + /// Enables externalized payload storage for the worker's data converter to mirror client behavior. + /// + /// The to configure. + /// The action to configure the . + /// The . + public static IDurableTaskWorkerBuilder UseExternalizedPayloads( + this IDurableTaskWorkerBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + builder.Services.AddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + + builder.Services + .AddOptions(builder.Name) + .PostConfigure>((opt, store, monitor) => + { + LargePayloadStorageOptions opts = monitor.Get(builder.Name); + DataConverter inner = opt.DataConverter ?? Converters.JsonDataConverter.Default; + opt.DataConverter = new LargePayloadDataConverter(inner, store, opts); + }); + + return builder; + } } diff --git a/src/Worker/Core/DurableTaskWorkerOptions.cs b/src/Worker/Core/DurableTaskWorkerOptions.cs index 703bbbd4d..c65ccdbd3 100644 --- a/src/Worker/Core/DurableTaskWorkerOptions.cs +++ b/src/Worker/Core/DurableTaskWorkerOptions.cs @@ -162,7 +162,6 @@ public DataConverter DataConverter /// internal bool DataConverterExplicitlySet { get; private set; } - /// /// Applies these option values to another. /// From 560ecabeb320ca90bc5a1fd9b3fa8e12f723ef68 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 12 Aug 2025 17:56:09 -0700 Subject: [PATCH 02/38] tests --- .../Converters/LargePayloadDataConverter.cs | 13 +- .../Converters/LargePayloadStorageOptions.cs | 10 +- .../LargePayloadTests.cs | 398 ++++++++++++++++++ 3 files changed, 419 insertions(+), 2 deletions(-) create mode 100644 test/Grpc.IntegrationTests/LargePayloadTests.cs diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Abstractions/Converters/LargePayloadDataConverter.cs index 327d17519..5748198de 100644 --- a/src/Abstractions/Converters/LargePayloadDataConverter.cs +++ b/src/Abstractions/Converters/LargePayloadDataConverter.cs @@ -78,6 +78,17 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay toDeserialize = this.payLoadStore.DownloadAsync(data, CancellationToken.None).GetAwaiter().GetResult(); } - return this.innerConverter.Deserialize(toDeserialize, targetType); + return this.innerConverter.Deserialize(StripArrayCharacters(toDeserialize), targetType); + } + + static string? StripArrayCharacters(string? input) + { + if (input != null && input.StartsWith('[') && input.EndsWith(']')) + { + // Strip the outer bracket characters + return input[1..^1]; + } + + return input; } } diff --git a/src/Abstractions/Converters/LargePayloadStorageOptions.cs b/src/Abstractions/Converters/LargePayloadStorageOptions.cs index 19585f601..3e4b1b0c4 100644 --- a/src/Abstractions/Converters/LargePayloadStorageOptions.cs +++ b/src/Abstractions/Converters/LargePayloadStorageOptions.cs @@ -9,6 +9,14 @@ namespace Microsoft.DurableTask.Converters; /// public sealed class LargePayloadStorageOptions { + /// + /// Initializes a new instance of the class. + /// Parameterless constructor required for options activation. + /// + public LargePayloadStorageOptions() + { + } + /// /// Initializes a new instance of the class. /// @@ -32,7 +40,7 @@ public LargePayloadStorageOptions(string connectionString) /// /// Gets or sets the Azure Storage connection string to the customer's storage account. Required. /// - public string ConnectionString { get; set; } + public string ConnectionString { get; set; } = string.Empty; /// /// Gets or sets the blob container name to use for payloads. Defaults to "durabletask-payloads". diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs new file mode 100644 index 000000000..d3cf89db3 --- /dev/null +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -0,0 +1,398 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Worker; +using Microsoft.Extensions.DependencyInjection; +using Xunit.Abstractions; + +namespace Microsoft.DurableTask.Grpc.Tests; + +public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture) +{ + [Fact] + public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() + { + string largeInput = new string('A', 1024 * 1024); // 1MB + TaskName orchestratorName = nameof(OrchestrationInput_IsExternalizedByClient_ResolvedByWorker); + + InMemoryPayloadStore fakeStore = new InMemoryPayloadStore(); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks.AddOrchestratorFunc( + orchestratorName, + (ctx, input) => Task.FromResult(input))); + + // Enable externalization on the worker + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // small threshold to force externalization for test data + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + + // Override store with in-memory test double + worker.Services.AddSingleton(fakeStore); + }, + client => + { + // Enable externalization on the client + client.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + + // Override store with in-memory test double + client.Services.AddSingleton(fakeStore); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: largeInput); + + OrchestrationMetadata completed = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.Equal(OrchestrationRuntimeStatus.Completed, completed.RuntimeStatus); + + // Validate that the input made a roundtrip and was resolved on the worker + string? echoed = completed.ReadOutputAs(); + Assert.NotNull(echoed); + Assert.Equal(largeInput.Length, echoed!.Length); + + // Ensure client externalized the input + Assert.True(fakeStore.UploadCount >= 1); + } + + [Fact] + public async Task ActivityInput_IsExternalizedByWorker_ResolvedByActivity() + { + string largeParam = new string('P', 700 * 1024); // 700KB + TaskName orchestratorName = nameof(ActivityInput_IsExternalizedByWorker_ResolvedByActivity); + TaskName activityName = "EchoLength"; + + InMemoryPayloadStore workerStore = new InMemoryPayloadStore(); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + (ctx, _) => ctx.CallActivityAsync(activityName, largeParam)) + .AddActivityFunc(activityName, (ctx, input) => input.Length)); + + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // force externalization for activity input + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + worker.Services.AddSingleton(workerStore); + }, + client => { /* client not needed for externalization path here */ }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName); + OrchestrationMetadata completed = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.Equal(OrchestrationRuntimeStatus.Completed, completed.RuntimeStatus); + Assert.Equal(largeParam.Length, completed.ReadOutputAs()); + + // Worker externalizes when sending activity input; worker resolves when delivering to activity + Assert.True(workerStore.UploadCount >= 1); + Assert.True(workerStore.DownloadCount >= 1); + } + + [Fact] + public async Task ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator() + { + string largeResult = new string('R', 850 * 1024); // 850KB + TaskName orchestratorName = nameof(ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator); + TaskName activityName = "ProduceLarge"; + + InMemoryPayloadStore workerStore = new InMemoryPayloadStore(); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks + .AddOrchestratorFunc( + orchestratorName, + async (ctx, _) => (await ctx.CallActivityAsync(activityName)).Length) + .AddActivityFunc(activityName, (ctx) => Task.FromResult(largeResult))); + + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // force externalization for activity result + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + worker.Services.AddSingleton(workerStore); + }, + client => { }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName); + OrchestrationMetadata completed = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.Equal(OrchestrationRuntimeStatus.Completed, completed.RuntimeStatus); + Assert.Equal(largeResult.Length, completed.ReadOutputAs()); + + // Worker externalizes activity output and downloads when the orchestrator reads it + Assert.True(workerStore.UploadCount >= 1); + Assert.True(workerStore.DownloadCount >= 1); + } + + [Fact] + public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() + { + string largeOutput = new string('Q', 900 * 1024); // 900KB + string smallInput = "input"; + TaskName orchestratorName = nameof(QueryCompletedInstance_DownloadsExternalizedOutputOnClient); + + Dictionary shared = new System.Collections.Generic.Dictionary(); + InMemoryPayloadStore workerStore = new InMemoryPayloadStore(shared); + InMemoryPayloadStore clientStore = new InMemoryPayloadStore(shared); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks.AddOrchestratorFunc( + orchestratorName, + (ctx, _) => Task.FromResult(largeOutput))); + + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // force externalization on worker + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + worker.Services.AddSingleton(workerStore); + }, + client => + { + client.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // allow client to resolve on query + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + client.Services.AddSingleton(clientStore); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: smallInput); + await server.Client.WaitForInstanceCompletionAsync(instanceId, getInputsAndOutputs: false, this.TimeoutToken); + + OrchestrationMetadata? queried = await server.Client.GetInstanceAsync(instanceId, getInputsAndOutputs: true); + + Assert.NotNull(queried); + Assert.Equal(OrchestrationRuntimeStatus.Completed, queried!.RuntimeStatus); + Assert.Equal(smallInput, queried.ReadInputAs()); + Assert.Equal(largeOutput, queried.ReadOutputAs()); + + Assert.True(workerStore.UploadCount == 0); + Assert.True(clientStore.DownloadCount == 1); + Assert.True(clientStore.UploadCount == 1); + } + + [Fact] + public async Task BelowThreshold_NotExternalized() + { + string smallPayload = new string('X', 64 * 1024); // 64KB + TaskName orchestratorName = nameof(BelowThreshold_NotExternalized); + + InMemoryPayloadStore workerStore = new InMemoryPayloadStore(); + InMemoryPayloadStore clientStore = new InMemoryPayloadStore(); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks.AddOrchestratorFunc( + orchestratorName, + (ctx, input) => Task.FromResult(input))); + + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 2 * 1024 * 1024; // 2MB, higher than payload + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + worker.Services.AddSingleton(workerStore); + }, + client => + { + client.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 2 * 1024 * 1024; // 2MB, higher than payload + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + client.Services.AddSingleton(clientStore); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName, input: smallPayload); + OrchestrationMetadata completed = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.Equal(OrchestrationRuntimeStatus.Completed, completed.RuntimeStatus); + Assert.Equal(smallPayload, completed.ReadOutputAs()); + + Assert.Equal(0, workerStore.UploadCount); + Assert.Equal(0, workerStore.DownloadCount); + Assert.Equal(0, clientStore.UploadCount); + Assert.Equal(0, clientStore.DownloadCount); + } + + [Fact] + public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() + { + string largeEvent = new string('E', 512 * 1024); // 512KB + TaskName orchestratorName = nameof(ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker); + const string EventName = "LargeEvent"; + + InMemoryPayloadStore fakeStore = new InMemoryPayloadStore(); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks.AddOrchestratorFunc( + orchestratorName, + async ctx => await ctx.WaitForExternalEvent(EventName))); + + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // force externalization + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + worker.Services.AddSingleton(fakeStore); + }, + client => + { + client.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // force externalization + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + client.Services.AddSingleton(fakeStore); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName); + await server.Client.WaitForInstanceStartAsync(instanceId, this.TimeoutToken); + + await server.Client.RaiseEventAsync(instanceId, EventName, largeEvent, this.TimeoutToken); + + OrchestrationMetadata completed = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.Equal(OrchestrationRuntimeStatus.Completed, completed.RuntimeStatus); + string? output = completed.ReadOutputAs(); + Assert.Equal(largeEvent, output); + Assert.True(fakeStore.UploadCount >= 1); + } + + [Fact] + public async Task OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery() + { + string largeOutput = new string('O', 768 * 1024); // 768KB + string largeStatus = new string('S', 600 * 1024); // 600KB + TaskName orchestratorName = nameof(OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery); + + InMemoryPayloadStore fakeStore = new InMemoryPayloadStore(); + + await using HostTestLifetime server = await this.StartWorkerAsync( + worker => + { + worker.AddTasks(tasks => tasks.AddOrchestratorFunc( + orchestratorName, + async (ctx, _) => + { + ctx.SetCustomStatus(largeStatus); + await ctx.CreateTimer(TimeSpan.Zero, CancellationToken.None); + return largeOutput; + })); + + worker.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // ensure externalization for status/output + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + worker.Services.AddSingleton(fakeStore); + }, + client => + { + client.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // ensure resolution on query + opts.ContainerName = "test"; + opts.ConnectionString = "UseDevelopmentStorage=true"; + }); + client.Services.AddSingleton(fakeStore); + }); + + string instanceId = await server.Client.ScheduleNewOrchestrationInstanceAsync(orchestratorName); + + OrchestrationMetadata completed = await server.Client.WaitForInstanceCompletionAsync( + instanceId, getInputsAndOutputs: true, this.TimeoutToken); + + Assert.Equal(OrchestrationRuntimeStatus.Completed, completed.RuntimeStatus); + Assert.Equal(largeOutput, completed.ReadOutputAs()); + Assert.Equal(largeStatus, completed.ReadCustomStatusAs()); + + // Worker may externalize both status and output + Assert.True(fakeStore.UploadCount >= 2); + } + + class InMemoryPayloadStore : IPayloadStore + { + readonly Dictionary tokenToPayload; + + public InMemoryPayloadStore() + : this(new Dictionary()) + { + } + + public InMemoryPayloadStore(Dictionary shared) + { + this.tokenToPayload = shared; + } + + int uploadCount; + public int UploadCount => this.uploadCount; + int downloadCount; + public int DownloadCount => this.downloadCount; + + public Task UploadAsync(string contentType, ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) + { + Interlocked.Increment(ref this.uploadCount); + string json = System.Text.Encoding.UTF8.GetString(payloadBytes.Span); + string token = $"dtp:v1:test:{Guid.NewGuid():N}"; + this.tokenToPayload[token] = json; + return Task.FromResult(token); + } + + public Task DownloadAsync(string token, CancellationToken cancellationToken) + { + Interlocked.Increment(ref this.downloadCount); + return Task.FromResult(this.tokenToPayload[token]); + } + } +} From 0f3e6248bae7ed28cb126f824e6e371eac987e52 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 13 Aug 2025 09:23:06 -0700 Subject: [PATCH 03/38] add sample --- .../LargePayloadConsoleApp.csproj | 23 ++++ samples/LargePayloadConsoleApp/Program.cs | 106 ++++++++++++++++++ samples/LargePayloadConsoleApp/README.md | 29 +++++ samples/LargePayloadConsoleApp/run.ps1 | 81 +++++++++++++ .../Converters/BlobPayloadStore.cs | 10 +- .../Converters/LargePayloadDataConverter.cs | 2 +- .../LargePayloadTests.cs | 2 +- 7 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj create mode 100644 samples/LargePayloadConsoleApp/Program.cs create mode 100644 samples/LargePayloadConsoleApp/README.md create mode 100644 samples/LargePayloadConsoleApp/run.ps1 diff --git a/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj b/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj new file mode 100644 index 000000000..af4c89059 --- /dev/null +++ b/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj @@ -0,0 +1,23 @@ + + + + Exe + net8.0 + enable + + + + + + + + + + + + + + + + + diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs new file mode 100644 index 000000000..5007dab5a --- /dev/null +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask; +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.AzureManaged; +using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Worker; +using Microsoft.DurableTask.Worker.AzureManaged; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Configuration; + +// Demonstrates Large Payload Externalization using Azure Blob Storage. +// This sample uses Azurite/emulator by default via UseDevelopmentStorage=true. + +HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); + +// Connection string for Durable Task Scheduler +string schedulerConnectionString = builder.Configuration.GetValue("DURABLE_TASK_SCHEDULER_CONNECTION_STRING") + ?? throw new InvalidOperationException("Missing required configuration 'DURABLE_TASK_SCHEDULER_CONNECTION_STRING'"); + +// Configure Durable Task client with Durable Task Scheduler and externalized payloads +builder.Services.AddDurableTaskClient(b => +{ + b.UseDurableTaskScheduler(schedulerConnectionString); + b.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + // Keep threshold small to force externalization for demo purposes + opts.ExternalizeThresholdBytes = 1024; // 1KB + // Default to local Azurite/emulator. Override via environment or appsettings if desired. + opts.ConnectionString = Environment.GetEnvironmentVariable("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; + opts.ContainerName = Environment.GetEnvironmentVariable("DURABLETASK_PAYLOAD_CONTAINER") ?? "durabletask-payloads"; + }); +}); + +// Configure Durable Task worker with tasks and externalized payloads +builder.Services.AddDurableTaskWorker(b => +{ + b.UseDurableTaskScheduler(schedulerConnectionString); + b.AddTasks(tasks => + { + // Orchestrator: call activity first, return its output (should equal original input) + tasks.AddOrchestratorFunc("LargeInputEcho", async (ctx, input) => + { + string echoed = await ctx.CallActivityAsync("Echo", input); + return echoed; + }); + + // Activity: validate it receives raw input (not token) and return it + tasks.AddActivityFunc("Echo", (ctx, value) => + { + if (value is null) + { + return string.Empty; + } + + // If we ever see a token in the activity, externalization is not being resolved correctly. + if (value.StartsWith("dts:v1:", StringComparison.Ordinal)) + { + throw new InvalidOperationException("Activity received a payload token instead of raw input."); + } + + return value; + }); + }); + b.UseExternalizedPayloads(opts => + { + opts.Enabled = true; + opts.ExternalizeThresholdBytes = 1024; // mirror client + opts.ConnectionString = Environment.GetEnvironmentVariable("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; + opts.ContainerName = Environment.GetEnvironmentVariable("DURABLETASK_PAYLOAD_CONTAINER") ?? "durabletask-payloads"; + }); +}); + +IHost host = builder.Build(); +await host.StartAsync(); + +await using DurableTaskClient client = host.Services.GetRequiredService(); + +// Option A: Directly pass an oversized input to orchestration to trigger externalization +string largeInput = new string('B', 1024 * 1024); // 1MB +string instanceId = await client.ScheduleNewOrchestrationInstanceAsync("LargeInputEcho", largeInput); +Console.WriteLine($"Started orchestration with direct large input. Instance: {instanceId}"); + + +using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); +OrchestrationMetadata result = await client.WaitForInstanceCompletionAsync( + instanceId, + getInputsAndOutputs: true, + cts.Token); + +Console.WriteLine($"RuntimeStatus: {result.RuntimeStatus}"); +Console.WriteLine($"UsesExternalStorage (result converter): {result.DataConverter?.UsesExternalStorage ?? false}"); +string deserializedInput = result.ReadInputAs() ?? string.Empty; +string deserializedOutput = result.ReadOutputAs() ?? string.Empty; + +Console.WriteLine($"SerializedInput: {result.SerializedInput}"); +Console.WriteLine($"SerializedOutput: {result.SerializedOutput}"); +Console.WriteLine($"Deserialized input equals original: {deserializedInput == largeInput}"); +Console.WriteLine($"Deserialized output equals original: {deserializedOutput == largeInput}"); +Console.WriteLine($"Deserialized input length: {deserializedInput.Length}"); + + + diff --git a/samples/LargePayloadConsoleApp/README.md b/samples/LargePayloadConsoleApp/README.md new file mode 100644 index 000000000..812098efd --- /dev/null +++ b/samples/LargePayloadConsoleApp/README.md @@ -0,0 +1,29 @@ +# Large Payload Externalization Sample + +This sample demonstrates configuring Durable Task to externalize large payloads to Azure Blob Storage using `UseExternalizedPayloads` on both client and worker, connecting via Durable Task Scheduler (no local sidecar). + +- Defaults to Azurite/Storage Emulator via `UseDevelopmentStorage=true`. +- Threshold is set to 1KB for demo, so even modest inputs are externalized. + +## Prerequisites + +- A Durable Task Scheduler connection string (e.g., from Azure portal) in `DURABLE_TASK_SCHEDULER_CONNECTION_STRING`. +- Optional: Run Azurite (if not using real Azure Storage) for payload storage tokens. + +## Configure + +Environment variables (optional): + +- `DURABLETASK_STORAGE`: Azure Storage connection string. Defaults to `UseDevelopmentStorage=true`. +- `DURABLETASK_PAYLOAD_CONTAINER`: Blob container name. Defaults to `durabletask-payloads`. + +## Run + +```bash +# from repo root +dotnet run --project samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj +``` + +The app starts an orchestration with a 1MB input, which is externalized by the client and resolved by the worker. The console shows a token-like serialized input and a deserialized input length. + + diff --git a/samples/LargePayloadConsoleApp/run.ps1 b/samples/LargePayloadConsoleApp/run.ps1 new file mode 100644 index 000000000..466f07d72 --- /dev/null +++ b/samples/LargePayloadConsoleApp/run.ps1 @@ -0,0 +1,81 @@ +Param( + [Parameter(Mandatory = $true)] + [string]$SchedulerConnectionString, + + [string]$StorageConnectionString = "UseDevelopmentStorage=true", + + [string]$PayloadContainer = "durabletask-payloads", + + [switch]$StartAzurite, + + [switch]$VerboseLogging +) + +$ErrorActionPreference = "Stop" + +function Write-Info($msg) { + Write-Host "[info] $msg" +} + +function Start-AzuriteDocker { + param( + [string]$ContainerName = "durabletask-azurite" + ) + + if (-not (Get-Command docker -ErrorAction SilentlyContinue)) { + Write-Info "Docker not found; skipping Azurite startup." + return $false + } + + try { + $existing = (docker ps -a --filter "name=$ContainerName" --format "{{.ID}}") + if ($existing) { + Write-Info "Starting existing Azurite container '$ContainerName'..." + docker start $ContainerName | Out-Null + return $true + } + + Write-Info "Launching Azurite in Docker as '$ContainerName' on ports 10000-10002..." + docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 --name $ContainerName mcr.microsoft.com/azure-storage/azurite | Out-Null + Start-Sleep -Seconds 2 + return $true + } + catch { + Write-Warning "Failed to start Azurite via Docker: $_" + return $false + } +} + +try { + # Set required/optional environment variables for the sample + $env:DURABLE_TASK_SCHEDULER_CONNECTION_STRING = $SchedulerConnectionString + $env:DURABLETASK_STORAGE = $StorageConnectionString + $env:DURABLETASK_PAYLOAD_CONTAINER = $PayloadContainer + + Write-Info "DURABLE_TASK_SCHEDULER_CONNECTION_STRING is set." + Write-Info "DURABLETASK_STORAGE = '$($env:DURABLETASK_STORAGE)'" + Write-Info "DURABLETASK_PAYLOAD_CONTAINER = '$($env:DURABLETASK_PAYLOAD_CONTAINER)'" + + if ($StartAzurite) { + $started = Start-AzuriteDocker + if ($started) { + Write-Info "Azurite is running (Docker)." + } + } + + $projectPath = Join-Path $PSScriptRoot "LargePayloadConsoleApp.csproj" + if (-not (Test-Path $projectPath)) { + throw "Project file not found at $projectPath" + } + + Write-Info "Running sample..." + $argsList = @("run", "--project", $projectPath) + if ($VerboseLogging) { $argsList += @("-v", "detailed") } + + & dotnet @argsList +} +catch { + Write-Error $_ + exit 1 +} + diff --git a/src/Abstractions/Converters/BlobPayloadStore.cs b/src/Abstractions/Converters/BlobPayloadStore.cs index c746a5170..a6cdbaec3 100644 --- a/src/Abstractions/Converters/BlobPayloadStore.cs +++ b/src/Abstractions/Converters/BlobPayloadStore.cs @@ -12,7 +12,7 @@ namespace Microsoft.DurableTask.Converters; /// /// Azure Blob Storage implementation of . -/// Stores payloads as blobs and returns opaque tokens in the form "dtp:v1:<container>:<blobName>". +/// Stores payloads as blobs and returns opaque tokens in the form "dts:v1:<container>:<blobName>". /// public sealed class BlobPayloadStore : IPayloadStore { @@ -44,7 +44,7 @@ public async Task UploadAsync(string contentType, ReadOnlyMemory p // One blob per payload using GUID-based name for uniqueness string timestamp = DateTimeOffset.UtcNow.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture); - string blobName = $"{timestamp}/{Guid.NewGuid():N}.bin"; + string blobName = $"{timestamp}/{Guid.NewGuid():N}"; BlobClient blob = this.containerClient.GetBlobClient(blobName); byte[] payloadBuffer = payloadBytes.ToArray(); @@ -77,16 +77,16 @@ public async Task DownloadAsync(string token, CancellationToken cancella return await reader.ReadToEndAsync(); } - static string EncodeToken(string container, string name) => $"dtp:v1:{container}:{name}"; + static string EncodeToken(string container, string name) => $"dts:v1:{container}:{name}"; static (string Container, string Name) DecodeToken(string token) { - if (!token.StartsWith("dtp:v1:", StringComparison.Ordinal)) + if (!token.StartsWith("dts:v1:", StringComparison.Ordinal)) { throw new ArgumentException("Invalid external payload token.", nameof(token)); } - string rest = token.Substring("dtp:v1:".Length); + string rest = token.Substring("dts:v1:".Length); int sep = rest.IndexOf(':'); if (sep <= 0 || sep >= rest.Length - 1) { diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Abstractions/Converters/LargePayloadDataConverter.cs index 5748198de..e3b9da09a 100644 --- a/src/Abstractions/Converters/LargePayloadDataConverter.cs +++ b/src/Abstractions/Converters/LargePayloadDataConverter.cs @@ -19,7 +19,7 @@ namespace Microsoft.DurableTask.Converters; /// Thrown when , , or is null. public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPayloadStore payloadStore, LargePayloadStorageOptions largePayloadStorageOptions) : DataConverter { - const string TokenPrefix = "dtp:v1:"; // matches BlobExternalPayloadStore + const string TokenPrefix = "dts:v1:"; // matches BlobExternalPayloadStore readonly DataConverter innerConverter = innerConverter ?? throw new ArgumentNullException(nameof(innerConverter)); readonly IPayloadStore payLoadStore = payloadStore ?? throw new ArgumentNullException(nameof(payloadStore)); diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index d3cf89db3..01ab3d1a4 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -384,7 +384,7 @@ public Task UploadAsync(string contentType, ReadOnlyMemory payload { Interlocked.Increment(ref this.uploadCount); string json = System.Text.Encoding.UTF8.GetString(payloadBytes.Span); - string token = $"dtp:v1:test:{Guid.NewGuid():N}"; + string token = $"dts:v1:test:{Guid.NewGuid():N}"; this.tokenToPayload[token] = json; return Task.FromResult(token); } From d2b514f42f8e04f24f4aac713a701880b8cadd66 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 13 Aug 2025 13:18:51 -0700 Subject: [PATCH 04/38] precise to second --- src/Abstractions/Converters/BlobPayloadStore.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Abstractions/Converters/BlobPayloadStore.cs b/src/Abstractions/Converters/BlobPayloadStore.cs index a6cdbaec3..d17daa95f 100644 --- a/src/Abstractions/Converters/BlobPayloadStore.cs +++ b/src/Abstractions/Converters/BlobPayloadStore.cs @@ -43,7 +43,7 @@ public async Task UploadAsync(string contentType, ReadOnlyMemory p await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false); // One blob per payload using GUID-based name for uniqueness - string timestamp = DateTimeOffset.UtcNow.ToString("yyyy/MM/dd/HH", CultureInfo.InvariantCulture); + string timestamp = DateTimeOffset.UtcNow.ToString("yyyy/MM/dd/HH/mm/ss", CultureInfo.InvariantCulture); string blobName = $"{timestamp}/{Guid.NewGuid():N}"; BlobClient blob = this.containerClient.GetBlobClient(blobName); From 9d2bfa37027a1fd4340ba90e01c405335277ac31 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 27 Aug 2025 16:00:01 -0700 Subject: [PATCH 05/38] some fb --- src/Abstractions/Converters/BlobPayloadStore.cs | 2 +- src/Abstractions/Converters/IPayloadStore.cs | 3 +-- src/Abstractions/Converters/LargePayloadDataConverter.cs | 2 +- src/Abstractions/DataConverter.cs | 8 +++++--- test/Grpc.IntegrationTests/LargePayloadTests.cs | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Abstractions/Converters/BlobPayloadStore.cs b/src/Abstractions/Converters/BlobPayloadStore.cs index d17daa95f..8b9d9d522 100644 --- a/src/Abstractions/Converters/BlobPayloadStore.cs +++ b/src/Abstractions/Converters/BlobPayloadStore.cs @@ -37,7 +37,7 @@ public BlobPayloadStore(LargePayloadStorageOptions options) } /// - public async Task UploadAsync(string contentType, ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) + public async Task UploadAsync(ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) { // Ensure container exists await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false); diff --git a/src/Abstractions/Converters/IPayloadStore.cs b/src/Abstractions/Converters/IPayloadStore.cs index 2dc094656..c7796abbc 100644 --- a/src/Abstractions/Converters/IPayloadStore.cs +++ b/src/Abstractions/Converters/IPayloadStore.cs @@ -11,11 +11,10 @@ public interface IPayloadStore /// /// Uploads a payload and returns an opaque reference token that can be embedded in orchestration messages. /// - /// The content type of the payload (e.g., application/json). /// The payload bytes. /// Cancellation token. /// Opaque reference token. - Task UploadAsync(string contentType, ReadOnlyMemory payloadBytes, CancellationToken cancellationToken); + Task UploadAsync(ReadOnlyMemory payloadBytes, CancellationToken cancellationToken); /// /// Downloads the payload referenced by the token. diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Abstractions/Converters/LargePayloadDataConverter.cs index e3b9da09a..06ae16689 100644 --- a/src/Abstractions/Converters/LargePayloadDataConverter.cs +++ b/src/Abstractions/Converters/LargePayloadDataConverter.cs @@ -55,7 +55,7 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay // Upload synchronously in this context by blocking on async. SDK call sites already run on threadpool. byte[] bytes = this.utf8.GetBytes(json); - string token = this.payLoadStore.UploadAsync("application/json", bytes, CancellationToken.None).GetAwaiter().GetResult(); + string token = this.payLoadStore.UploadAsync(bytes, CancellationToken.None).GetAwaiter().GetResult(); return token; } diff --git a/src/Abstractions/DataConverter.cs b/src/Abstractions/DataConverter.cs index 6d33be6a5..a37f80812 100644 --- a/src/Abstractions/DataConverter.cs +++ b/src/Abstractions/DataConverter.cs @@ -10,9 +10,11 @@ namespace Microsoft.DurableTask; /// /// /// Implementations of this abstract class are free to use any serialization method. The default implementation -/// uses the JSON serializer from the System.Text.Json namespace. Implementations may optionally externalize -/// large payloads and return an opaque reference string that can be resolved during deserialization. -/// These methods all accept null values, in which case the return value should also be null. +/// uses the JSON serializer from the System.Text.Json namespace. Currently only strings are supported as +/// the serialized representation of data. Byte array payloads and streams are not supported by this abstraction. +/// Note that these methods all accept null values, in which case the return value should also be null. +/// Implementations may choose to return a pointer or reference (such as an external token) to the data +/// instead of the actual serialized data itself. /// public abstract class DataConverter { diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 01ab3d1a4..87ecda0a3 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -380,7 +380,7 @@ public InMemoryPayloadStore(Dictionary shared) int downloadCount; public int DownloadCount => this.downloadCount; - public Task UploadAsync(string contentType, ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) + public Task UploadAsync(ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) { Interlocked.Increment(ref this.uploadCount); string json = System.Text.Encoding.UTF8.GetString(payloadBytes.Span); From ef0961afb5c61d62151b4f204d4e618b8c9f1cda Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 27 Aug 2025 18:52:38 -0700 Subject: [PATCH 06/38] rename dts to blob --- samples/LargePayloadConsoleApp/Program.cs | 2 +- src/Abstractions/Converters/BlobPayloadStore.cs | 8 ++++---- src/Abstractions/Converters/LargePayloadDataConverter.cs | 2 +- test/Grpc.IntegrationTests/LargePayloadTests.cs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index 5007dab5a..df8cabb4c 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -57,7 +57,7 @@ } // If we ever see a token in the activity, externalization is not being resolved correctly. - if (value.StartsWith("dts:v1:", StringComparison.Ordinal)) + if (value.StartsWith("blob:v1:", StringComparison.Ordinal)) { throw new InvalidOperationException("Activity received a payload token instead of raw input."); } diff --git a/src/Abstractions/Converters/BlobPayloadStore.cs b/src/Abstractions/Converters/BlobPayloadStore.cs index 8b9d9d522..637ccf125 100644 --- a/src/Abstractions/Converters/BlobPayloadStore.cs +++ b/src/Abstractions/Converters/BlobPayloadStore.cs @@ -12,7 +12,7 @@ namespace Microsoft.DurableTask.Converters; /// /// Azure Blob Storage implementation of . -/// Stores payloads as blobs and returns opaque tokens in the form "dts:v1:<container>:<blobName>". +/// Stores payloads as blobs and returns opaque tokens in the form "blob:v1:<container>:<blobName>". /// public sealed class BlobPayloadStore : IPayloadStore { @@ -77,16 +77,16 @@ public async Task DownloadAsync(string token, CancellationToken cancella return await reader.ReadToEndAsync(); } - static string EncodeToken(string container, string name) => $"dts:v1:{container}:{name}"; + static string EncodeToken(string container, string name) => $"blob:v1:{container}:{name}"; static (string Container, string Name) DecodeToken(string token) { - if (!token.StartsWith("dts:v1:", StringComparison.Ordinal)) + if (!token.StartsWith("blob:v1:", StringComparison.Ordinal)) { throw new ArgumentException("Invalid external payload token.", nameof(token)); } - string rest = token.Substring("dts:v1:".Length); + string rest = token.Substring("blob:v1:".Length); int sep = rest.IndexOf(':'); if (sep <= 0 || sep >= rest.Length - 1) { diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Abstractions/Converters/LargePayloadDataConverter.cs index 06ae16689..69080e0d2 100644 --- a/src/Abstractions/Converters/LargePayloadDataConverter.cs +++ b/src/Abstractions/Converters/LargePayloadDataConverter.cs @@ -19,7 +19,7 @@ namespace Microsoft.DurableTask.Converters; /// Thrown when , , or is null. public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPayloadStore payloadStore, LargePayloadStorageOptions largePayloadStorageOptions) : DataConverter { - const string TokenPrefix = "dts:v1:"; // matches BlobExternalPayloadStore + const string TokenPrefix = "blob:v1:"; // matches BlobExternalPayloadStore readonly DataConverter innerConverter = innerConverter ?? throw new ArgumentNullException(nameof(innerConverter)); readonly IPayloadStore payLoadStore = payloadStore ?? throw new ArgumentNullException(nameof(payloadStore)); diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 87ecda0a3..58d1f7906 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -384,7 +384,7 @@ public Task UploadAsync(ReadOnlyMemory payloadBytes, CancellationT { Interlocked.Increment(ref this.uploadCount); string json = System.Text.Encoding.UTF8.GetString(payloadBytes.Span); - string token = $"dts:v1:test:{Guid.NewGuid():N}"; + string token = $"blob:v1:test:{Guid.NewGuid():N}"; this.tokenToPayload[token] = json; return Task.FromResult(token); } From 6387ad2fccb69c97b5b88413a56049e963a9dc04 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:34:59 -0700 Subject: [PATCH 07/38] some fb --- test/Grpc.IntegrationTests/LargePayloadTests.cs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 58d1f7906..5ffb5d2af 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -2,7 +2,9 @@ // Licensed under the MIT License. using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Entities; using Microsoft.DurableTask.Worker; using Microsoft.Extensions.DependencyInjection; using Xunit.Abstractions; @@ -11,6 +13,7 @@ namespace Microsoft.DurableTask.Grpc.Tests; public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture sidecarFixture) : IntegrationTestBase(output, sidecarFixture) { + // Validates client externalizes a large orchestration input and worker resolves it. [Fact] public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() { @@ -69,6 +72,7 @@ public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() Assert.True(fakeStore.UploadCount >= 1); } + // Validates worker externalizes large activity input and delivers resolved payload to activity. [Fact] public async Task ActivityInput_IsExternalizedByWorker_ResolvedByActivity() { @@ -110,6 +114,7 @@ public async Task ActivityInput_IsExternalizedByWorker_ResolvedByActivity() Assert.True(workerStore.DownloadCount >= 1); } + // Validates worker externalizes large activity output which is resolved by the orchestrator. [Fact] public async Task ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator() { @@ -151,6 +156,7 @@ public async Task ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator() Assert.True(workerStore.DownloadCount >= 1); } + // Ensures querying a completed instance downloads and resolves an externalized output on the client. [Fact] public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() { @@ -205,6 +211,7 @@ public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() Assert.True(clientStore.UploadCount == 1); } + // Ensures payloads below the threshold are not externalized by client or worker. [Fact] public async Task BelowThreshold_NotExternalized() { @@ -255,6 +262,7 @@ public async Task BelowThreshold_NotExternalized() Assert.Equal(0, clientStore.DownloadCount); } + // Validates client externalizes a large external event payload and worker resolves it. [Fact] public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() { @@ -306,6 +314,7 @@ public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() Assert.True(fakeStore.UploadCount >= 1); } + // Validates worker externalizes both output and custom status; client resolves them on query. [Fact] public async Task OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery() { From b4527cc5d5bcb83b0e56213357bacc57949248d5 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Thu, 28 Aug 2025 09:57:28 -0700 Subject: [PATCH 08/38] enabled always --- samples/LargePayloadConsoleApp/Program.cs | 2 -- .../Converters/LargePayloadDataConverter.cs | 8 ++------ .../Converters/LargePayloadStorageOptions.cs | 5 ----- test/Grpc.IntegrationTests/LargePayloadTests.cs | 12 ------------ 4 files changed, 2 insertions(+), 25 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index df8cabb4c..d962df4bc 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -26,7 +26,6 @@ b.UseDurableTaskScheduler(schedulerConnectionString); b.UseExternalizedPayloads(opts => { - opts.Enabled = true; // Keep threshold small to force externalization for demo purposes opts.ExternalizeThresholdBytes = 1024; // 1KB // Default to local Azurite/emulator. Override via environment or appsettings if desired. @@ -67,7 +66,6 @@ }); b.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // mirror client opts.ConnectionString = Environment.GetEnvironmentVariable("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; opts.ContainerName = Environment.GetEnvironmentVariable("DURABLETASK_PAYLOAD_CONTAINER") ?? "durabletask-payloads"; diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Abstractions/Converters/LargePayloadDataConverter.cs index 69080e0d2..48db66e66 100644 --- a/src/Abstractions/Converters/LargePayloadDataConverter.cs +++ b/src/Abstractions/Converters/LargePayloadDataConverter.cs @@ -27,7 +27,7 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay readonly Encoding utf8 = new UTF8Encoding(false); /// - public override bool UsesExternalStorage => this.largePayloadStorageOptions.Enabled || this.innerConverter.UsesExternalStorage; + public override bool UsesExternalStorage => true; /// /// Serializes the value to a JSON string and uploads it to the external payload store if it exceeds the configured threshold. @@ -42,10 +42,6 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay } string json = this.innerConverter.Serialize(value) ?? "null"; - if (!this.largePayloadStorageOptions.Enabled) - { - return json; - } int byteCount = this.utf8.GetByteCount(json); if (byteCount < this.largePayloadStorageOptions.ExternalizeThresholdBytes) @@ -73,7 +69,7 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay } string toDeserialize = data; - if (this.largePayloadStorageOptions.Enabled && data.StartsWith(TokenPrefix, StringComparison.Ordinal)) + if (data.StartsWith(TokenPrefix, StringComparison.Ordinal)) { toDeserialize = this.payLoadStore.DownloadAsync(data, CancellationToken.None).GetAwaiter().GetResult(); } diff --git a/src/Abstractions/Converters/LargePayloadStorageOptions.cs b/src/Abstractions/Converters/LargePayloadStorageOptions.cs index 3e4b1b0c4..d43e189d9 100644 --- a/src/Abstractions/Converters/LargePayloadStorageOptions.cs +++ b/src/Abstractions/Converters/LargePayloadStorageOptions.cs @@ -27,11 +27,6 @@ public LargePayloadStorageOptions(string connectionString) this.ConnectionString = connectionString; } - /// - /// Gets or sets a value indicating whether externalized payload storage is enabled. - /// - public bool Enabled { get; set; } = true; - /// /// Gets or sets the threshold in bytes at which payloads are externalized. Default is 900_000 bytes. /// diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 5ffb5d2af..5f64f8e5f 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -32,7 +32,6 @@ public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() // Enable externalization on the worker worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // small threshold to force externalization for test data opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -46,7 +45,6 @@ public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() // Enable externalization on the client client.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -93,7 +91,6 @@ public async Task ActivityInput_IsExternalizedByWorker_ResolvedByActivity() worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // force externalization for activity input opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -135,7 +132,6 @@ public async Task ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator() worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // force externalization for activity result opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -177,7 +173,6 @@ public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // force externalization on worker opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -188,7 +183,6 @@ public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() { client.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // allow client to resolve on query opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -230,7 +224,6 @@ public async Task BelowThreshold_NotExternalized() worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 2 * 1024 * 1024; // 2MB, higher than payload opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -241,7 +234,6 @@ public async Task BelowThreshold_NotExternalized() { client.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 2 * 1024 * 1024; // 2MB, higher than payload opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -281,7 +273,6 @@ public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // force externalization opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -292,7 +283,6 @@ public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() { client.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // force externalization opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -338,7 +328,6 @@ public async Task OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery() worker.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // ensure externalization for status/output opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; @@ -349,7 +338,6 @@ public async Task OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery() { client.UseExternalizedPayloads(opts => { - opts.Enabled = true; opts.ExternalizeThresholdBytes = 1024; // ensure resolution on query opts.ContainerName = "test"; opts.ConnectionString = "UseDevelopmentStorage=true"; From 9168dc90124e9b5b6b9dd401df0884825d0e93dd Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 31 Aug 2025 20:10:32 -0700 Subject: [PATCH 09/38] split package --- .../LargePayloadConsoleApp.csproj | 1 + src/Abstractions/Abstractions.csproj | 1 - .../DurableTaskClientBuilderExtensions.cs | 36 +------------- .../AzureBlobPayloads.csproj | 28 +++++++++++ .../Converters/BlobPayloadStore.cs | 4 +- .../Converters/LargePayloadDataConverter.cs | 2 + ...ientBuilderExtensions.AzureBlobPayloads.cs | 47 +++++++++++++++++++ ...rkerBuilderExtensions.AzureBlobPayloads.cs | 46 ++++++++++++++++++ .../DurableTaskWorkerBuilderExtensions.cs | 33 +------------ .../Grpc.IntegrationTests.csproj | 1 + 10 files changed, 130 insertions(+), 69 deletions(-) create mode 100644 src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj rename src/{Abstractions => Extensions/AzureBlobPayloads}/Converters/BlobPayloadStore.cs (99%) rename src/{Abstractions => Extensions/AzureBlobPayloads}/Converters/LargePayloadDataConverter.cs (99%) create mode 100644 src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs create mode 100644 src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs diff --git a/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj b/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj index af4c89059..b0f2914c7 100644 --- a/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj +++ b/samples/LargePayloadConsoleApp/LargePayloadConsoleApp.csproj @@ -16,6 +16,7 @@ + diff --git a/src/Abstractions/Abstractions.csproj b/src/Abstractions/Abstractions.csproj index ab32b4888..db8be76ab 100644 --- a/src/Abstractions/Abstractions.csproj +++ b/src/Abstractions/Abstractions.csproj @@ -13,7 +13,6 @@ - diff --git a/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs b/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs index 9e072a9a6..b3e91a8f5 100644 --- a/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs +++ b/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask.Converters; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Options; @@ -103,38 +102,5 @@ public static IDurableTaskClientBuilder UseDefaultVersion(this IDurableTaskClien return builder; } - /// - /// Enables externalized payload storage using Azure Blob Storage. - /// Registers , and wraps the - /// configured in an for this client name. - /// - /// The to configure. - /// The action to configure the . - /// The . - public static IDurableTaskClientBuilder UseExternalizedPayloads( - this IDurableTaskClientBuilder builder, - Action configure) - { - Check.NotNull(builder); - Check.NotNull(configure); - - builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); - - // Wrap DataConverter for this named client without building a ServiceProvider - builder.Services - .AddOptions(builder.Name) - .PostConfigure>((opt, store, monitor) => - { - LargePayloadStorageOptions opts = monitor.Get(builder.Name); - DataConverter inner = opt.DataConverter ?? Converters.JsonDataConverter.Default; - opt.DataConverter = new LargePayloadDataConverter(inner, store, opts); - }); - - return builder; - } + // Large payload enablement moved to Microsoft.DurableTask.Extensions.AzureBlobPayloads package. } diff --git a/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj b/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj new file mode 100644 index 000000000..54883acf4 --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj @@ -0,0 +1,28 @@ + + + + netstandard2.0 + Azure Blob Storage externalized payload support for Durable Task. + Microsoft.DurableTask.Extensions.AzureBlobPayloads + Microsoft.DurableTask + true + + + + + + + + + + + + + + + + + + + + diff --git a/src/Abstractions/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs similarity index 99% rename from src/Abstractions/Converters/BlobPayloadStore.cs rename to src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index 637ccf125..4e1275c68 100644 --- a/src/Abstractions/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -49,7 +49,7 @@ public async Task UploadAsync(ReadOnlyMemory payloadBytes, Cancell byte[] payloadBuffer = payloadBytes.ToArray(); - // Compress and upload streaming + // Compress and upload streaming using Stream blobStream = await blob.OpenWriteAsync(overwrite: true, cancellationToken: cancellationToken).ConfigureAwait(false); using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); using MemoryStream payloadStream = new(payloadBuffer, writable: false); @@ -96,3 +96,5 @@ public async Task DownloadAsync(string token, CancellationToken cancella return (rest.Substring(0, sep), rest.Substring(sep + 1)); } } + + diff --git a/src/Abstractions/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs similarity index 99% rename from src/Abstractions/Converters/LargePayloadDataConverter.cs rename to src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index 48db66e66..344c226c3 100644 --- a/src/Abstractions/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -88,3 +88,5 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay return input; } } + + diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs new file mode 100644 index 000000000..ef77d50fa --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Client; +using Microsoft.DurableTask.Converters; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.Client; + +/// +/// Extension methods to enable externalized payloads using Azure Blob Storage for Durable Task Client. +/// +public static class DurableTaskClientBuilderExtensionsAzureBlobPayloads +{ + /// + /// Enables externalized payload storage using Azure Blob Storage for the specified client builder. + /// + public static IDurableTaskClientBuilder UseExternalizedPayloads( + this IDurableTaskClientBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + builder.Services.AddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + + builder.Services + .AddOptions(builder.Name) + .PostConfigure>((opt, store, monitor) => + { + LargePayloadStorageOptions opts = monitor.Get(builder.Name); + DataConverter inner = opt.DataConverter ?? Converters.JsonDataConverter.Default; + opt.DataConverter = new LargePayloadDataConverter(inner, store, opts); + }); + + return builder; + } +} + + diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs new file mode 100644 index 000000000..8ae08fffa --- /dev/null +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskWorkerBuilderExtensions.AzureBlobPayloads.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Worker; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace Microsoft.DurableTask.Worker; + +/// +/// Extension methods to enable externalized payloads using Azure Blob Storage for Durable Task Worker. +/// +public static class DurableTaskWorkerBuilderExtensionsAzureBlobPayloads +{ + /// + /// Enables externalized payload storage using Azure Blob Storage for the specified worker builder. + /// + public static IDurableTaskWorkerBuilder UseExternalizedPayloads( + this IDurableTaskWorkerBuilder builder, + Action configure) + { + Check.NotNull(builder); + Check.NotNull(configure); + + builder.Services.Configure(builder.Name, configure); + builder.Services.AddSingleton(sp => + { + LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); + return new BlobPayloadStore(opts); + }); + + builder.Services + .AddOptions(builder.Name) + .PostConfigure>((opt, store, monitor) => + { + LargePayloadStorageOptions opts = monitor.Get(builder.Name); + DataConverter inner = opt.DataConverter ?? Converters.JsonDataConverter.Default; + opt.DataConverter = new LargePayloadDataConverter(inner, store, opts); + }); + + return builder; + } +} + + diff --git a/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs b/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs index 61e0d21dd..05cb4bc4b 100644 --- a/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs +++ b/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs @@ -4,7 +4,6 @@ using Microsoft.DurableTask.Worker.Hosting; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Options; -using Microsoft.DurableTask.Converters; using static Microsoft.DurableTask.Worker.DurableTaskWorkerOptions; namespace Microsoft.DurableTask.Worker; @@ -139,35 +138,5 @@ public static IDurableTaskWorkerBuilder UseOrchestrationFilter(this IDurableTask return builder; } - /// - /// Enables externalized payload storage for the worker's data converter to mirror client behavior. - /// - /// The to configure. - /// The action to configure the . - /// The . - public static IDurableTaskWorkerBuilder UseExternalizedPayloads( - this IDurableTaskWorkerBuilder builder, - Action configure) - { - Check.NotNull(builder); - Check.NotNull(configure); - - builder.Services.Configure(builder.Name, configure); - builder.Services.AddSingleton(sp => - { - LargePayloadStorageOptions opts = sp.GetRequiredService>().Get(builder.Name); - return new BlobPayloadStore(opts); - }); - - builder.Services - .AddOptions(builder.Name) - .PostConfigure>((opt, store, monitor) => - { - LargePayloadStorageOptions opts = monitor.Get(builder.Name); - DataConverter inner = opt.DataConverter ?? Converters.JsonDataConverter.Default; - opt.DataConverter = new LargePayloadDataConverter(inner, store, opts); - }); - - return builder; - } + // Large payload enablement moved to Microsoft.DurableTask.Extensions.AzureBlobPayloads package. } diff --git a/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj b/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj index e6b0aee76..ba2c8b687 100644 --- a/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj +++ b/test/Grpc.IntegrationTests/Grpc.IntegrationTests.csproj @@ -7,6 +7,7 @@ + From b5bedd081c6a2a6d31090187e2b7e1fc63edff5b Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 31 Aug 2025 21:08:19 -0700 Subject: [PATCH 10/38] update sample --- samples/LargePayloadConsoleApp/Program.cs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index d962df4bc..6ee592c32 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -28,9 +28,8 @@ { // Keep threshold small to force externalization for demo purposes opts.ExternalizeThresholdBytes = 1024; // 1KB - // Default to local Azurite/emulator. Override via environment or appsettings if desired. - opts.ConnectionString = Environment.GetEnvironmentVariable("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; - opts.ContainerName = Environment.GetEnvironmentVariable("DURABLETASK_PAYLOAD_CONTAINER") ?? "durabletask-payloads"; + opts.ConnectionString = builder.Configuration.GetValue("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; + opts.ContainerName = builder.Configuration.GetValue("DURABLETASK_PAYLOAD_CONTAINER"); }); }); @@ -67,8 +66,8 @@ b.UseExternalizedPayloads(opts => { opts.ExternalizeThresholdBytes = 1024; // mirror client - opts.ConnectionString = Environment.GetEnvironmentVariable("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; - opts.ContainerName = Environment.GetEnvironmentVariable("DURABLETASK_PAYLOAD_CONTAINER") ?? "durabletask-payloads"; + opts.ConnectionString = builder.Configuration.GetValue("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; + opts.ContainerName = builder.Configuration.GetValue("DURABLETASK_PAYLOAD_CONTAINER"); }); }); From 6cebf0e822f874c35ddb54d1decb5fb1c5ddf635 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 31 Aug 2025 21:33:39 -0700 Subject: [PATCH 11/38] remove enablestorage --- samples/LargePayloadConsoleApp/Program.cs | 1 - src/Abstractions/DataConverter.cs | 5 ----- .../Converters/LargePayloadDataConverter.cs | 3 --- 3 files changed, 9 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index 6ee592c32..10984292a 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -89,7 +89,6 @@ cts.Token); Console.WriteLine($"RuntimeStatus: {result.RuntimeStatus}"); -Console.WriteLine($"UsesExternalStorage (result converter): {result.DataConverter?.UsesExternalStorage ?? false}"); string deserializedInput = result.ReadInputAs() ?? string.Empty; string deserializedOutput = result.ReadOutputAs() ?? string.Empty; diff --git a/src/Abstractions/DataConverter.cs b/src/Abstractions/DataConverter.cs index a37f80812..6c623c814 100644 --- a/src/Abstractions/DataConverter.cs +++ b/src/Abstractions/DataConverter.cs @@ -18,11 +18,6 @@ namespace Microsoft.DurableTask; /// public abstract class DataConverter { - /// - /// Gets a value indicating whether this converter may return an external reference token instead of inline JSON. - /// - public virtual bool UsesExternalStorage => false; - /// /// Serializes into a text string. /// diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index 344c226c3..a28237945 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -26,9 +26,6 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay readonly LargePayloadStorageOptions largePayloadStorageOptions = largePayloadStorageOptions ?? throw new ArgumentNullException(nameof(largePayloadStorageOptions)); readonly Encoding utf8 = new UTF8Encoding(false); - /// - public override bool UsesExternalStorage => true; - /// /// Serializes the value to a JSON string and uploads it to the external payload store if it exceeds the configured threshold. /// From ce51c0d130481af3a231b3de4b7374a57c459698 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Mon, 1 Sep 2025 09:50:33 -0700 Subject: [PATCH 12/38] comment --- .../AzureBlobPayloads/Converters/LargePayloadDataConverter.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index a28237945..5c106ba1f 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -24,6 +24,10 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay readonly DataConverter innerConverter = innerConverter ?? throw new ArgumentNullException(nameof(innerConverter)); readonly IPayloadStore payLoadStore = payloadStore ?? throw new ArgumentNullException(nameof(payloadStore)); readonly LargePayloadStorageOptions largePayloadStorageOptions = largePayloadStorageOptions ?? throw new ArgumentNullException(nameof(largePayloadStorageOptions)); + // Use UTF-8 without a BOM (encoderShouldEmitUTF8Identifier=false). JSON in UTF-8 should not include a + // byte order mark per RFC 8259, and omitting it avoids hidden extra bytes that could skew the + // externalization threshold calculation and prevents interop issues with strict JSON parsers. + // A few legacy tools rely on a BOM for encoding detection, but modern JSON tooling assumes BOM-less UTF-8. readonly Encoding utf8 = new UTF8Encoding(false); /// From ecf89de5de3edb3d835df7a04e5c10bab6d53d33 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Mon, 1 Sep 2025 10:13:55 -0700 Subject: [PATCH 13/38] testname update --- .../LargePayloadTests.cs | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index 5f64f8e5f..dc7150b3a 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -15,10 +15,10 @@ public class LargePayloadTests(ITestOutputHelper output, GrpcSidecarFixture side { // Validates client externalizes a large orchestration input and worker resolves it. [Fact] - public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() + public async Task LargeOrchestrationInput() { string largeInput = new string('A', 1024 * 1024); // 1MB - TaskName orchestratorName = nameof(OrchestrationInput_IsExternalizedByClient_ResolvedByWorker); + TaskName orchestratorName = nameof(LargeOrchestrationInput); InMemoryPayloadStore fakeStore = new InMemoryPayloadStore(); @@ -72,10 +72,10 @@ public async Task OrchestrationInput_IsExternalizedByClient_ResolvedByWorker() // Validates worker externalizes large activity input and delivers resolved payload to activity. [Fact] - public async Task ActivityInput_IsExternalizedByWorker_ResolvedByActivity() + public async Task LargeActivityInput() { string largeParam = new string('P', 700 * 1024); // 700KB - TaskName orchestratorName = nameof(ActivityInput_IsExternalizedByWorker_ResolvedByActivity); + TaskName orchestratorName = nameof(LargeActivityInput); TaskName activityName = "EchoLength"; InMemoryPayloadStore workerStore = new InMemoryPayloadStore(); @@ -113,10 +113,10 @@ public async Task ActivityInput_IsExternalizedByWorker_ResolvedByActivity() // Validates worker externalizes large activity output which is resolved by the orchestrator. [Fact] - public async Task ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator() + public async Task LargeActivityOutput() { string largeResult = new string('R', 850 * 1024); // 850KB - TaskName orchestratorName = nameof(ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator); + TaskName orchestratorName = nameof(LargeActivityOutput); TaskName activityName = "ProduceLarge"; InMemoryPayloadStore workerStore = new InMemoryPayloadStore(); @@ -154,11 +154,11 @@ public async Task ActivityOutput_IsExternalizedByWorker_ResolvedByOrchestrator() // Ensures querying a completed instance downloads and resolves an externalized output on the client. [Fact] - public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() + public async Task LargeOrchestrationOutput() { string largeOutput = new string('Q', 900 * 1024); // 900KB string smallInput = "input"; - TaskName orchestratorName = nameof(QueryCompletedInstance_DownloadsExternalizedOutputOnClient); + TaskName orchestratorName = nameof(LargeOrchestrationOutput); Dictionary shared = new System.Collections.Generic.Dictionary(); InMemoryPayloadStore workerStore = new InMemoryPayloadStore(shared); @@ -207,10 +207,10 @@ public async Task QueryCompletedInstance_DownloadsExternalizedOutputOnClient() // Ensures payloads below the threshold are not externalized by client or worker. [Fact] - public async Task BelowThreshold_NotExternalized() + public async Task NoLargePayloads() { string smallPayload = new string('X', 64 * 1024); // 64KB - TaskName orchestratorName = nameof(BelowThreshold_NotExternalized); + TaskName orchestratorName = nameof(NoLargePayloads); InMemoryPayloadStore workerStore = new InMemoryPayloadStore(); InMemoryPayloadStore clientStore = new InMemoryPayloadStore(); @@ -256,10 +256,10 @@ public async Task BelowThreshold_NotExternalized() // Validates client externalizes a large external event payload and worker resolves it. [Fact] - public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() + public async Task LargeExternalEvent() { string largeEvent = new string('E', 512 * 1024); // 512KB - TaskName orchestratorName = nameof(ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker); + TaskName orchestratorName = nameof(LargeExternalEvent); const string EventName = "LargeEvent"; InMemoryPayloadStore fakeStore = new InMemoryPayloadStore(); @@ -306,11 +306,11 @@ public async Task ExternalEventPayload_IsExternalizedByClient_ResolvedByWorker() // Validates worker externalizes both output and custom status; client resolves them on query. [Fact] - public async Task OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery() + public async Task LargeOutputAndCustomStatus() { string largeOutput = new string('O', 768 * 1024); // 768KB string largeStatus = new string('S', 600 * 1024); // 600KB - TaskName orchestratorName = nameof(OutputAndCustomStatus_ExternalizedByWorker_ResolvedOnQuery); + TaskName orchestratorName = nameof(LargeOutputAndCustomStatus); InMemoryPayloadStore fakeStore = new InMemoryPayloadStore(); From 3bd6c9ae863b412f42b0ef54a9c1461a17cadbce Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 2 Sep 2025 15:30:27 -0700 Subject: [PATCH 14/38] add entity sample for largepayload --- samples/LargePayloadConsoleApp/Program.cs | 101 +++++++++++++++++++++- 1 file changed, 97 insertions(+), 4 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index 10984292a..9baa35f99 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -1,15 +1,15 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. -using Microsoft.DurableTask; using Microsoft.DurableTask.Client; using Microsoft.DurableTask.Client.AzureManaged; -using Microsoft.DurableTask.Converters; +using Microsoft.DurableTask.Client.Entities; +using Microsoft.DurableTask.Entities; using Microsoft.DurableTask.Worker; using Microsoft.DurableTask.Worker.AzureManaged; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Configuration; // Demonstrates Large Payload Externalization using Azure Blob Storage. // This sample uses Azurite/emulator by default via UseDevelopmentStorage=true. @@ -24,6 +24,8 @@ builder.Services.AddDurableTaskClient(b => { b.UseDurableTaskScheduler(schedulerConnectionString); + // Ensure entity APIs are enabled for the client + b.Configure(o => o.EnableEntitySupport = true); b.UseExternalizedPayloads(opts => { // Keep threshold small to force externalization for demo purposes @@ -62,6 +64,38 @@ return value; }); + + // Entity samples + // 1) Large entity operation input (worker externalizes input; entity receives resolved payload) + tasks.AddOrchestratorFunc( + "LargeEntityOperationInput", + (ctx, _) => ctx.Entities.CallEntityAsync( + new EntityInstanceId(nameof(EchoLengthEntity), "1"), + operationName: "EchoLength", + input: new string('E', 700 * 1024))); + tasks.AddEntity(nameof(EchoLengthEntity)); + + // 2) Large entity operation output (worker externalizes output; orchestrator reads resolved payload) + tasks.AddOrchestratorFunc( + "LargeEntityOperationOutput", + async (ctx, _) => (await ctx.Entities.CallEntityAsync( + new EntityInstanceId(nameof(LargeResultEntity), "1"), + operationName: "Produce", + input: 850 * 1024)).Length); + tasks.AddEntity(nameof(LargeResultEntity)); + + // 3) Large entity state (worker externalizes state; client resolves on query) + tasks.AddOrchestratorFunc( + "LargeEntityState", + async (ctx, _) => + { + await ctx.Entities.CallEntityAsync( + new EntityInstanceId(nameof(StateEntity), "1"), + operationName: "Set", + input: new string('S', 900 * 1024)); + return null; + }); + tasks.AddEntity(nameof(StateEntity)); }); b.UseExternalizedPayloads(opts => { @@ -69,6 +103,8 @@ opts.ConnectionString = builder.Configuration.GetValue("DURABLETASK_STORAGE") ?? "UseDevelopmentStorage=true"; opts.ContainerName = builder.Configuration.GetValue("DURABLETASK_PAYLOAD_CONTAINER"); }); + // Ensure entity APIs are enabled for the worker + b.Configure(o => o.EnableEntitySupport = true); }); IHost host = builder.Build(); @@ -82,7 +118,7 @@ Console.WriteLine($"Started orchestration with direct large input. Instance: {instanceId}"); -using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); +using CancellationTokenSource cts = new CancellationTokenSource(TimeSpan.FromSeconds(120)); OrchestrationMetadata result = await client.WaitForInstanceCompletionAsync( instanceId, getInputsAndOutputs: true, @@ -100,3 +136,60 @@ +// Run entity samples +Console.WriteLine(); +Console.WriteLine("Running LargeEntityOperationInput..."); +string entityInputInstance = await client.ScheduleNewOrchestrationInstanceAsync("LargeEntityOperationInput"); +OrchestrationMetadata entityInputResult = await client.WaitForInstanceCompletionAsync(entityInputInstance, getInputsAndOutputs: true, cts.Token); +Console.WriteLine($"Status: {entityInputResult.RuntimeStatus}, Output length: {entityInputResult.ReadOutputAs()}"); + +Console.WriteLine(); +Console.WriteLine("Running LargeEntityOperationOutput..."); +string entityOutputInstance = await client.ScheduleNewOrchestrationInstanceAsync("LargeEntityOperationOutput"); +OrchestrationMetadata entityOutputResult = await client.WaitForInstanceCompletionAsync(entityOutputInstance, getInputsAndOutputs: true, cts.Token); +Console.WriteLine($"Status: {entityOutputResult.RuntimeStatus}, Output length: {entityOutputResult.ReadOutputAs()}"); + +Console.WriteLine(); +Console.WriteLine("Running LargeEntityState and querying state..."); +string entityStateInstance = await client.ScheduleNewOrchestrationInstanceAsync("LargeEntityState"); +OrchestrationMetadata entityStateOrch = await client.WaitForInstanceCompletionAsync(entityStateInstance, getInputsAndOutputs: true, cts.Token); +Console.WriteLine($"Status: {entityStateOrch.RuntimeStatus}"); +EntityMetadata? state = await client.Entities.GetEntityAsync(new EntityInstanceId(nameof(StateEntity), "1"), includeState: true); +Console.WriteLine($"State length: {state?.State?.Length ?? 0}"); + + + + + +public class EchoLengthEntity : TaskEntity +{ + public int EchoLength(string input) + { + return input.Length; + } +} + +public class LargeResultEntity : TaskEntity +{ + public string Produce(int length) + { + return new string('R', length); + } +} + +public class StateEntity : TaskEntity +{ + protected override string? InitializeState(TaskEntityOperation entityOperation) + { + // Avoid Activator.CreateInstance() which throws; start as null (no state) + return null; + } + + public void Set(string value) + { + this.State = value; + } +} + + + From 20d3e8ff038612caa63661e6c796a1e8278e6dc6 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 17:12:35 -0700 Subject: [PATCH 15/38] some fb --- .../DurableTaskClientBuilderExtensions.cs | 2 -- .../AzureBlobPayloads/Converters/BlobPayloadStore.cs | 12 ++++++------ .../DurableTaskWorkerBuilderExtensions.cs | 2 -- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs b/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs index b3e91a8f5..5c8547592 100644 --- a/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs +++ b/src/Client/Core/DependencyInjection/DurableTaskClientBuilderExtensions.cs @@ -101,6 +101,4 @@ public static IDurableTaskClientBuilder UseDefaultVersion(this IDurableTaskClien builder.Configure(options => options.DefaultVersion = version); return builder; } - - // Large payload enablement moved to Microsoft.DurableTask.Extensions.AzureBlobPayloads package. } diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index 4e1275c68..c9e287d22 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -40,7 +40,7 @@ public BlobPayloadStore(LargePayloadStorageOptions options) public async Task UploadAsync(ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) { // Ensure container exists - await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, cancellationToken: cancellationToken).ConfigureAwait(false); + await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, default, default, cancellationToken); // One blob per payload using GUID-based name for uniqueness string timestamp = DateTimeOffset.UtcNow.ToString("yyyy/MM/dd/HH/mm/ss", CultureInfo.InvariantCulture); @@ -50,13 +50,13 @@ public async Task UploadAsync(ReadOnlyMemory payloadBytes, Cancell byte[] payloadBuffer = payloadBytes.ToArray(); // Compress and upload streaming - using Stream blobStream = await blob.OpenWriteAsync(overwrite: true, cancellationToken: cancellationToken).ConfigureAwait(false); + using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); using MemoryStream payloadStream = new(payloadBuffer, writable: false); - await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken).ConfigureAwait(false); - await compressedBlobStream.FlushAsync(cancellationToken).ConfigureAwait(false); - await blobStream.FlushAsync(cancellationToken).ConfigureAwait(false); + await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken); + await compressedBlobStream.FlushAsync(cancellationToken); + await blobStream.FlushAsync(cancellationToken); return EncodeToken(this.containerClient.Name, blobName); } @@ -71,7 +71,7 @@ public async Task DownloadAsync(string token, CancellationToken cancella } BlobClient blob = this.containerClient.GetBlobClient(name); - using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken); using GZipStream decompressedBlobStream = new GZipStream(result.Content, CompressionMode.Decompress); using StreamReader reader = new(decompressedBlobStream, Encoding.UTF8); return await reader.ReadToEndAsync(); diff --git a/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs b/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs index 05cb4bc4b..3f349b710 100644 --- a/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs +++ b/src/Worker/Core/DependencyInjection/DurableTaskWorkerBuilderExtensions.cs @@ -137,6 +137,4 @@ public static IDurableTaskWorkerBuilder UseOrchestrationFilter(this IDurableTask builder.Services.AddSingleton(filter); return builder; } - - // Large payload enablement moved to Microsoft.DurableTask.Extensions.AzureBlobPayloads package. } From e6536804c432bb411496402f51f0ab1b4f637272 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:11:12 -0700 Subject: [PATCH 16/38] fix --- .../AzureBlobPayloads/Converters/BlobPayloadStore.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index c9e287d22..f24d43782 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -4,7 +4,6 @@ using System.Globalization; using System.IO.Compression; using System.Text; -using Azure; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -71,7 +70,7 @@ public async Task DownloadAsync(string token, CancellationToken cancella } BlobClient blob = this.containerClient.GetBlobClient(name); - using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken); + using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: cancellationToken); using GZipStream decompressedBlobStream = new GZipStream(result.Content, CompressionMode.Decompress); using StreamReader reader = new(decompressedBlobStream, Encoding.UTF8); return await reader.ReadToEndAsync(); @@ -96,5 +95,3 @@ public async Task DownloadAsync(string token, CancellationToken cancella return (rest.Substring(0, sep), rest.Substring(sep + 1)); } } - - From 499c2815d65fbd5bd64bf773ee89dda26eaefdca Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:35:44 -0700 Subject: [PATCH 17/38] fb --- src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj b/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj index 54883acf4..338022860 100644 --- a/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj +++ b/src/Extensions/AzureBlobPayloads/AzureBlobPayloads.csproj @@ -1,7 +1,7 @@ - netstandard2.0 + netstandard2.0;net6.0 Azure Blob Storage externalized payload support for Durable Task. Microsoft.DurableTask.Extensions.AzureBlobPayloads Microsoft.DurableTask From e841e5e40f658a72925b0b61d16b407e3c6a194a Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:41:47 -0700 Subject: [PATCH 18/38] fb --- .../Converters/LargePayloadDataConverter.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index 5c106ba1f..1e80d5938 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -37,13 +37,13 @@ public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPay /// The serialized value or the token if externalized. public override string? Serialize(object? value) { - if (value is null) + string? json = this.innerConverter.Serialize(value); + + if (string.IsNullOrEmpty(json)) { return null; } - string json = this.innerConverter.Serialize(value) ?? "null"; - int byteCount = this.utf8.GetByteCount(json); if (byteCount < this.largePayloadStorageOptions.ExternalizeThresholdBytes) { From f182442a247a7d3f1c7609773c9e52fc1eb8a0cc Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:42:58 -0700 Subject: [PATCH 19/38] fb --- .../Converters/LargePayloadDataConverter.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index 1e80d5938..8130b80e4 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -17,7 +17,11 @@ namespace Microsoft.DurableTask.Converters; /// The external payload store to use. /// The options for the externalizing data converter. /// Thrown when , , or is null. -public sealed class LargePayloadDataConverter(DataConverter innerConverter, IPayloadStore payloadStore, LargePayloadStorageOptions largePayloadStorageOptions) : DataConverter +public sealed class LargePayloadDataConverter( + DataConverter innerConverter, + IPayloadStore payloadStore, + LargePayloadStorageOptions largePayloadStorageOptions +) : DataConverter { const string TokenPrefix = "blob:v1:"; // matches BlobExternalPayloadStore From e484f4296b18b0a98bcb4332e1c447bc34561f92 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 20:57:13 -0700 Subject: [PATCH 20/38] fb --- src/Abstractions/Converters/IPayloadStore.cs | 8 ++++++++ .../AzureBlobPayloads/Converters/BlobPayloadStore.cs | 12 ++++++++++++ .../Converters/LargePayloadDataConverter.cs | 3 +-- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Abstractions/Converters/IPayloadStore.cs b/src/Abstractions/Converters/IPayloadStore.cs index c7796abbc..c2a6ff6cb 100644 --- a/src/Abstractions/Converters/IPayloadStore.cs +++ b/src/Abstractions/Converters/IPayloadStore.cs @@ -23,4 +23,12 @@ public interface IPayloadStore /// Cancellation token. /// Payload string. Task DownloadAsync(string token, CancellationToken cancellationToken); + + /// + /// Returns true if the specified value appears to be a token understood by this store. + /// Implementations should not throw for unknown tokens. + /// + /// The value to check. + /// true if the value is a token issued by this store; otherwise, false. + bool IsKnownPayloadToken(string value); } diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index f24d43782..92ce00b72 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -15,6 +15,7 @@ namespace Microsoft.DurableTask.Converters; /// public sealed class BlobPayloadStore : IPayloadStore { + const string TokenPrefix = "blob:v1:"; readonly BlobContainerClient containerClient; readonly LargePayloadStorageOptions options; @@ -76,6 +77,17 @@ public async Task DownloadAsync(string token, CancellationToken cancella return await reader.ReadToEndAsync(); } + /// + public bool IsKnownPayloadToken(string value) + { + if (string.IsNullOrEmpty(value)) + { + return false; + } + + return value.StartsWith(TokenPrefix, StringComparison.Ordinal); + } + static string EncodeToken(string container, string name) => $"blob:v1:{container}:{name}"; static (string Container, string Name) DecodeToken(string token) diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index 8130b80e4..e0f7315d4 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -23,7 +23,6 @@ public sealed class LargePayloadDataConverter( LargePayloadStorageOptions largePayloadStorageOptions ) : DataConverter { - const string TokenPrefix = "blob:v1:"; // matches BlobExternalPayloadStore readonly DataConverter innerConverter = innerConverter ?? throw new ArgumentNullException(nameof(innerConverter)); readonly IPayloadStore payLoadStore = payloadStore ?? throw new ArgumentNullException(nameof(payloadStore)); @@ -74,7 +73,7 @@ LargePayloadStorageOptions largePayloadStorageOptions } string toDeserialize = data; - if (data.StartsWith(TokenPrefix, StringComparison.Ordinal)) + if (this.payLoadStore.IsKnownPayloadToken(data)) { toDeserialize = this.payLoadStore.DownloadAsync(data, CancellationToken.None).GetAwaiter().GetResult(); } From fb6d3fbf949648de5d1f523af4364e67ef147a07 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Fri, 5 Sep 2025 21:15:51 -0700 Subject: [PATCH 21/38] test --- test/Grpc.IntegrationTests/LargePayloadTests.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/Grpc.IntegrationTests/LargePayloadTests.cs b/test/Grpc.IntegrationTests/LargePayloadTests.cs index dc7150b3a..a8a092095 100644 --- a/test/Grpc.IntegrationTests/LargePayloadTests.cs +++ b/test/Grpc.IntegrationTests/LargePayloadTests.cs @@ -360,6 +360,7 @@ public async Task LargeOutputAndCustomStatus() class InMemoryPayloadStore : IPayloadStore { + const string TokenPrefix = "blob:v1:"; readonly Dictionary tokenToPayload; public InMemoryPayloadStore() @@ -391,5 +392,11 @@ public Task DownloadAsync(string token, CancellationToken cancellationTo Interlocked.Increment(ref this.downloadCount); return Task.FromResult(this.tokenToPayload[token]); } + + public bool IsKnownPayloadToken(string value) + { + return value.StartsWith(TokenPrefix, StringComparison.Ordinal); + } + } } From f734b1bbeb60d5cffcd6b40b602665dbf34d1d3a Mon Sep 17 00:00:00 2001 From: wangbill <12449837+YunchuWang@users.noreply.github.com> Date: Mon, 8 Sep 2025 10:07:56 -0700 Subject: [PATCH 22/38] enable compression --- .../Converters/LargePayloadStorageOptions.cs | 6 +++ .../Converters/BlobPayloadStore.cs | 39 ++++++++++++++----- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/src/Abstractions/Converters/LargePayloadStorageOptions.cs b/src/Abstractions/Converters/LargePayloadStorageOptions.cs index d43e189d9..af005854b 100644 --- a/src/Abstractions/Converters/LargePayloadStorageOptions.cs +++ b/src/Abstractions/Converters/LargePayloadStorageOptions.cs @@ -41,4 +41,10 @@ public LargePayloadStorageOptions(string connectionString) /// Gets or sets the blob container name to use for payloads. Defaults to "durabletask-payloads". /// public string ContainerName { get; set; } = "durabletask-payloads"; + + /// + /// Gets or sets a value indicating whether payloads should be gzip-compressed when stored. + /// Defaults to true for reduced storage and bandwidth. + /// + public bool CompressPayloads { get; set; } = true; } diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index 92ce00b72..3b83bbd30 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -49,14 +49,24 @@ public async Task UploadAsync(ReadOnlyMemory payloadBytes, Cancell byte[] payloadBuffer = payloadBytes.ToArray(); - // Compress and upload streaming - using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); - using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); - using MemoryStream payloadStream = new(payloadBuffer, writable: false); + // Upload streaming, optionally compressing and marking ContentEncoding + if (this.options.CompressPayloads) + { + using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); + using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); + using MemoryStream payloadStream = new(payloadBuffer, writable: false); - await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken); - await compressedBlobStream.FlushAsync(cancellationToken); - await blobStream.FlushAsync(cancellationToken); + await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken); + await compressedBlobStream.FlushAsync(cancellationToken); + await blobStream.FlushAsync(cancellationToken); + } + else + { + using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); + using MemoryStream payloadStream = new(payloadBuffer, writable: false); + await payloadStream.CopyToAsync(blobStream, bufferSize: 81920, cancellationToken); + await blobStream.FlushAsync(cancellationToken); + } return EncodeToken(this.containerClient.Name, blobName); } @@ -72,9 +82,18 @@ public async Task DownloadAsync(string token, CancellationToken cancella BlobClient blob = this.containerClient.GetBlobClient(name); using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: cancellationToken); - using GZipStream decompressedBlobStream = new GZipStream(result.Content, CompressionMode.Decompress); - using StreamReader reader = new(decompressedBlobStream, Encoding.UTF8); - return await reader.ReadToEndAsync(); + Stream contentStream = result.Content; + if (this.options.CompressPayloads) + { + using GZipStream decompressedBlobStream = new(contentStream, CompressionMode.Decompress); + using StreamReader reader = new(decompressedBlobStream, Encoding.UTF8); + return await reader.ReadToEndAsync(); + } + else + { + using StreamReader reader = new(contentStream, Encoding.UTF8); + return await reader.ReadToEndAsync(); + } } /// From 50a210e17590eb3129cbe2e50e113df0cf403a57 Mon Sep 17 00:00:00 2001 From: wangbill <12449837+YunchuWang@users.noreply.github.com> Date: Mon, 8 Sep 2025 13:34:28 -0700 Subject: [PATCH 23/38] update sample --- samples/LargePayloadConsoleApp/Program.cs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index 9baa35f99..0cef4a601 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -139,23 +139,32 @@ await ctx.Entities.CallEntityAsync( // Run entity samples Console.WriteLine(); Console.WriteLine("Running LargeEntityOperationInput..."); +string largeEntityInput = new string('E', 700 * 1024); // 700KB string entityInputInstance = await client.ScheduleNewOrchestrationInstanceAsync("LargeEntityOperationInput"); OrchestrationMetadata entityInputResult = await client.WaitForInstanceCompletionAsync(entityInputInstance, getInputsAndOutputs: true, cts.Token); -Console.WriteLine($"Status: {entityInputResult.RuntimeStatus}, Output length: {entityInputResult.ReadOutputAs()}"); +int entityInputLength = entityInputResult.ReadOutputAs(); +Console.WriteLine($"Status: {entityInputResult.RuntimeStatus}, Output length: {entityInputLength}"); +Console.WriteLine($"Deserialized input length equals original: {entityInputLength == largeEntityInput.Length}"); Console.WriteLine(); Console.WriteLine("Running LargeEntityOperationOutput..."); +int largeEntityOutputLength = 850 * 1024; // 850KB string entityOutputInstance = await client.ScheduleNewOrchestrationInstanceAsync("LargeEntityOperationOutput"); OrchestrationMetadata entityOutputResult = await client.WaitForInstanceCompletionAsync(entityOutputInstance, getInputsAndOutputs: true, cts.Token); -Console.WriteLine($"Status: {entityOutputResult.RuntimeStatus}, Output length: {entityOutputResult.ReadOutputAs()}"); +int entityOutputLength = entityOutputResult.ReadOutputAs(); +Console.WriteLine($"Status: {entityOutputResult.RuntimeStatus}, Output length: {entityOutputLength}"); +Console.WriteLine($"Deserialized output length equals original: {entityOutputLength == largeEntityOutputLength}"); Console.WriteLine(); Console.WriteLine("Running LargeEntityState and querying state..."); +string largeEntityState = new string('S', 900 * 1024); // 900KB string entityStateInstance = await client.ScheduleNewOrchestrationInstanceAsync("LargeEntityState"); OrchestrationMetadata entityStateOrch = await client.WaitForInstanceCompletionAsync(entityStateInstance, getInputsAndOutputs: true, cts.Token); Console.WriteLine($"Status: {entityStateOrch.RuntimeStatus}"); EntityMetadata? state = await client.Entities.GetEntityAsync(new EntityInstanceId(nameof(StateEntity), "1"), includeState: true); -Console.WriteLine($"State length: {state?.State?.Length ?? 0}"); +int stateLength = state?.State?.Length ?? 0; +Console.WriteLine($"State length: {stateLength}"); +Console.WriteLine($"Deserialized state equals original: {state?.State == largeEntityState}"); From 833406fcd5cdb6583170c894fd08bb85685226f6 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 9 Sep 2025 07:50:20 -0700 Subject: [PATCH 24/38] add gzip encoding header to detect for decompression --- Microsoft.DurableTask.sln | 14 +++++++++++ .../Converters/BlobPayloadStore.cs | 23 +++++++++++-------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/Microsoft.DurableTask.sln b/Microsoft.DurableTask.sln index 26c2e80de..383f2e158 100644 --- a/Microsoft.DurableTask.sln +++ b/Microsoft.DurableTask.sln @@ -93,6 +93,10 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScheduleWebApp", "samples\S EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ScheduledTasks.Tests", "test\ScheduledTasks.Tests\ScheduledTasks.Tests.csproj", "{D2779F32-A548-44F8-B60A-6AC018966C79}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LargePayloadConsoleApp", "samples\LargePayloadConsoleApp\LargePayloadConsoleApp.csproj", "{6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureBlobPayloads", "src\Extensions\AzureBlobPayloads\AzureBlobPayloads.csproj", "{FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -247,6 +251,14 @@ Global {D2779F32-A548-44F8-B60A-6AC018966C79}.Debug|Any CPU.Build.0 = Debug|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.ActiveCfg = Release|Any CPU {D2779F32-A548-44F8-B60A-6AC018966C79}.Release|Any CPU.Build.0 = Release|Any CPU + {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC}.Release|Any CPU.Build.0 = Release|Any CPU + {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -293,6 +305,8 @@ Global {A89B766C-987F-4C9F-8937-D0AB9FE640C8} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {100348B5-4D97-4A3F-B777-AB14F276F8FE} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} {D2779F32-A548-44F8-B60A-6AC018966C79} = {E5637F81-2FB9-4CD7-900D-455363B142A7} + {6EB9D002-62C8-D6C1-62A8-14C54CA6DBBC} = {EFF7632B-821E-4CFC-B4A0-ED4B24296B17} + {FE1DA748-D6DB-E168-BC42-6DBBCEAF229C} = {8AFC9781-F6F1-4696-BB4A-9ED7CA9D612B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {AB41CB55-35EA-4986-A522-387AB3402E71} diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index 3b83bbd30..1c6adf31a 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -52,7 +52,11 @@ public async Task UploadAsync(ReadOnlyMemory payloadBytes, Cancell // Upload streaming, optionally compressing and marking ContentEncoding if (this.options.CompressPayloads) { - using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); + BlobOpenWriteOptions writeOptions = new() + { + HttpHeaders = new BlobHttpHeaders { ContentEncoding = "gzip" }, + }; + using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); using MemoryStream payloadStream = new(payloadBuffer, writable: false); @@ -83,17 +87,18 @@ public async Task DownloadAsync(string token, CancellationToken cancella BlobClient blob = this.containerClient.GetBlobClient(name); using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: cancellationToken); Stream contentStream = result.Content; - if (this.options.CompressPayloads) - { - using GZipStream decompressedBlobStream = new(contentStream, CompressionMode.Decompress); - using StreamReader reader = new(decompressedBlobStream, Encoding.UTF8); - return await reader.ReadToEndAsync(); - } - else + bool isGzip = string.Equals( + result.Details.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase); + + if (isGzip) { - using StreamReader reader = new(contentStream, Encoding.UTF8); + using GZipStream decompressed = new(contentStream, CompressionMode.Decompress); + using StreamReader reader = new(decompressed, Encoding.UTF8); return await reader.ReadToEndAsync(); } + + using StreamReader uncompressedReader = new(contentStream, Encoding.UTF8); + return await uncompressedReader.ReadToEndAsync(); } /// From 5ebe0b099023ae7ae850f03b5f52ff200345c5d2 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Tue, 9 Sep 2025 08:48:07 -0700 Subject: [PATCH 25/38] initial add async versions ser/deser --- src/Abstractions/DataConverter.cs | 42 +++++++++++++++++++ .../Converters/LargePayloadDataConverter.cs | 36 +++++++++++----- 2 files changed, 68 insertions(+), 10 deletions(-) diff --git a/src/Abstractions/DataConverter.cs b/src/Abstractions/DataConverter.cs index 6c623c814..78369f35b 100644 --- a/src/Abstractions/DataConverter.cs +++ b/src/Abstractions/DataConverter.cs @@ -49,4 +49,46 @@ public abstract class DataConverter /// [return: NotNullIfNotNull("data")] public virtual T? Deserialize(string? data) => (T?)(this.Deserialize(data, typeof(T)) ?? default); + + /// + /// Asynchronously serializes into a text string. + /// Default implementation delegates to . + /// + /// The value to be serialized. + /// Cancellation token. + /// A task whose result is the serialized string or null. + public virtual ValueTask SerializeAsync(object? value, CancellationToken cancellationToken = default) + { + return new ValueTask(this.Serialize(value)); + } + + /// + /// Asynchronously deserializes into an object of type . + /// Default implementation delegates to . + /// + /// The text data to be deserialized. + /// The type to deserialize to. + /// Cancellation token. + /// A task whose result is the deserialized value or null. + public virtual ValueTask DeserializeAsync( + string? data, + Type targetType, + CancellationToken cancellationToken = default) + { + return new ValueTask(this.Deserialize(data, targetType)); + } + + /// + /// Asynchronously deserializes into an object of type . + /// + /// The type to deserialize to. + /// The text data to be deserialized. + /// Cancellation token. + /// A task whose result is the deserialized value or null. + public virtual ValueTask DeserializeAsync( + string? data, + CancellationToken cancellationToken = default) + { + return new ValueTask(this.Deserialize(data)); + } } diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index e0f7315d4..de2a6ab16 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +using System.Diagnostics.CodeAnalysis; using System.Text; namespace Microsoft.DurableTask.Converters; @@ -20,25 +21,39 @@ namespace Microsoft.DurableTask.Converters; public sealed class LargePayloadDataConverter( DataConverter innerConverter, IPayloadStore payloadStore, - LargePayloadStorageOptions largePayloadStorageOptions -) : DataConverter + LargePayloadStorageOptions largePayloadStorageOptions) : DataConverter { - readonly DataConverter innerConverter = innerConverter ?? throw new ArgumentNullException(nameof(innerConverter)); readonly IPayloadStore payLoadStore = payloadStore ?? throw new ArgumentNullException(nameof(payloadStore)); readonly LargePayloadStorageOptions largePayloadStorageOptions = largePayloadStorageOptions ?? throw new ArgumentNullException(nameof(largePayloadStorageOptions)); + // Use UTF-8 without a BOM (encoderShouldEmitUTF8Identifier=false). JSON in UTF-8 should not include a // byte order mark per RFC 8259, and omitting it avoids hidden extra bytes that could skew the // externalization threshold calculation and prevents interop issues with strict JSON parsers. // A few legacy tools rely on a BOM for encoding detection, but modern JSON tooling assumes BOM-less UTF-8. readonly Encoding utf8 = new UTF8Encoding(false); + /// + [return: NotNullIfNotNull("value")] + public override string? Serialize(object? value) + { + throw new NotImplementedException(); + } + + /// + [return: NotNullIfNotNull("data")] + public override object? Deserialize(string? data, Type targetType) + { + throw new NotImplementedException(); + } + /// /// Serializes the value to a JSON string and uploads it to the external payload store if it exceeds the configured threshold. /// /// The value to serialize. + /// Cancellation token. /// The serialized value or the token if externalized. - public override string? Serialize(object? value) + public override async ValueTask SerializeAsync(object? value, CancellationToken cancellationToken = default) { string? json = this.innerConverter.Serialize(value); @@ -55,8 +70,7 @@ LargePayloadStorageOptions largePayloadStorageOptions // Upload synchronously in this context by blocking on async. SDK call sites already run on threadpool. byte[] bytes = this.utf8.GetBytes(json); - string token = this.payLoadStore.UploadAsync(bytes, CancellationToken.None).GetAwaiter().GetResult(); - return token; + return await this.payLoadStore.UploadAsync(bytes, cancellationToken); } /// @@ -64,8 +78,12 @@ LargePayloadStorageOptions largePayloadStorageOptions /// /// The JSON string or token. /// The type to deserialize to. + /// Cancellation token. /// The deserialized value. - public override object? Deserialize(string? data, Type targetType) + public override async ValueTask DeserializeAsync( + string? data, + Type targetType, + CancellationToken cancellationToken = default) { if (data is null) { @@ -75,7 +93,7 @@ LargePayloadStorageOptions largePayloadStorageOptions string toDeserialize = data; if (this.payLoadStore.IsKnownPayloadToken(data)) { - toDeserialize = this.payLoadStore.DownloadAsync(data, CancellationToken.None).GetAwaiter().GetResult(); + toDeserialize = await this.payLoadStore.DownloadAsync(data, CancellationToken.None); } return this.innerConverter.Deserialize(StripArrayCharacters(toDeserialize), targetType); @@ -92,5 +110,3 @@ LargePayloadStorageOptions largePayloadStorageOptions return input; } } - - From db91bf6b20e85867bac9bb1c588b58f5b4126ae4 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 10 Sep 2025 21:12:55 -0700 Subject: [PATCH 26/38] retry on blob upload/download --- .../Converters/BlobPayloadStore.cs | 150 +++++++++++++----- 1 file changed, 113 insertions(+), 37 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index 1c6adf31a..e60cd403f 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -4,6 +4,8 @@ using System.Globalization; using System.IO.Compression; using System.Text; +using Azure; +using Azure.Core; using Azure.Storage.Blobs; using Azure.Storage.Blobs.Models; @@ -16,6 +18,10 @@ namespace Microsoft.DurableTask.Converters; public sealed class BlobPayloadStore : IPayloadStore { const string TokenPrefix = "blob:v1:"; + + // Jitter RNG for retry backoff + static readonly object RandomLock = new object(); + static readonly Random SharedRandom = new Random(); readonly BlobContainerClient containerClient; readonly LargePayloadStorageOptions options; @@ -32,47 +38,64 @@ public BlobPayloadStore(LargePayloadStorageOptions options) Check.NotNullOrEmpty(options.ConnectionString, nameof(options.ConnectionString)); Check.NotNullOrEmpty(options.ContainerName, nameof(options.ContainerName)); - BlobServiceClient serviceClient = new(options.ConnectionString); + BlobClientOptions clientOptions = new() + { + Retry = + { + Mode = RetryMode.Exponential, + MaxRetries = 8, + Delay = TimeSpan.FromMilliseconds(250), + MaxDelay = TimeSpan.FromSeconds(10), + NetworkTimeout = TimeSpan.FromMinutes(2), + }, + }; + BlobServiceClient serviceClient = new(options.ConnectionString, clientOptions); this.containerClient = serviceClient.GetBlobContainerClient(options.ContainerName); } /// public async Task UploadAsync(ReadOnlyMemory payloadBytes, CancellationToken cancellationToken) { - // Ensure container exists - await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, default, default, cancellationToken); - - // One blob per payload using GUID-based name for uniqueness + // One blob per payload using GUID-based name for uniqueness (stable across retries) string timestamp = DateTimeOffset.UtcNow.ToString("yyyy/MM/dd/HH/mm/ss", CultureInfo.InvariantCulture); string blobName = $"{timestamp}/{Guid.NewGuid():N}"; BlobClient blob = this.containerClient.GetBlobClient(blobName); byte[] payloadBuffer = payloadBytes.ToArray(); - // Upload streaming, optionally compressing and marking ContentEncoding - if (this.options.CompressPayloads) + string token = await WithTransientRetryAsync( + async ct => { - BlobOpenWriteOptions writeOptions = new() + // Ensure container exists (idempotent) + await this.containerClient.CreateIfNotExistsAsync(PublicAccessType.None, default, default, ct); + + if (this.options.CompressPayloads) { - HttpHeaders = new BlobHttpHeaders { ContentEncoding = "gzip" }, - }; - using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, cancellationToken); - using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); - using MemoryStream payloadStream = new(payloadBuffer, writable: false); - - await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, cancellationToken); - await compressedBlobStream.FlushAsync(cancellationToken); - await blobStream.FlushAsync(cancellationToken); - } - else - { - using Stream blobStream = await blob.OpenWriteAsync(true, default, cancellationToken); - using MemoryStream payloadStream = new(payloadBuffer, writable: false); - await payloadStream.CopyToAsync(blobStream, bufferSize: 81920, cancellationToken); - await blobStream.FlushAsync(cancellationToken); - } + BlobOpenWriteOptions writeOptions = new() + { + HttpHeaders = new BlobHttpHeaders { ContentEncoding = "gzip" }, + }; + using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, ct); + using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); + using MemoryStream payloadStream = new(payloadBuffer, writable: false); + + await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, ct); + await compressedBlobStream.FlushAsync(ct); + await blobStream.FlushAsync(ct); + } + else + { + using Stream blobStream = await blob.OpenWriteAsync(true, default, ct); + using MemoryStream payloadStream = new(payloadBuffer, writable: false); + await payloadStream.CopyToAsync(blobStream, bufferSize: 81920, ct); + await blobStream.FlushAsync(ct); + } + + return EncodeToken(this.containerClient.Name, blobName); + }, + cancellationToken); - return EncodeToken(this.containerClient.Name, blobName); + return token; } /// @@ -85,20 +108,26 @@ public async Task DownloadAsync(string token, CancellationToken cancella } BlobClient blob = this.containerClient.GetBlobClient(name); - using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: cancellationToken); - Stream contentStream = result.Content; - bool isGzip = string.Equals( - result.Details.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase); - if (isGzip) + return await WithTransientRetryAsync( + async ct => { - using GZipStream decompressed = new(contentStream, CompressionMode.Decompress); - using StreamReader reader = new(decompressed, Encoding.UTF8); - return await reader.ReadToEndAsync(); - } + using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: ct); + Stream contentStream = result.Content; + bool isGzip = string.Equals( + result.Details.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase); - using StreamReader uncompressedReader = new(contentStream, Encoding.UTF8); - return await uncompressedReader.ReadToEndAsync(); + if (isGzip) + { + using GZipStream decompressed = new(contentStream, CompressionMode.Decompress); + using StreamReader reader = new(decompressed, Encoding.UTF8); + return await reader.ReadToEndAsync(); + } + + using StreamReader uncompressedReader = new(contentStream, Encoding.UTF8); + return await uncompressedReader.ReadToEndAsync(); + }, + cancellationToken); } /// @@ -130,4 +159,51 @@ public bool IsKnownPayloadToken(string value) return (rest.Substring(0, sep), rest.Substring(sep + 1)); } + + static async Task WithTransientRetryAsync(Func> operation, CancellationToken cancellationToken) + { + const int maxAttempts = 8; + TimeSpan baseDelay = TimeSpan.FromMilliseconds(250); + int attempt = 0; + + while (true) + { + cancellationToken.ThrowIfCancellationRequested(); + try + { + return await operation(cancellationToken); + } + catch (RequestFailedException ex) when (IsTransient(ex) && attempt < maxAttempts - 1) + { + attempt++; + TimeSpan delay = ComputeBackoff(baseDelay, attempt); + await Task.Delay(delay, cancellationToken); + } + catch (IOException) when (attempt < maxAttempts - 1) + { + attempt++; + TimeSpan delay = ComputeBackoff(baseDelay, attempt); + await Task.Delay(delay, cancellationToken); + } + } + } + + static bool IsTransient(RequestFailedException ex) + { + return ex.Status == 503 || ex.Status == 502 || ex.Status == 500 || ex.Status == 429 || + string.Equals(ex.ErrorCode, "ServerBusy", StringComparison.OrdinalIgnoreCase) || + string.Equals(ex.ErrorCode, "OperationTimedOut", StringComparison.OrdinalIgnoreCase); + } + + static TimeSpan ComputeBackoff(TimeSpan baseDelay, int attempt) + { + double factor = Math.Pow(2, Math.Min(attempt, 6)); + int jitterMs; + lock (RandomLock) + { + jitterMs = SharedRandom.Next(0, 100); + } + return TimeSpan.FromMilliseconds(Math.Min((baseDelay.TotalMilliseconds * factor) + jitterMs, 10_000)); + } + } From c1a87b936803eb886e9baf1caadbeb5596c6567a Mon Sep 17 00:00:00 2001 From: wangbill <12449837+YunchuWang@users.noreply.github.com> Date: Wed, 10 Sep 2025 11:07:39 -0700 Subject: [PATCH 27/38] update calllers --- src/Client/Core/DurableTaskClientOptions.cs | 25 ++++++ src/Client/Core/OrchestrationMetadata.cs | 81 +++++++++++++++++++ src/Client/Grpc/GrpcDurableTaskClient.cs | 37 ++++++--- .../ShimDurableTaskClient.cs | 25 ++++-- ...ientBuilderExtensions.AzureBlobPayloads.cs | 5 +- src/Worker/Core/DurableTaskWorkerOptions.cs | 7 ++ .../Shims/TaskOrchestrationContextWrapper.cs | 26 +++++- .../Shims/TaskOrchestrationEntityContext.cs | 12 ++- .../Core/Shims/TaskOrchestrationShim.cs | 24 +++++- 9 files changed, 215 insertions(+), 27 deletions(-) diff --git a/src/Client/Core/DurableTaskClientOptions.cs b/src/Client/Core/DurableTaskClientOptions.cs index 05f19e7bd..575768bd6 100644 --- a/src/Client/Core/DurableTaskClientOptions.cs +++ b/src/Client/Core/DurableTaskClientOptions.cs @@ -12,6 +12,7 @@ public class DurableTaskClientOptions { DataConverter dataConverter = JsonDataConverter.Default; bool enableEntitySupport; + bool enableLargePayloadSupport; /// /// Gets or sets the version of orchestrations that will be created. @@ -69,6 +70,20 @@ public bool EnableEntitySupport } } + /// + /// Gets or sets a value indicating whether this client should support large payloads using async serialization/deserialization. + /// When enabled, the client will use async methods for serialization and deserialization to support externalized payloads. + /// + public bool EnableLargePayloadSupport + { + get => this.enableLargePayloadSupport; + set + { + this.enableLargePayloadSupport = value; + this.LargePayloadSupportExplicitlySet = true; + } + } + /// /// Gets a value indicating whether was explicitly set or not. /// @@ -85,6 +100,11 @@ public bool EnableEntitySupport /// internal bool EntitySupportExplicitlySet { get; private set; } + /// + /// Gets a value indicating whether was explicitly set or not. + /// + internal bool LargePayloadSupportExplicitlySet { get; private set; } + /// /// Applies these option values to another. /// @@ -104,6 +124,11 @@ internal void ApplyTo(DurableTaskClientOptions other) other.EnableEntitySupport = this.EnableEntitySupport; } + if (!other.LargePayloadSupportExplicitlySet) + { + other.EnableLargePayloadSupport = this.EnableLargePayloadSupport; + } + if (!string.IsNullOrWhiteSpace(this.DefaultVersion)) { other.DefaultVersion = this.DefaultVersion; diff --git a/src/Client/Core/OrchestrationMetadata.cs b/src/Client/Core/OrchestrationMetadata.cs index a1cf3d9fa..17f493d55 100644 --- a/src/Client/Core/OrchestrationMetadata.cs +++ b/src/Client/Core/OrchestrationMetadata.cs @@ -196,6 +196,87 @@ public OrchestrationMetadata(string name, string instanceId) return this.DataConverter.Deserialize(this.SerializedCustomStatus); } + /// + /// Asynchronously deserializes the orchestration's input into an object of the specified type. + /// + /// + /// This method can only be used when inputs and outputs are explicitly requested from the + /// or + /// method that produced + /// this object. + /// + /// The type to deserialize the orchestration input into. + /// Cancellation token. + /// Returns the deserialized input value. + /// + /// Thrown if this metadata object was fetched without the option to read inputs and outputs. + /// + public async ValueTask ReadInputAsAsync(CancellationToken cancellationToken = default) + { + if (!this.RequestedInputsAndOutputs) + { + throw new InvalidOperationException( + $"The {nameof(this.ReadInputAsAsync)} method can only be used on {nameof(OrchestrationMetadata)} objects " + + "that are fetched with the option to include input data."); + } + + return await this.DataConverter.DeserializeAsync(this.SerializedInput, cancellationToken); + } + + /// + /// Asynchronously deserializes the orchestration's output into an object of the specified type. + /// + /// + /// This method can only be used when inputs and outputs are explicitly requested from the + /// or + /// method that produced + /// this object. + /// + /// The type to deserialize the orchestration output into. + /// Cancellation token. + /// Returns the deserialized output value. + /// + /// Thrown if this metadata object was fetched without the option to read inputs and outputs. + /// + public async ValueTask ReadOutputAsAsync(CancellationToken cancellationToken = default) + { + if (!this.RequestedInputsAndOutputs) + { + throw new InvalidOperationException( + $"The {nameof(this.ReadOutputAsAsync)} method can only be used on {nameof(OrchestrationMetadata)} objects " + + "that are fetched with the option to include output data."); + } + + return await this.DataConverter.DeserializeAsync(this.SerializedOutput, cancellationToken); + } + + /// + /// Asynchronously deserializes the orchestration's custom status value into an object of the specified type. + /// + /// + /// This method can only be used when inputs and outputs are explicitly requested from the + /// or + /// method that produced + /// this object. + /// + /// The type to deserialize the orchestration' custom status into. + /// Cancellation token. + /// Returns the deserialized custom status value. + /// + /// Thrown if this metadata object was fetched without the option to read inputs and outputs. + /// + public async ValueTask ReadCustomStatusAsAsync(CancellationToken cancellationToken = default) + { + if (!this.RequestedInputsAndOutputs) + { + throw new InvalidOperationException( + $"The {nameof(this.ReadCustomStatusAsAsync)} method can only be used on {nameof(OrchestrationMetadata)}" + + " objects that are fetched with the option to include input and output data."); + } + + return await this.DataConverter.DeserializeAsync(this.SerializedCustomStatus, cancellationToken); + } + /// /// Generates a user-friendly string representation of the current metadata object. /// diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index c38682a3c..65a24ec16 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -64,6 +64,11 @@ public GrpcDurableTaskClient(string name, GrpcDurableTaskClientOptions options, DataConverter DataConverter => this.options.DataConverter; + /// + /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). + /// + bool SupportsAsyncSerialization => this.options.EnableLargePayloadSupport; + /// public override ValueTask DisposeAsync() { @@ -90,12 +95,16 @@ public override async Task ScheduleNewOrchestrationInstanceAsync( version = this.options.DefaultVersion; } + string instanceId = options?.InstanceId ?? Guid.NewGuid().ToString("N"); + var request = new P.CreateInstanceRequest { Name = orchestratorName.Name, Version = version, - InstanceId = options?.InstanceId ?? Guid.NewGuid().ToString("N"), - Input = this.DataConverter.Serialize(input), + InstanceId = instanceId, + Input = this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(input, cancellation) + : this.DataConverter.Serialize(input), RequestTime = DateTimeOffset.UtcNow.ToTimestamp(), }; @@ -109,11 +118,17 @@ public override async Task ScheduleNewOrchestrationInstanceAsync( } DateTimeOffset? startAt = options?.StartAt; - this.logger.SchedulingOrchestration( - request.InstanceId, - orchestratorName, - sizeInBytes: request.Input != null ? Encoding.UTF8.GetByteCount(request.Input) : 0, - startAt.GetValueOrDefault(DateTimeOffset.UtcNow)); + string name = orchestratorName.Name ?? string.Empty; + string? serializedInput = request.Input; + int sizeInBytes = 0; + if (!string.IsNullOrEmpty(serializedInput)) + { + sizeInBytes = Encoding.UTF8.GetByteCount(serializedInput!); + } + + DateTimeOffset startTime = startAt.GetValueOrDefault(DateTimeOffset.UtcNow); + + this.logger.SchedulingOrchestration(instanceId, name, sizeInBytes, startTime); if (startAt.HasValue) { @@ -141,7 +156,9 @@ public override async Task RaiseEventAsync( { InstanceId = instanceId, Name = eventName, - Input = this.DataConverter.Serialize(eventPayload), + Input = this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(eventPayload, cancellation) + : this.DataConverter.Serialize(eventPayload), }; using Activity? traceActivity = TraceHelper.StartActivityForNewEventRaisedFromClient(request, instanceId); @@ -161,7 +178,9 @@ public override async Task TerminateInstanceAsync( this.logger.TerminatingInstance(instanceId); - string? serializedOutput = this.DataConverter.Serialize(output); + string? serializedOutput = this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(output, cancellation) + : this.DataConverter.Serialize(output); await this.sidecarClient.TerminateInstanceAsync( new P.TerminateRequest { diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs index bb77aab21..198d04fae 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs @@ -67,6 +67,11 @@ public override DurableEntityClient Entities DataConverter DataConverter => this.options.DataConverter; + /// + /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). + /// + bool SupportsAsyncSerialization => this.options.EnableLargePayloadSupport; + IOrchestrationServiceClient Client => this.options.Client!; IOrchestrationServicePurgeClient PurgeClient => this.CastClient(); @@ -141,14 +146,16 @@ public override async Task PurgeAllInstancesAsync( } /// - public override Task RaiseEventAsync( + public override async Task RaiseEventAsync( string instanceId, string eventName, object? eventPayload = null, CancellationToken cancellation = default) { Check.NotNullOrEmpty(instanceId); Check.NotNullOrEmpty(eventName); - string? serializedInput = this.DataConverter.Serialize(eventPayload); - return this.SendInstanceMessageAsync( + string? serializedInput = this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(eventPayload, cancellation) + : this.DataConverter.Serialize(eventPayload); + await this.SendInstanceMessageAsync( instanceId, new EventRaisedEvent(-1, serializedInput) { Name = eventName }, cancellation); } @@ -167,7 +174,9 @@ public override async Task ScheduleNewOrchestrationInstanceAsync( ExecutionId = Guid.NewGuid().ToString("N"), }; - string? serializedInput = this.DataConverter.Serialize(input); + string? serializedInput = this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(input, cancellation) + : this.DataConverter.Serialize(input); var tags = new Dictionary(); if (options?.Tags != null) @@ -207,16 +216,18 @@ public override Task ResumeInstanceAsync( => this.SendInstanceMessageAsync(instanceId, new ExecutionResumedEvent(-1, reason), cancellation); /// - public override Task TerminateInstanceAsync( + public override async Task TerminateInstanceAsync( string instanceId, TerminateInstanceOptions? options = null, CancellationToken cancellation = default) { object? output = options?.Output; Check.NotNullOrEmpty(instanceId); cancellation.ThrowIfCancellationRequested(); - string? reason = this.DataConverter.Serialize(output); + string? reason = this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(output, cancellation) + : this.DataConverter.Serialize(output); // TODO: Support recursive termination of sub-orchestrations - return this.Client.ForceTerminateTaskOrchestrationAsync(instanceId, reason); + await this.Client.ForceTerminateTaskOrchestrationAsync(instanceId, reason); } /// diff --git a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs index ef77d50fa..19f057726 100644 --- a/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs +++ b/src/Extensions/AzureBlobPayloads/DependencyInjection/DurableTaskClientBuilderExtensions.AzureBlobPayloads.cs @@ -17,6 +17,9 @@ public static class DurableTaskClientBuilderExtensionsAzureBlobPayloads /// /// Enables externalized payload storage using Azure Blob Storage for the specified client builder. /// + /// The client builder. + /// The configure action. + /// The original builder, for call chaining. public static IDurableTaskClientBuilder UseExternalizedPayloads( this IDurableTaskClientBuilder builder, Action configure) @@ -43,5 +46,3 @@ public static IDurableTaskClientBuilder UseExternalizedPayloads( return builder; } } - - diff --git a/src/Worker/Core/DurableTaskWorkerOptions.cs b/src/Worker/Core/DurableTaskWorkerOptions.cs index c65ccdbd3..2f3552118 100644 --- a/src/Worker/Core/DurableTaskWorkerOptions.cs +++ b/src/Worker/Core/DurableTaskWorkerOptions.cs @@ -87,6 +87,12 @@ public DataConverter DataConverter /// public bool EnableEntitySupport { get; set; } + /// + /// Gets or sets a value indicating whether this worker should support large payloads using async serialization/deserialization. + /// When enabled, the worker will use async methods for serialization and deserialization to support externalized payloads. + /// + public bool EnableLargePayloadSupport { get; set; } + /// /// Gets or sets the maximum timer interval for the /// method. @@ -174,6 +180,7 @@ internal void ApplyTo(DurableTaskWorkerOptions other) other.DataConverter = this.DataConverter; other.MaximumTimerInterval = this.MaximumTimerInterval; other.EnableEntitySupport = this.EnableEntitySupport; + other.EnableLargePayloadSupport = this.EnableLargePayloadSupport; other.Versioning = this.Versioning; other.OrchestrationFilter = this.OrchestrationFilter; } diff --git a/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs b/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs index 5bf3fbe55..852315b9a 100644 --- a/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs +++ b/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs @@ -113,6 +113,11 @@ public override TaskOrchestrationEntityFeature Entities /// internal DataConverter DataConverter => this.invocationContext.Options.DataConverter; + /// + /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). + /// + bool SupportsAsyncSerialization => this.invocationContext.Options.EnableLargePayloadSupport; + /// protected override ILoggerFactory LoggerFactory => this.invocationContext.LoggerFactory; @@ -283,7 +288,9 @@ public override Task WaitForExternalEvent(string eventName, CancellationTo // Return immediately if this external event has already arrived. if (this.externalEventBuffer.TryTake(eventName, out string? bufferedEventPayload)) { - return Task.FromResult(this.DataConverter.Deserialize(bufferedEventPayload)); + return this.SupportsAsyncSerialization + ? this.DataConverter.DeserializeAsync(bufferedEventPayload, cancellationToken).AsTask() + : Task.FromResult(this.DataConverter.Deserialize(bufferedEventPayload)); } // Create a task completion source that will be set when the external event arrives. @@ -414,7 +421,7 @@ internal void ExitCriticalSectionIfNeeded() /// /// The name of the event to complete. /// The serialized event payload. - internal void CompleteExternalEvent(string eventName, string rawEventPayload) + internal async Task CompleteExternalEvent(string eventName, string rawEventPayload) { if (this.externalEventSources.TryGetValue(eventName, out Queue? waiters)) { @@ -429,7 +436,9 @@ internal void CompleteExternalEvent(string eventName, string rawEventPayload) } else { - value = this.DataConverter.Deserialize(rawEventPayload, waiter.EventType); + value = this.SupportsAsyncSerialization + ? await this.DataConverter.DeserializeAsync(rawEventPayload, waiter.EventType, CancellationToken.None) + : this.DataConverter.Deserialize(rawEventPayload, waiter.EventType); } // Events are completed in FIFO order. Remove the key if the last event was delivered. @@ -448,6 +457,17 @@ internal void CompleteExternalEvent(string eventName, string rawEventPayload) } } + /// + /// Gets the serialized custom status. + /// + /// The custom status serialized to a string, or null if there is not custom status. + internal async ValueTask GetSerializedCustomStatusAsync(CancellationToken cancellationToken = default) + { + return this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(this.customStatus, cancellationToken) + : this.DataConverter.Serialize(this.customStatus); + } + /// /// Gets the serialized custom status. /// diff --git a/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs b/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs index 330fd1888..0934b955a 100644 --- a/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs +++ b/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs @@ -98,7 +98,9 @@ public override async Task CallEntityAsync(EntityInstanceId id } else { - return this.wrapper.DataConverter.Deserialize(operationResult.Result!); + return this.wrapper.SupportsAsyncSerialization + ? await this.wrapper.DataConverter.DeserializeAsync(operationResult.Result) + : this.wrapper.DataConverter.Deserialize(operationResult.Result); } } @@ -173,7 +175,7 @@ static TaskFailureDetails ConvertFailureDetails(FailureDetails failureDetails) async Task CallEntityInternalAsync(EntityInstanceId id, string operationName, object? input) { string instanceId = id.ToString(); - Guid requestId = this.SendOperationMessage(instanceId, operationName, input, oneWay: false, scheduledTime: null); + Guid requestId = await this.SendOperationMessage(instanceId, operationName, input, oneWay: false, scheduledTime: null); OperationResult response = await this.wrapper.WaitForExternalEvent(requestId.ToString()); @@ -186,7 +188,7 @@ async Task CallEntityInternalAsync(EntityInstanceId id, string return response; } - Guid SendOperationMessage(string instanceId, string operationName, object? input, bool oneWay, DateTimeOffset? scheduledTime) + async Task SendOperationMessage(string instanceId, string operationName, object? input, bool oneWay, DateTimeOffset? scheduledTime) { if (!this.EntityContext.ValidateOperationTransition(instanceId, oneWay, out string? errorMessage)) { @@ -194,7 +196,9 @@ Guid SendOperationMessage(string instanceId, string operationName, object? input } Guid guid = this.wrapper.NewGuid(); // deterministically replayable unique id for this request - string? serializedInput = this.wrapper.DataConverter.Serialize(input); + string? serializedInput = this.wrapper.SupportsAsyncSerialization + ? await this.wrapper.DataConverter.SerializeAsync(input, CancellationToken.None) + : this.wrapper.DataConverter.Serialize(input); var target = new OrchestrationInstance() { InstanceId = instanceId }; EntityMessageEvent entityMessageEvent = this.EntityContext.EmitRequestMessage( diff --git a/src/Worker/Core/Shims/TaskOrchestrationShim.cs b/src/Worker/Core/Shims/TaskOrchestrationShim.cs index 127038a38..3d4fdc042 100644 --- a/src/Worker/Core/Shims/TaskOrchestrationShim.cs +++ b/src/Worker/Core/Shims/TaskOrchestrationShim.cs @@ -55,6 +55,11 @@ public TaskOrchestrationShim( DataConverter DataConverter => this.invocationContext.Options.DataConverter; + /// + /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). + /// + bool SupportsAsyncSerialization => this.invocationContext.Options.EnableLargePayloadSupport; + /// public override async Task Execute(OrchestrationContext innerContext, string rawInput) { @@ -63,7 +68,9 @@ public TaskOrchestrationShim( innerContext.MessageDataConverter = converterShim; innerContext.ErrorDataConverter = converterShim; - object? input = this.DataConverter.Deserialize(rawInput, this.implementation.InputType); + object? input = this.SupportsAsyncSerialization + ? await this.DataConverter.DeserializeAsync(rawInput, this.implementation.InputType) + : this.DataConverter.Deserialize(rawInput, this.implementation.InputType); this.wrapperContext = new(innerContext, this.invocationContext, input, this.properties); string instanceId = innerContext.OrchestrationInstance.InstanceId; @@ -82,7 +89,9 @@ public TaskOrchestrationShim( } // Return the output (if any) as a serialized string. - return this.DataConverter.Serialize(output); + return this.SupportsAsyncSerialization + ? await this.DataConverter.SerializeAsync(output) + : this.DataConverter.Serialize(output); } catch (TaskFailedException e) { @@ -105,6 +114,17 @@ public TaskOrchestrationShim( } } + /// + public override async Task GetStatusAsync() + { + if (this.wrapperContext == null) + { + return null; + } + + return await this.wrapperContext.GetSerializedCustomStatusAsync(); + } + /// public override string? GetStatus() { From 4d4aeba854ab5b8fb09cf684072b8a2b81e0ea2f Mon Sep 17 00:00:00 2001 From: wangbill <12449837+YunchuWang@users.noreply.github.com> Date: Thu, 11 Sep 2025 13:18:46 -0700 Subject: [PATCH 28/38] continue updating async --- .../Entities/TaskEntityOperation.cs | 7 ++ src/Client/Core/SerializedData.cs | 18 +++ src/Client/Grpc/GrpcDurableEntityClient.cs | 105 ++++++++++++++++-- src/Client/Grpc/GrpcDurableTaskClient.cs | 2 +- .../ShimDurableEntityClient.cs | 25 +++-- .../Converters/LargePayloadDataConverter.cs | 4 +- .../Core/Shims/DurableTaskShimFactory.cs | 4 +- src/Worker/Core/Shims/TaskActivityShim.cs | 14 ++- src/Worker/Core/Shims/TaskEntityShim.cs | 104 ++++++++++------- .../ScheduledTasks.Tests.csproj | 4 - 10 files changed, 219 insertions(+), 68 deletions(-) diff --git a/src/Abstractions/Entities/TaskEntityOperation.cs b/src/Abstractions/Entities/TaskEntityOperation.cs index 4f6963130..f0acd8712 100644 --- a/src/Abstractions/Entities/TaskEntityOperation.cs +++ b/src/Abstractions/Entities/TaskEntityOperation.cs @@ -42,6 +42,13 @@ public abstract class TaskEntityOperation /// The deserialized input type. public abstract object? GetInput(Type inputType); + /// + /// Gets the input for this operation. + /// + /// The type to deserialize the input as. + /// The deserialized input type. + public abstract Task GetInputAsync(Type inputType); + /// public override string ToString() { diff --git a/src/Client/Core/SerializedData.cs b/src/Client/Core/SerializedData.cs index 7bd2c127d..b20defff8 100644 --- a/src/Client/Core/SerializedData.cs +++ b/src/Client/Core/SerializedData.cs @@ -32,6 +32,13 @@ public sealed class SerializedData(string data, DataConverter? converter = null) /// The deserialized type. public T ReadAs() => this.Converter.Deserialize(this.Value); + /// + /// Deserializes the data into . + /// + /// The type to deserialize into. + /// The deserialized type. + public async Task ReadAsAsync() => await this.Converter.DeserializeAsync(this.Value); + /// /// Creates a new instance of from the specified data. /// @@ -43,4 +50,15 @@ internal static SerializedData Create(object data, DataConverter? converter = nu converter ??= JsonDataConverter.Default; return new SerializedData(converter.Serialize(data), converter); } + + /// + /// Creates a new instance of from the specified data. + /// + /// The data to serialize. + /// The data converter. + /// Serialized data. + internal static async Task CreateAsync(object data, DataConverter converter) + { + return new SerializedData(await converter.SerializeAsync(data), converter); + } } diff --git a/src/Client/Grpc/GrpcDurableEntityClient.cs b/src/Client/Grpc/GrpcDurableEntityClient.cs index fd8e63926..83032fe61 100644 --- a/src/Client/Grpc/GrpcDurableEntityClient.cs +++ b/src/Client/Grpc/GrpcDurableEntityClient.cs @@ -19,6 +19,7 @@ class GrpcDurableEntityClient : DurableEntityClient readonly TaskHubSidecarServiceClient sidecarClient; readonly DataConverter dataConverter; readonly ILogger logger; + readonly bool enableLargePayloadSupport; /// /// Initializes a new instance of the class. @@ -27,13 +28,15 @@ class GrpcDurableEntityClient : DurableEntityClient /// The data converter. /// The client for the GRPC connection to the sidecar. /// The logger for logging client requests. + /// Whether to use async serialization for large payloads. public GrpcDurableEntityClient( - string name, DataConverter dataConverter, TaskHubSidecarServiceClient sidecarClient, ILogger logger) + string name, DataConverter dataConverter, TaskHubSidecarServiceClient sidecarClient, ILogger logger, bool enableLargePayloadSupport = false) : base(name) { this.dataConverter = dataConverter; this.sidecarClient = sidecarClient; this.logger = logger; + this.enableLargePayloadSupport = enableLargePayloadSupport; } /// @@ -54,11 +57,13 @@ public override async Task SignalEntityAsync( InstanceId = id.ToString(), RequestId = requestId.ToString(), Name = operationName, - Input = this.dataConverter.Serialize(input), - ScheduledTime = scheduledTime?.ToTimestamp(), + Input = this.enableLargePayloadSupport + ? await this.dataConverter.SerializeAsync(input, cancellation) + : this.dataConverter.Serialize(input), + ScheduledTime = scheduledTime?.ToTimestamp(), RequestTime = DateTimeOffset.UtcNow.ToTimestamp(), - }; - + }; + if (Activity.Current is { } activity) { request.ParentTraceContext ??= new P.TraceContext(); @@ -86,7 +91,7 @@ public override async Task SignalEntityAsync( /// public override Task?> GetEntityAsync( EntityInstanceId id, bool includeState = false, CancellationToken cancellation = default) - => this.GetEntityCoreAsync(id, includeState, (e, s) => this.ToEntityMetadata(e, s), cancellation); + => this.GetEntityCoreAsync(id, includeState, this.ToEntityMetadata, cancellation); /// public override AsyncPageable GetAllEntitiesAsync(EntityQuery? filter = null) @@ -94,7 +99,7 @@ public override AsyncPageable GetAllEntitiesAsync(EntityQuery? f /// public override AsyncPageable> GetAllEntitiesAsync(EntityQuery? filter = null) - => this.GetAllEntitiesCoreAsync(filter, (x, s) => this.ToEntityMetadata(x, s)); + => this.GetAllEntitiesCoreAsync(filter, this.ToEntityMetadata); /// public override async Task CleanEntityStorageAsync( @@ -170,6 +175,36 @@ public override async Task CleanEntityStorageAsync( } } + async Task GetEntityCoreAsync( + EntityInstanceId id, + bool includeState, + Func> select, + CancellationToken cancellation) + where TMetadata : class + { + Check.NotNullOrEmpty(id.Name); + Check.NotNull(id.Key); + + P.GetEntityRequest request = new() + { + InstanceId = id.ToString(), + IncludeState = includeState, + }; + + try + { + P.GetEntityResponse response = await this.sidecarClient + .GetEntityAsync(request, cancellationToken: cancellation); + + return response.Exists ? await select(response.Entity, includeState) : null; + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.GetEntityAsync)} operation was canceled.", e, cancellation); + } + } + AsyncPageable GetAllEntitiesCoreAsync( EntityQuery? filter, Func select) where TMetadata : class @@ -216,6 +251,54 @@ AsyncPageable GetAllEntitiesCoreAsync( }); } + AsyncPageable GetAllEntitiesCoreAsync( + EntityQuery? filter, Func> select) + where TMetadata : class + { + bool includeState = filter?.IncludeState ?? true; + bool includeTransient = filter?.IncludeTransient ?? false; + string startsWith = filter?.InstanceIdStartsWith ?? string.Empty; + DateTimeOffset? lastModifiedFrom = filter?.LastModifiedFrom; + DateTimeOffset? lastModifiedTo = filter?.LastModifiedTo; + + return Pageable.Create(async (continuation, pageSize, cancellation) => + { + pageSize ??= filter?.PageSize; + + try + { + P.QueryEntitiesResponse response = await this.sidecarClient.QueryEntitiesAsync( + new P.QueryEntitiesRequest + { + Query = new P.EntityQuery + { + InstanceIdStartsWith = startsWith, + LastModifiedFrom = lastModifiedFrom?.ToTimestamp(), + LastModifiedTo = lastModifiedTo?.ToTimestamp(), + IncludeState = includeState, + IncludeTransient = includeTransient, + PageSize = pageSize, + ContinuationToken = continuation ?? filter?.ContinuationToken, + }, + }, + cancellationToken: cancellation); + + List values = new(); + foreach (var entity in response.Entities) + { + values.Add(await select(entity, includeState)); + } + + return new Page(values, response.ContinuationToken); + } + catch (RpcException e) when (e.StatusCode == StatusCode.Cancelled) + { + throw new OperationCanceledException( + $"The {nameof(this.GetAllEntitiesAsync)} operation was canceled.", e, cancellation); + } + }); + } + EntityMetadata ToEntityMetadata(P.EntityMetadata metadata, bool includeState) { var coreEntityId = DTCore.Entities.EntityId.FromString(metadata.InstanceId); @@ -231,7 +314,7 @@ EntityMetadata ToEntityMetadata(P.EntityMetadata metadata, bool includeState) }; } - EntityMetadata ToEntityMetadata(P.EntityMetadata metadata, bool includeState) + async ValueTask> ToEntityMetadata(P.EntityMetadata metadata, bool includeState) { var coreEntityId = DTCore.Entities.EntityId.FromString(metadata.InstanceId); EntityInstanceId entityId = new(coreEntityId.Name, coreEntityId.Key); @@ -240,7 +323,11 @@ EntityMetadata ToEntityMetadata(P.EntityMetadata metadata, bool includeSta if (includeState && hasState) { - T? data = includeState ? this.dataConverter.Deserialize(metadata.SerializedState) : default; + T? data = includeState + ? (this.enableLargePayloadSupport + ? await this.dataConverter.DeserializeAsync(metadata.SerializedState, CancellationToken.None) + : this.dataConverter.Deserialize(metadata.SerializedState)) + : default; return new EntityMetadata(entityId, data) { LastModifiedTime = lastModified, diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 5eb99d855..758a52f26 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -54,7 +54,7 @@ public GrpcDurableTaskClient(string name, GrpcDurableTaskClientOptions options, if (this.options.EnableEntitySupport) { - this.entityClient = new GrpcDurableEntityClient(this.Name, this.DataConverter, this.sidecarClient, logger); + this.entityClient = new GrpcDurableEntityClient(this.Name, this.DataConverter, this.sidecarClient, logger, this.options.EnableLargePayloadSupport); } } diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs index 4d798479d..a2814aa01 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs @@ -3,7 +3,7 @@ using System.Diagnostics; using DurableTask.Core; -using DurableTask.Core.Entities; +using DurableTask.Core.Entities; using DurableTask.Core.Tracing; using Microsoft.DurableTask.Client.Entities; using Microsoft.DurableTask.Entities; @@ -26,6 +26,11 @@ class ShimDurableEntityClient(string name, ShimDurableTaskClientOptions options) DataConverter Converter => this.options.DataConverter; + /// + /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). + /// + bool SupportsAsyncSerialization => this.options.EnableLargePayloadSupport; + /// public override async Task CleanEntityStorageAsync( CleanEntityStorageRequest? request = null, @@ -82,7 +87,9 @@ public override async Task SignalEntityAsync( Check.NotNull(id.Key); DateTimeOffset? scheduledTime = options?.SignalTime; - string? serializedInput = this.Converter.Serialize(input); + string? serializedInput = this.SupportsAsyncSerialization + ? await this.Converter.SerializeAsync(input, cancellation) + : this.Converter.Serialize(input); EntityMessageEvent eventToSend = ClientEntityHelpers.EmitOperationSignal( new OrchestrationInstance() { InstanceId = id.ToString() }, @@ -92,9 +99,9 @@ public override async Task SignalEntityAsync( EntityMessageEvent.GetCappedScheduledTime( DateTime.UtcNow, this.options.Entities.MaxSignalDelayTimeOrDefault, - scheduledTime?.UtcDateTime), - Activity.Current is { } activity ? new DistributedTraceContext(activity.Id!, activity.TraceStateString) : null, - requestTime: DateTimeOffset.UtcNow, + scheduledTime?.UtcDateTime), + Activity.Current is { } activity ? new DistributedTraceContext(activity.Id!, activity.TraceStateString) : null, + requestTime: DateTimeOffset.UtcNow, createTrace: true); await this.options.Client!.SendTaskOrchestrationMessageAsync(eventToSend.AsTaskMessage()); @@ -132,11 +139,15 @@ AsyncPageable GetAllEntitiesAsync( }); } - EntityMetadata Convert(EntityBackendQueries.EntityMetadata metadata) + async Task> Convert(EntityBackendQueries.EntityMetadata metadata) { + T? state = this.SupportsAsyncSerialization + ? await this.Converter.DeserializeAsync(metadata.SerializedState) + : this.Converter.Deserialize(metadata.SerializedState); + return new( metadata.EntityId.ConvertFromCore(), - this.Converter.Deserialize(metadata.SerializedState)) + state) { LastModifiedTime = metadata.LastModifiedTime, BacklogQueueSize = metadata.BacklogQueueSize, diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index de2a6ab16..edcf1a6fa 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -55,7 +55,7 @@ public sealed class LargePayloadDataConverter( /// The serialized value or the token if externalized. public override async ValueTask SerializeAsync(object? value, CancellationToken cancellationToken = default) { - string? json = this.innerConverter.Serialize(value); + string? json = await this.innerConverter.SerializeAsync(value, cancellationToken); if (string.IsNullOrEmpty(json)) { @@ -96,7 +96,7 @@ public sealed class LargePayloadDataConverter( toDeserialize = await this.payLoadStore.DownloadAsync(data, CancellationToken.None); } - return this.innerConverter.Deserialize(StripArrayCharacters(toDeserialize), targetType); + return await this.innerConverter.DeserializeAsync(StripArrayCharacters(toDeserialize), targetType, cancellationToken); } static string? StripArrayCharacters(string? input) diff --git a/src/Worker/Core/Shims/DurableTaskShimFactory.cs b/src/Worker/Core/Shims/DurableTaskShimFactory.cs index 584b7eeb8..e222c5ec3 100644 --- a/src/Worker/Core/Shims/DurableTaskShimFactory.cs +++ b/src/Worker/Core/Shims/DurableTaskShimFactory.cs @@ -50,7 +50,7 @@ public TaskActivity CreateActivity(TaskName name, ITaskActivity activity) { Check.NotDefault(name); Check.NotNull(activity); - return new TaskActivityShim(this.loggerFactory, this.options.DataConverter, name, activity); + return new TaskActivityShim(this.loggerFactory, this.options.DataConverter, name, activity, this.options.EnableLargePayloadSupport); } /// @@ -151,6 +151,6 @@ public TaskEntity CreateEntity(TaskName name, ITaskEntity entity, EntityId entit // In the future we may consider caching those shims and reusing them, which can reduce // deserialization and allocation overheads. ILogger logger = this.loggerFactory.CreateLogger(entity.GetType()); - return new TaskEntityShim(this.options.DataConverter, entity, entityId, logger); + return new TaskEntityShim(this.options.DataConverter, entity, entityId, logger, this.options.EnableLargePayloadSupport); } } diff --git a/src/Worker/Core/Shims/TaskActivityShim.cs b/src/Worker/Core/Shims/TaskActivityShim.cs index ae1f3bb06..f29c7828e 100644 --- a/src/Worker/Core/Shims/TaskActivityShim.cs +++ b/src/Worker/Core/Shims/TaskActivityShim.cs @@ -15,6 +15,7 @@ class TaskActivityShim : TaskActivity readonly ILogger logger; readonly DataConverter dataConverter; readonly TaskName name; + readonly bool enableLargePayloadSupport; /// /// Initializes a new instance of the class. @@ -23,16 +24,19 @@ class TaskActivityShim : TaskActivity /// The data converter. /// The name of the activity. /// The activity implementation to wrap. + /// Whether to use async serialization for large payloads. public TaskActivityShim( ILoggerFactory loggerFactory, DataConverter dataConverter, TaskName name, - ITaskActivity implementation) + ITaskActivity implementation, + bool enableLargePayloadSupport = false) { this.logger = Logs.CreateWorkerLogger(Check.NotNull(loggerFactory), "Activities"); this.dataConverter = Check.NotNull(dataConverter); this.name = Check.NotDefault(name); this.implementation = Check.NotNull(implementation); + this.enableLargePayloadSupport = enableLargePayloadSupport; } /// @@ -40,7 +44,9 @@ public TaskActivityShim( { Check.NotNull(coreContext); string? strippedRawInput = StripArrayCharacters(rawInput); - object? deserializedInput = this.dataConverter.Deserialize(strippedRawInput, this.implementation.InputType); + object? deserializedInput = this.enableLargePayloadSupport + ? await this.dataConverter.DeserializeAsync(strippedRawInput, this.implementation.InputType) + : this.dataConverter.Deserialize(strippedRawInput, this.implementation.InputType); TaskActivityContextWrapper contextWrapper = new(coreContext, this.name); string instanceId = coreContext.OrchestrationInstance.InstanceId; @@ -51,7 +57,9 @@ public TaskActivityShim( object? output = await this.implementation.RunAsync(contextWrapper, deserializedInput); // Return the output (if any) as a serialized string. - string? serializedOutput = this.dataConverter.Serialize(output); + string? serializedOutput = this.enableLargePayloadSupport + ? await this.dataConverter.SerializeAsync(output) + : this.dataConverter.Serialize(output); this.logger.ActivityCompleted(instanceId, this.name); return serializedOutput; diff --git a/src/Worker/Core/Shims/TaskEntityShim.cs b/src/Worker/Core/Shims/TaskEntityShim.cs index bad6c7f08..fd44235cc 100644 --- a/src/Worker/Core/Shims/TaskEntityShim.cs +++ b/src/Worker/Core/Shims/TaskEntityShim.cs @@ -19,6 +19,7 @@ class TaskEntityShim : DTCore.Entities.TaskEntity readonly DataConverter dataConverter; readonly ITaskEntity taskEntity; readonly EntityInstanceId entityId; + readonly bool enableLargePayloadSupport; readonly StateShim state; readonly ContextShim context; @@ -32,14 +33,16 @@ class TaskEntityShim : DTCore.Entities.TaskEntity /// The task entity. /// The entity ID. /// The logger. + /// Whether to use async serialization for large payloads. public TaskEntityShim( - DataConverter dataConverter, ITaskEntity taskEntity, EntityId entityId, ILogger logger) + DataConverter dataConverter, ITaskEntity taskEntity, EntityId entityId, ILogger logger, bool enableLargePayloadSupport = false) { this.dataConverter = Check.NotNull(dataConverter); this.taskEntity = Check.NotNull(taskEntity); this.entityId = new EntityInstanceId(entityId.Name, entityId.Key); - this.state = new StateShim(dataConverter); - this.context = new ContextShim(this.entityId, dataConverter); + this.enableLargePayloadSupport = enableLargePayloadSupport; + this.state = new StateShim(dataConverter, enableLargePayloadSupport); + this.context = new ContextShim(this.entityId, dataConverter, enableLargePayloadSupport); this.operation = new OperationShim(this); this.logger = logger; } @@ -58,24 +61,26 @@ public override async Task ExecuteOperationBatchAsync(EntityB List results = new(); foreach (OperationRequest current in operations.Operations!) - { + { var startTime = DateTime.UtcNow; - this.operation.SetNameAndInput(current.Operation!, current.Input); - - // The trace context of the current operation becomes the parent trace context of the TaskEntityContext. - // That way, if processing this operation request leads to the TaskEntityContext signaling another entity or starting an orchestration, - // then the parent trace context of these actions will be set to the trace context of whatever operation request triggered them + this.operation.SetNameAndInput(current.Operation!, current.Input); + + // The trace context of the current operation becomes the parent trace context of the TaskEntityContext. + // That way, if processing this operation request leads to the TaskEntityContext signaling another entity or starting an orchestration, + // then the parent trace context of these actions will be set to the trace context of whatever operation request triggered them this.context.ParentTraceContext = current.TraceContext; try - { + { object? result = await this.taskEntity.RunAsync(this.operation); - string? serializedResult = this.dataConverter.Serialize(result); - results.Add(new OperationResult() - { - Result = serializedResult, - StartTimeUtc = startTime, - EndTimeUtc = DateTime.UtcNow, + string? serializedResult = this.enableLargePayloadSupport + ? await this.dataConverter.SerializeAsync(result) + : this.dataConverter.Serialize(result); + results.Add(new OperationResult() + { + Result = serializedResult, + StartTimeUtc = startTime, + EndTimeUtc = DateTime.UtcNow, }); // the user code completed without exception, so we commit the current state and actions. @@ -87,8 +92,8 @@ public override async Task ExecuteOperationBatchAsync(EntityB this.logger.OperationError(applicationException, this.entityId, current.Operation!); results.Add(new OperationResult() { - FailureDetails = new FailureDetails(applicationException), - StartTimeUtc = startTime, + FailureDetails = new FailureDetails(applicationException), + StartTimeUtc = startTime, EndTimeUtc = DateTime.UtcNow, }); @@ -116,14 +121,16 @@ public override async Task ExecuteOperationBatchAsync(EntityB class StateShim : TaskEntityState { readonly DataConverter dataConverter; + readonly bool enableLargePayloadSupport; string? value; object? cachedValue; string? checkpointValue; - public StateShim(DataConverter dataConverter) + public StateShim(DataConverter dataConverter, bool enableLargePayloadSupport = false) { this.dataConverter = dataConverter; + this.enableLargePayloadSupport = enableLargePayloadSupport; } /// @@ -159,20 +166,24 @@ public void Reset() this.cachedValue = null; } - public override object? GetState(Type type) + public override async Task GetState(Type type) { if (this.cachedValue?.GetType() is Type t && t.IsAssignableFrom(type)) { return this.cachedValue; } - this.cachedValue = this.dataConverter.Deserialize(this.value, type); + this.cachedValue = this.enableLargePayloadSupport + ? await this.dataConverter.DeserializeAsync(this.value, type) + : this.dataConverter.Deserialize(this.value, type); return this.cachedValue; } - public override void SetState(object? state) + public override async void SetState(object? state) { - this.value = this.dataConverter.Serialize(state); + this.value = this.enableLargePayloadSupport + ? await this.dataConverter.SerializeAsync(state) + : this.dataConverter.Serialize(state); this.cachedValue = state; } } @@ -181,16 +192,18 @@ class ContextShim : TaskEntityContext { readonly EntityInstanceId entityInstanceId; readonly DataConverter dataConverter; + readonly bool enableLargePayloadSupport; List operationActions; - int checkpointPosition; - + int checkpointPosition; + DistributedTraceContext? parentTraceContext; - public ContextShim(EntityInstanceId entityInstanceId, DataConverter dataConverter) + public ContextShim(EntityInstanceId entityInstanceId, DataConverter dataConverter, bool enableLargePayloadSupport = false) { this.entityInstanceId = entityInstanceId; this.dataConverter = dataConverter; + this.enableLargePayloadSupport = enableLargePayloadSupport; this.operationActions = new List(); } @@ -198,12 +211,12 @@ public ContextShim(EntityInstanceId entityInstanceId, DataConverter dataConverte public int CurrentPosition => this.operationActions.Count; - public override EntityInstanceId Id => this.entityInstanceId; - - public DistributedTraceContext? ParentTraceContext - { - get => this.parentTraceContext; - set => this.parentTraceContext = value; + public override EntityInstanceId Id => this.entityInstanceId; + + public DistributedTraceContext? ParentTraceContext + { + get => this.parentTraceContext; + set => this.parentTraceContext = value; } public void Commit() @@ -222,7 +235,7 @@ public void Reset() this.checkpointPosition = 0; } - public override void SignalEntity(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) + public override async void SignalEntity(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) { Check.NotDefault(id); @@ -230,14 +243,16 @@ public override void SignalEntity(EntityInstanceId id, string operationName, obj { InstanceId = id.ToString(), Name = operationName, - Input = this.dataConverter.Serialize(input), - ScheduledTime = options?.SignalTime?.UtcDateTime, - RequestTime = DateTimeOffset.UtcNow, + Input = this.enableLargePayloadSupport + ? await this.dataConverter.SerializeAsync(input) + : this.dataConverter.Serialize(input), + ScheduledTime = options?.SignalTime?.UtcDateTime, + RequestTime = DateTimeOffset.UtcNow, ParentTraceContext = this.parentTraceContext, }); } - public override string ScheduleNewOrchestration(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + public override async Task ScheduleNewOrchestration(TaskName name, object? input = null, StartOrchestrationOptions? options = null) { Check.NotEntity(true, options?.InstanceId); @@ -247,9 +262,11 @@ public override string ScheduleNewOrchestration(TaskName name, object? input = n Name = name.Name, Version = options?.Version ?? string.Empty, InstanceId = instanceId, - Input = this.dataConverter.Serialize(input), - ScheduledStartTime = options?.StartAt?.UtcDateTime, - RequestTime = DateTimeOffset.UtcNow, + Input = this.enableLargePayloadSupport + ? await this.dataConverter.SerializeAsync(input) + : this.dataConverter.Serialize(input), + ScheduledStartTime = options?.StartAt?.UtcDateTime, + RequestTime = DateTimeOffset.UtcNow, ParentTraceContext = this.parentTraceContext, }); return instanceId; @@ -281,6 +298,13 @@ public OperationShim(TaskEntityShim taskEntityShim) return this.taskEntityShim.dataConverter.Deserialize(this.input, inputType); } + public override async Task GetInputAsync(Type inputType) + { + return this.taskEntityShim.enableLargePayloadSupport + ? await this.taskEntityShim.dataConverter.DeserializeAsync(this.input, inputType) + : this.taskEntityShim.dataConverter.Deserialize(this.input, inputType); + } + public void SetNameAndInput(string name, string? input) { this.name = name; diff --git a/test/ScheduledTasks.Tests/ScheduledTasks.Tests.csproj b/test/ScheduledTasks.Tests/ScheduledTasks.Tests.csproj index 33726a624..04f0d7205 100644 --- a/test/ScheduledTasks.Tests/ScheduledTasks.Tests.csproj +++ b/test/ScheduledTasks.Tests/ScheduledTasks.Tests.csproj @@ -9,10 +9,6 @@ true - - - - From 62dda3e38e2e39539538a7db82fe4636214991a6 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sat, 13 Sep 2025 10:08:12 -0700 Subject: [PATCH 29/38] more update --- .../Entities/TaskEntityContext.cs | 12 +++++++ src/Abstractions/Entities/TaskEntityState.cs | 11 +++++++ .../ShimDurableEntityClient.cs | 6 ++-- src/Worker/Core/Shims/TaskEntityShim.cs | 33 +++++++++++++++++-- .../Entities/Mocks/TestEntityOperation.cs | 5 +++ 5 files changed, 62 insertions(+), 5 deletions(-) diff --git a/src/Abstractions/Entities/TaskEntityContext.cs b/src/Abstractions/Entities/TaskEntityContext.cs index 9de1ff46e..b79ca49ad 100644 --- a/src/Abstractions/Entities/TaskEntityContext.cs +++ b/src/Abstractions/Entities/TaskEntityContext.cs @@ -53,4 +53,16 @@ public virtual string ScheduleNewOrchestration(TaskName name, StartOrchestration /// The instance id for the new orchestration. public abstract string ScheduleNewOrchestration( TaskName name, object? input = null, StartOrchestrationOptions? options = null); + + /// + /// Starts an orchestration. + /// + /// The name of the orchestration to start. + /// The input for the orchestration. + /// The options for starting the orchestration. + /// The instance id for the new orchestration. + public virtual async Task ScheduleNewOrchestrationAsync(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + { + return await Task.FromResult(this.ScheduleNewOrchestration(name, input, options)); + } } diff --git a/src/Abstractions/Entities/TaskEntityState.cs b/src/Abstractions/Entities/TaskEntityState.cs index 6f1a61f8e..b3e1a27e6 100644 --- a/src/Abstractions/Entities/TaskEntityState.cs +++ b/src/Abstractions/Entities/TaskEntityState.cs @@ -35,6 +35,17 @@ public abstract class TaskEntityState return defaultValue; } + /// + /// Asynchronously gets the current state of the entity. This will return null if no state is present, regardless if + /// is a value-type or not. + /// + /// The type to retrieve the state as. + /// The entity state. + public virtual Task GetStateAsync(Type type) + { + return Task.FromResult(this.GetState(type)); + } + /// /// Gets the current state of the entity. This will return null if no state is present, regardless if /// is a value-type or not. diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs index a2814aa01..1cd7b87b8 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs @@ -72,7 +72,7 @@ public override AsyncPageable> GetAllEntitiesAsync(EntityQu /// public override async Task?> GetEntityAsync( EntityInstanceId id, bool includeState = true, CancellationToken cancellation = default) - => this.Convert(await this.Queries.GetEntityAsync( + => await this.Convert(await this.Queries.GetEntityAsync( id.ConvertToCore(), includeState, false, cancellation)); /// @@ -155,14 +155,14 @@ async Task> Convert(EntityBackendQueries.EntityMetadata met }; } - EntityMetadata? Convert(EntityBackendQueries.EntityMetadata? metadata) + async Task?> Convert(EntityBackendQueries.EntityMetadata? metadata) { if (metadata is null) { return null; } - return this.Convert(metadata.Value); + return await this.Convert(metadata.Value); } EntityMetadata Convert(EntityBackendQueries.EntityMetadata metadata) diff --git a/src/Worker/Core/Shims/TaskEntityShim.cs b/src/Worker/Core/Shims/TaskEntityShim.cs index fd44235cc..3db016f82 100644 --- a/src/Worker/Core/Shims/TaskEntityShim.cs +++ b/src/Worker/Core/Shims/TaskEntityShim.cs @@ -166,7 +166,18 @@ public void Reset() this.cachedValue = null; } - public override async Task GetState(Type type) + public override object? GetState(Type type) + { + if (this.cachedValue?.GetType() is Type t && t.IsAssignableFrom(type)) + { + return this.cachedValue; + } + + this.cachedValue = this.dataConverter.Deserialize(this.value, type); + return this.cachedValue; + } + + public override async Task GetStateAsync(Type type) { if (this.cachedValue?.GetType() is Type t && t.IsAssignableFrom(type)) { @@ -252,7 +263,25 @@ public override async void SignalEntity(EntityInstanceId id, string operationNam }); } - public override async Task ScheduleNewOrchestration(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + public override string ScheduleNewOrchestration(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + { + Check.NotEntity(true, options?.InstanceId); + + string instanceId = options?.InstanceId ?? Guid.NewGuid().ToString("N"); + this.operationActions.Add(new StartNewOrchestrationOperationAction() + { + Name = name.Name, + Version = options?.Version ?? string.Empty, + InstanceId = instanceId, + Input = this.dataConverter.Serialize(input), + ScheduledStartTime = options?.StartAt?.UtcDateTime, + RequestTime = DateTimeOffset.UtcNow, + ParentTraceContext = this.parentTraceContext, + }); + return instanceId; + } + + public override async Task ScheduleNewOrchestrationAsync(TaskName name, object? input = null, StartOrchestrationOptions? options = null) { Check.NotEntity(true, options?.InstanceId); diff --git a/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs b/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs index 64372d804..5f5d9c1a7 100644 --- a/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs +++ b/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs @@ -54,4 +54,9 @@ public TestEntityOperation(string name, TaskEntityState state, Optional return this.input.Value; } + + public override Task GetInputAsync(Type inputType) + { + return Task.FromResult(this.GetInput(inputType)); + } } From 4f0e5e342353d529ef124fc3455b2e96e84e443e Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sat, 13 Sep 2025 10:16:28 -0700 Subject: [PATCH 30/38] update all async --- .../ShimDurableEntityClient.cs | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs index 1cd7b87b8..86bd1241e 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs @@ -61,7 +61,9 @@ public override AsyncPageable GetAllEntitiesAsync(EntityQuery? f /// public override AsyncPageable> GetAllEntitiesAsync(EntityQuery? filter = null) - => this.GetAllEntitiesAsync(this.Convert, filter); + => this.SupportsAsyncSerialization + ? this.GetAllEntitiesAsync(this.Convert, filter) + : this.GetAllEntitiesAsync(this.ConvertSync, filter); /// public override async Task GetEntityAsync( @@ -139,6 +141,39 @@ AsyncPageable GetAllEntitiesAsync( }); } + AsyncPageable GetAllEntitiesAsync( + Func> selectAsync, + EntityQuery? filter) + where TMetadata : notnull + { + bool includeState = filter?.IncludeState ?? true; + bool includeTransient = filter?.IncludeTransient ?? false; + string startsWith = filter?.InstanceIdStartsWith ?? string.Empty; + DateTime? lastModifiedFrom = filter?.LastModifiedFrom?.UtcDateTime; + DateTime? lastModifiedTo = filter?.LastModifiedTo?.UtcDateTime; + + return Pageable.Create(async (continuation, size, cancellation) => + { + continuation ??= filter?.ContinuationToken; + size ??= filter?.PageSize; + EntityBackendQueries.EntityQueryResult result = await this.Queries.QueryEntitiesAsync( + new EntityBackendQueries.EntityQuery() + { + InstanceIdStartsWith = startsWith, + LastModifiedFrom = lastModifiedFrom, + LastModifiedTo = lastModifiedTo, + IncludeTransient = includeTransient, + IncludeState = includeState, + ContinuationToken = continuation, + PageSize = size, + }, + cancellation); + + TMetadata[] items = await Task.WhenAll(result.Results.Select(selectAsync)); + return new Page(items, result.ContinuationToken); + }); + } + async Task> Convert(EntityBackendQueries.EntityMetadata metadata) { T? state = this.SupportsAsyncSerialization @@ -155,6 +190,20 @@ async Task> Convert(EntityBackendQueries.EntityMetadata met }; } + EntityMetadata ConvertSync(EntityBackendQueries.EntityMetadata metadata) + { + T? state = this.Converter.Deserialize(metadata.SerializedState); + + return new( + metadata.EntityId.ConvertFromCore(), + state) + { + LastModifiedTime = metadata.LastModifiedTime, + BacklogQueueSize = metadata.BacklogQueueSize, + LockedBy = metadata.LockedBy, + }; + } + async Task?> Convert(EntityBackendQueries.EntityMetadata? metadata) { if (metadata is null) From af7a4c8062ce5a4c2e5792d4ee2cce952a1e7cea Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sat, 13 Sep 2025 21:41:15 -0700 Subject: [PATCH 31/38] enhance --- src/Abstractions/Entities/TaskEntityContext.cs | 9 ++++++--- src/Abstractions/Entities/TaskEntityState.cs | 11 +++++++++++ src/Worker/Core/Shims/TaskEntityShim.cs | 13 +++++++++++-- .../Entities/Mocks/TestEntityState.cs | 6 ++++++ 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/Abstractions/Entities/TaskEntityContext.cs b/src/Abstractions/Entities/TaskEntityContext.cs index b79ca49ad..7d97e557f 100644 --- a/src/Abstractions/Entities/TaskEntityContext.cs +++ b/src/Abstractions/Entities/TaskEntityContext.cs @@ -60,9 +60,12 @@ public abstract string ScheduleNewOrchestration( /// The name of the orchestration to start. /// The input for the orchestration. /// The options for starting the orchestration. - /// The instance id for the new orchestration. - public virtual async Task ScheduleNewOrchestrationAsync(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + /// A task representing the asynchronous operation. + public virtual Task ScheduleNewOrchestrationAsync( + TaskName name, + object? input = null, + StartOrchestrationOptions? options = null) { - return await Task.FromResult(this.ScheduleNewOrchestration(name, input, options)); + return Task.FromResult(this.ScheduleNewOrchestration(name, input, options)); } } diff --git a/src/Abstractions/Entities/TaskEntityState.cs b/src/Abstractions/Entities/TaskEntityState.cs index b3e1a27e6..9ee407579 100644 --- a/src/Abstractions/Entities/TaskEntityState.cs +++ b/src/Abstractions/Entities/TaskEntityState.cs @@ -59,4 +59,15 @@ public abstract class TaskEntityState /// /// The state to set. public abstract void SetState(object? state); + + /// + /// Asynchronously sets the entity state. Setting of null will delete entity state. + /// + /// The state to set. + /// A task representing the asynchronous operation. + public virtual Task SetStateAsync(object? state) + { + this.SetState(state); + return Task.CompletedTask; + } } diff --git a/src/Worker/Core/Shims/TaskEntityShim.cs b/src/Worker/Core/Shims/TaskEntityShim.cs index 3db016f82..3468ea880 100644 --- a/src/Worker/Core/Shims/TaskEntityShim.cs +++ b/src/Worker/Core/Shims/TaskEntityShim.cs @@ -190,7 +190,13 @@ public void Reset() return this.cachedValue; } - public override async void SetState(object? state) + public override void SetState(object? state) + { + this.value = this.dataConverter.Serialize(state); + this.cachedValue = state; + } + + public override async Task SetStateAsync(object? state) { this.value = this.enableLargePayloadSupport ? await this.dataConverter.SerializeAsync(state) @@ -281,7 +287,10 @@ public override string ScheduleNewOrchestration(TaskName name, object? input = n return instanceId; } - public override async Task ScheduleNewOrchestrationAsync(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + public override async Task ScheduleNewOrchestrationAsync( + TaskName name, + object? input = null, + StartOrchestrationOptions? options = null) { Check.NotEntity(true, options?.InstanceId); diff --git a/test/Abstractions.Tests/Entities/Mocks/TestEntityState.cs b/test/Abstractions.Tests/Entities/Mocks/TestEntityState.cs index 0477f15ca..a9c096fae 100644 --- a/test/Abstractions.Tests/Entities/Mocks/TestEntityState.cs +++ b/test/Abstractions.Tests/Entities/Mocks/TestEntityState.cs @@ -28,4 +28,10 @@ public override void SetState(object? state) { this.State = state; } + + public override Task SetStateAsync(object? state) + { + this.SetState(state); + return Task.CompletedTask; + } } From 281fed28067c61f2893cd27d3d2bf310fc4cc189 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sat, 13 Sep 2025 23:20:59 -0700 Subject: [PATCH 32/38] disallow sync calls when largepayload enabled --- src/Abstractions/Entities/TaskEntityContext.cs | 4 ++-- src/Abstractions/Entities/TaskEntityOperation.cs | 5 ++++- src/Shared/Core/Validation/Check.cs | 16 ++++++++++++++++ src/Worker/Core/Shims/TaskEntityShim.cs | 5 ++++- src/Worker/Core/Worker.csproj | 1 + .../Entities/Mocks/TestEntityOperation.cs | 5 ----- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/Abstractions/Entities/TaskEntityContext.cs b/src/Abstractions/Entities/TaskEntityContext.cs index 7d97e557f..a638c0fbd 100644 --- a/src/Abstractions/Entities/TaskEntityContext.cs +++ b/src/Abstractions/Entities/TaskEntityContext.cs @@ -61,11 +61,11 @@ public abstract string ScheduleNewOrchestration( /// The input for the orchestration. /// The options for starting the orchestration. /// A task representing the asynchronous operation. - public virtual Task ScheduleNewOrchestrationAsync( + public virtual ValueTask ScheduleNewOrchestrationAsync( TaskName name, object? input = null, StartOrchestrationOptions? options = null) { - return Task.FromResult(this.ScheduleNewOrchestration(name, input, options)); + return new ValueTask(this.ScheduleNewOrchestration(name, input, options)); } } diff --git a/src/Abstractions/Entities/TaskEntityOperation.cs b/src/Abstractions/Entities/TaskEntityOperation.cs index f0acd8712..5ee438df6 100644 --- a/src/Abstractions/Entities/TaskEntityOperation.cs +++ b/src/Abstractions/Entities/TaskEntityOperation.cs @@ -47,7 +47,10 @@ public abstract class TaskEntityOperation /// /// The type to deserialize the input as. /// The deserialized input type. - public abstract Task GetInputAsync(Type inputType); + public virtual Task GetInputAsync(Type inputType) + { + return Task.FromResult(this.GetInput(inputType)); + } /// public override string ToString() diff --git a/src/Shared/Core/Validation/Check.cs b/src/Shared/Core/Validation/Check.cs index 76c154749..dd8b10c8b 100644 --- a/src/Shared/Core/Validation/Check.cs +++ b/src/Shared/Core/Validation/Check.cs @@ -108,6 +108,22 @@ public static void NotEntity(bool entitySupportEnabled, string? instanceId, [Cal } } + /// + /// Checks if the supplied type is a concrete non-abstract type and implements the provided generic type. + /// Throws if the conditions are not met. + /// + /// Whether large payload support is enabled. + /// The name of the operation. + public static void ThrowIfLargePayloadEnabled(bool largePayloadSupportEnabled, string operationName) + { + if (largePayloadSupportEnabled) + { + throw new NotSupportedException( + $"Operation '{operationName}' is not supported when LargePayload is enabled. " + + "Use the async version of this method instead."); + } + } + /// /// Checks if the supplied type is a concrete non-abstract type and implements the provided generic type. /// Throws if the conditions are not met. diff --git a/src/Worker/Core/Shims/TaskEntityShim.cs b/src/Worker/Core/Shims/TaskEntityShim.cs index 3468ea880..ed36bf8c9 100644 --- a/src/Worker/Core/Shims/TaskEntityShim.cs +++ b/src/Worker/Core/Shims/TaskEntityShim.cs @@ -168,6 +168,7 @@ public void Reset() public override object? GetState(Type type) { + Check.ThrowIfLargePayloadEnabled(this.enableLargePayloadSupport, nameof(this.GetState)); if (this.cachedValue?.GetType() is Type t && t.IsAssignableFrom(type)) { return this.cachedValue; @@ -272,6 +273,7 @@ public override async void SignalEntity(EntityInstanceId id, string operationNam public override string ScheduleNewOrchestration(TaskName name, object? input = null, StartOrchestrationOptions? options = null) { Check.NotEntity(true, options?.InstanceId); + Check.ThrowIfLargePayloadEnabled(this.enableLargePayloadSupport, nameof(this.ScheduleNewOrchestration)); string instanceId = options?.InstanceId ?? Guid.NewGuid().ToString("N"); this.operationActions.Add(new StartNewOrchestrationOperationAction() @@ -287,7 +289,7 @@ public override string ScheduleNewOrchestration(TaskName name, object? input = n return instanceId; } - public override async Task ScheduleNewOrchestrationAsync( + public override async ValueTask ScheduleNewOrchestrationAsync( TaskName name, object? input = null, StartOrchestrationOptions? options = null) @@ -333,6 +335,7 @@ public OperationShim(TaskEntityShim taskEntityShim) public override object? GetInput(Type inputType) { + Check.ThrowIfLargePayloadEnabled(this.taskEntityShim.enableLargePayloadSupport, nameof(this.GetInput)); return this.taskEntityShim.dataConverter.Deserialize(this.input, inputType); } diff --git a/src/Worker/Core/Worker.csproj b/src/Worker/Core/Worker.csproj index 5791edf73..2351efe56 100644 --- a/src/Worker/Core/Worker.csproj +++ b/src/Worker/Core/Worker.csproj @@ -9,6 +9,7 @@ The worker is responsible for processing durable task work items. + diff --git a/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs b/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs index 5f5d9c1a7..64372d804 100644 --- a/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs +++ b/test/Abstractions.Tests/Entities/Mocks/TestEntityOperation.cs @@ -54,9 +54,4 @@ public TestEntityOperation(string name, TaskEntityState state, Optional return this.input.Value; } - - public override Task GetInputAsync(Type inputType) - { - return Task.FromResult(this.GetInput(inputType)); - } } From 52ec85c256aae29799be2edf54920bc842eaf0c0 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sat, 13 Sep 2025 23:56:31 -0700 Subject: [PATCH 33/38] more --- src/Client/Core/OrchestrationMetadata.cs | 28 +++++++++++++++++-- src/Client/Core/SerializedData.cs | 27 ++++++++++++++---- src/Client/Grpc/GrpcDurableEntityClient.cs | 11 ++++++-- src/Client/Grpc/GrpcDurableTaskClient.cs | 1 + .../ShimDurableEntityClient.cs | 7 ++++- .../ShimDurableTaskClient.cs | 1 + 6 files changed, 65 insertions(+), 10 deletions(-) diff --git a/src/Client/Core/OrchestrationMetadata.cs b/src/Client/Core/OrchestrationMetadata.cs index 17f493d55..8955978a8 100644 --- a/src/Client/Core/OrchestrationMetadata.cs +++ b/src/Client/Core/OrchestrationMetadata.cs @@ -80,6 +80,12 @@ public OrchestrationMetadata(string name, string instanceId) /// The serialized custom status or null. public string? SerializedCustomStatus { get; init; } + /// + /// Gets a value indicating whether large payload support is enabled. + /// + /// true if large payload support is enabled; false otherwise. + public bool EnableLargePayloadSupport { get; init; } + /// /// Gets the tags associated with the orchestration instance. /// @@ -141,6 +147,7 @@ public OrchestrationMetadata(string name, string instanceId) "that are fetched with the option to include input data."); } + Check.ThrowIfLargePayloadEnabled(this.EnableLargePayloadSupport, nameof(this.ReadInputAs)); return this.DataConverter.Deserialize(this.SerializedInput); } @@ -167,6 +174,7 @@ public OrchestrationMetadata(string name, string instanceId) "that are fetched with the option to include output data."); } + Check.ThrowIfLargePayloadEnabled(this.EnableLargePayloadSupport, nameof(this.ReadOutputAs)); return this.DataConverter.Deserialize(this.SerializedOutput); } @@ -193,6 +201,7 @@ public OrchestrationMetadata(string name, string instanceId) + " objects that are fetched with the option to include input and output data."); } + Check.ThrowIfLargePayloadEnabled(this.EnableLargePayloadSupport, nameof(this.ReadCustomStatusAs)); return this.DataConverter.Deserialize(this.SerializedCustomStatus); } @@ -220,7 +229,12 @@ public OrchestrationMetadata(string name, string instanceId) "that are fetched with the option to include input data."); } - return await this.DataConverter.DeserializeAsync(this.SerializedInput, cancellationToken); + if (this.EnableLargePayloadSupport) + { + return await this.DataConverter.DeserializeAsync(this.SerializedInput, cancellationToken); + } + + return this.DataConverter.Deserialize(this.SerializedInput); } /// @@ -247,7 +261,12 @@ public OrchestrationMetadata(string name, string instanceId) "that are fetched with the option to include output data."); } - return await this.DataConverter.DeserializeAsync(this.SerializedOutput, cancellationToken); + if (this.EnableLargePayloadSupport) + { + return await this.DataConverter.DeserializeAsync(this.SerializedOutput, cancellationToken); + } + + return this.DataConverter.Deserialize(this.SerializedOutput); } /// @@ -274,6 +293,11 @@ public OrchestrationMetadata(string name, string instanceId) " objects that are fetched with the option to include input and output data."); } + if (this.EnableLargePayloadSupport) + { + return await this.DataConverter.DeserializeAsync(this.SerializedCustomStatus, cancellationToken); + } + return await this.DataConverter.DeserializeAsync(this.SerializedCustomStatus, cancellationToken); } diff --git a/src/Client/Core/SerializedData.cs b/src/Client/Core/SerializedData.cs index b20defff8..0dc792910 100644 --- a/src/Client/Core/SerializedData.cs +++ b/src/Client/Core/SerializedData.cs @@ -20,6 +20,11 @@ public sealed class SerializedData(string data, DataConverter? converter = null) /// public string Value { get; } = Check.NotNull(data); + /// + /// Gets a value indicating whether large payload support is enabled. + /// + public bool EnableLargePayloadSupport { get; init; } + /// /// Gets the data converter. /// @@ -30,7 +35,11 @@ public sealed class SerializedData(string data, DataConverter? converter = null) /// /// The type to deserialize into. /// The deserialized type. - public T ReadAs() => this.Converter.Deserialize(this.Value); + public T ReadAs() + { + Check.ThrowIfLargePayloadEnabled(this.EnableLargePayloadSupport, nameof(this.ReadAs)); + return this.Converter.Deserialize(this.Value); + } /// /// Deserializes the data into . @@ -44,11 +53,15 @@ public sealed class SerializedData(string data, DataConverter? converter = null) /// /// The data to serialize. /// The data converter. + /// Whether to use async serialization for large payloads. /// Serialized data. - internal static SerializedData Create(object data, DataConverter? converter = null) + internal static SerializedData Create(object data, DataConverter? converter = null, bool enableLargePayloadSupport = false) { converter ??= JsonDataConverter.Default; - return new SerializedData(converter.Serialize(data), converter); + return new SerializedData(converter.Serialize(data), converter) + { + EnableLargePayloadSupport = enableLargePayloadSupport, + }; } /// @@ -56,9 +69,13 @@ internal static SerializedData Create(object data, DataConverter? converter = nu /// /// The data to serialize. /// The data converter. + /// Whether to use async serialization for large payloads. /// Serialized data. - internal static async Task CreateAsync(object data, DataConverter converter) + internal static async Task CreateAsync(object data, DataConverter converter, bool enableLargePayloadSupport = false) { - return new SerializedData(await converter.SerializeAsync(data), converter); + return new SerializedData(await converter.SerializeAsync(data), converter) + { + EnableLargePayloadSupport = enableLargePayloadSupport, + }; } } diff --git a/src/Client/Grpc/GrpcDurableEntityClient.cs b/src/Client/Grpc/GrpcDurableEntityClient.cs index 83032fe61..c28390a1f 100644 --- a/src/Client/Grpc/GrpcDurableEntityClient.cs +++ b/src/Client/Grpc/GrpcDurableEntityClient.cs @@ -30,7 +30,11 @@ class GrpcDurableEntityClient : DurableEntityClient /// The logger for logging client requests. /// Whether to use async serialization for large payloads. public GrpcDurableEntityClient( - string name, DataConverter dataConverter, TaskHubSidecarServiceClient sidecarClient, ILogger logger, bool enableLargePayloadSupport = false) + string name, + DataConverter dataConverter, + TaskHubSidecarServiceClient sidecarClient, + ILogger logger, + bool enableLargePayloadSupport = false) : base(name) { this.dataConverter = dataConverter; @@ -305,7 +309,10 @@ EntityMetadata ToEntityMetadata(P.EntityMetadata metadata, bool includeState) EntityInstanceId entityId = new(coreEntityId.Name, coreEntityId.Key); bool hasState = metadata.SerializedState != null; - SerializedData? data = (includeState && hasState) ? new(metadata.SerializedState!, this.dataConverter) : null; + SerializedData? data = (includeState && hasState) + ? new SerializedData(metadata.SerializedState!, this.dataConverter) + { EnableLargePayloadSupport = this.enableLargePayloadSupport } + : null; return new EntityMetadata(entityId, data) { LastModifiedTime = metadata.LastModifiedTime.ToDateTimeOffset(), diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 758a52f26..5706eb658 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -521,6 +521,7 @@ OrchestrationMetadata CreateMetadata(P.OrchestrationState state, bool includeInp SerializedCustomStatus = state.CustomStatus, FailureDetails = state.FailureDetails.ToTaskFailureDetails(), DataConverter = includeInputsAndOutputs ? this.DataConverter : null, + EnableLargePayloadSupport = this.SupportsAsyncSerialization, Tags = new Dictionary(state.Tags), }; diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs index 86bd1241e..e50672804 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs @@ -216,7 +216,12 @@ EntityMetadata ConvertSync(EntityBackendQueries.EntityMetadata metadata) EntityMetadata Convert(EntityBackendQueries.EntityMetadata metadata) { - SerializedData? data = metadata.SerializedState is null ? null : new(metadata.SerializedState, this.Converter); + SerializedData? data = metadata.SerializedState is null + ? null + : new SerializedData(metadata.SerializedState, this.Converter) + { + EnableLargePayloadSupport = this.SupportsAsyncSerialization, + }; return new(new EntityInstanceId(metadata.EntityId.Name, metadata.EntityId.Key), data) { LastModifiedTime = metadata.LastModifiedTime, diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs index 1a8479a9d..d378e7f6e 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableTaskClient.cs @@ -336,6 +336,7 @@ public override async Task RestartAsync( SerializedOutput = state.Output, SerializedCustomStatus = state.Status, FailureDetails = state.FailureDetails?.ConvertFromCore(), + EnableLargePayloadSupport = this.SupportsAsyncSerialization, }; } From a5549a3d5e14f0fb3e0583eabc4d2def4d71ec2f Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:18:57 -0700 Subject: [PATCH 34/38] more --- src/Client/Grpc/GrpcDurableEntityClient.cs | 4 +-- .../ShimDurableEntityClient.cs | 29 +++---------------- 2 files changed, 6 insertions(+), 27 deletions(-) diff --git a/src/Client/Grpc/GrpcDurableEntityClient.cs b/src/Client/Grpc/GrpcDurableEntityClient.cs index c28390a1f..826b26857 100644 --- a/src/Client/Grpc/GrpcDurableEntityClient.cs +++ b/src/Client/Grpc/GrpcDurableEntityClient.cs @@ -95,7 +95,7 @@ public override async Task SignalEntityAsync( /// public override Task?> GetEntityAsync( EntityInstanceId id, bool includeState = false, CancellationToken cancellation = default) - => this.GetEntityCoreAsync(id, includeState, this.ToEntityMetadata, cancellation); + => this.GetEntityCoreAsync(id, includeState, (e, s) => this.ToEntityMetadata(e, s), cancellation); /// public override AsyncPageable GetAllEntitiesAsync(EntityQuery? filter = null) @@ -103,7 +103,7 @@ public override AsyncPageable GetAllEntitiesAsync(EntityQuery? f /// public override AsyncPageable> GetAllEntitiesAsync(EntityQuery? filter = null) - => this.GetAllEntitiesCoreAsync(filter, this.ToEntityMetadata); + => this.GetAllEntitiesCoreAsync(filter, (x, s) => this.ToEntityMetadata(x, s)); /// public override async Task CleanEntityStorageAsync( diff --git a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs index e50672804..88d941050 100644 --- a/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs +++ b/src/Client/OrchestrationServiceClientShim/ShimDurableEntityClient.cs @@ -26,11 +26,6 @@ class ShimDurableEntityClient(string name, ShimDurableTaskClientOptions options) DataConverter Converter => this.options.DataConverter; - /// - /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). - /// - bool SupportsAsyncSerialization => this.options.EnableLargePayloadSupport; - /// public override async Task CleanEntityStorageAsync( CleanEntityStorageRequest? request = null, @@ -61,9 +56,7 @@ public override AsyncPageable GetAllEntitiesAsync(EntityQuery? f /// public override AsyncPageable> GetAllEntitiesAsync(EntityQuery? filter = null) - => this.SupportsAsyncSerialization - ? this.GetAllEntitiesAsync(this.Convert, filter) - : this.GetAllEntitiesAsync(this.ConvertSync, filter); + => this.GetAllEntitiesAsync(this.Convert, filter); /// public override async Task GetEntityAsync( @@ -89,7 +82,7 @@ public override async Task SignalEntityAsync( Check.NotNull(id.Key); DateTimeOffset? scheduledTime = options?.SignalTime; - string? serializedInput = this.SupportsAsyncSerialization + string? serializedInput = this.options.EnableLargePayloadSupport ? await this.Converter.SerializeAsync(input, cancellation) : this.Converter.Serialize(input); @@ -176,7 +169,7 @@ AsyncPageable GetAllEntitiesAsync( async Task> Convert(EntityBackendQueries.EntityMetadata metadata) { - T? state = this.SupportsAsyncSerialization + T? state = this.options.EnableLargePayloadSupport ? await this.Converter.DeserializeAsync(metadata.SerializedState) : this.Converter.Deserialize(metadata.SerializedState); @@ -190,20 +183,6 @@ async Task> Convert(EntityBackendQueries.EntityMetadata met }; } - EntityMetadata ConvertSync(EntityBackendQueries.EntityMetadata metadata) - { - T? state = this.Converter.Deserialize(metadata.SerializedState); - - return new( - metadata.EntityId.ConvertFromCore(), - state) - { - LastModifiedTime = metadata.LastModifiedTime, - BacklogQueueSize = metadata.BacklogQueueSize, - LockedBy = metadata.LockedBy, - }; - } - async Task?> Convert(EntityBackendQueries.EntityMetadata? metadata) { if (metadata is null) @@ -220,7 +199,7 @@ EntityMetadata Convert(EntityBackendQueries.EntityMetadata metadata) ? null : new SerializedData(metadata.SerializedState, this.Converter) { - EnableLargePayloadSupport = this.SupportsAsyncSerialization, + EnableLargePayloadSupport = this.options.EnableLargePayloadSupport, }; return new(new EntityInstanceId(metadata.EntityId.Name, metadata.EntityId.Key), data) { From 738892b9e42dc7e08e2a8f15990c33730d723cbe Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 14 Sep 2025 11:34:20 -0700 Subject: [PATCH 35/38] refactor --- .../Converters/BlobPayloadStore.cs | 28 +++++++++++-------- .../Converters/LargePayloadDataConverter.cs | 4 +-- .../Core/Shims/DurableTaskShimFactory.cs | 14 ++++++++-- 3 files changed, 31 insertions(+), 15 deletions(-) diff --git a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs index e60cd403f..1f101288d 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/BlobPayloadStore.cs @@ -18,6 +18,12 @@ namespace Microsoft.DurableTask.Converters; public sealed class BlobPayloadStore : IPayloadStore { const string TokenPrefix = "blob:v1:"; + const string ContentEncodingGzip = "gzip"; + const int DefaultCopyBufferSize = 81920; + const int MaxRetryAttempts = 8; + const int BaseDelayMs = 250; + const int MaxDelayMs = 10_000; + const int MaxJitterMs = 100; // Jitter RNG for retry backoff static readonly object RandomLock = new object(); @@ -43,8 +49,8 @@ public BlobPayloadStore(LargePayloadStorageOptions options) Retry = { Mode = RetryMode.Exponential, - MaxRetries = 8, - Delay = TimeSpan.FromMilliseconds(250), + MaxRetries = MaxRetryAttempts, + Delay = TimeSpan.FromMilliseconds(BaseDelayMs), MaxDelay = TimeSpan.FromSeconds(10), NetworkTimeout = TimeSpan.FromMinutes(2), }, @@ -73,13 +79,13 @@ public async Task UploadAsync(ReadOnlyMemory payloadBytes, Cancell { BlobOpenWriteOptions writeOptions = new() { - HttpHeaders = new BlobHttpHeaders { ContentEncoding = "gzip" }, + HttpHeaders = new BlobHttpHeaders { ContentEncoding = ContentEncodingGzip }, }; using Stream blobStream = await blob.OpenWriteAsync(true, writeOptions, ct); using GZipStream compressedBlobStream = new(blobStream, CompressionLevel.Optimal, leaveOpen: true); using MemoryStream payloadStream = new(payloadBuffer, writable: false); - await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: 81920, ct); + await payloadStream.CopyToAsync(compressedBlobStream, bufferSize: DefaultCopyBufferSize, ct); await compressedBlobStream.FlushAsync(ct); await blobStream.FlushAsync(ct); } @@ -87,7 +93,7 @@ public async Task UploadAsync(ReadOnlyMemory payloadBytes, Cancell { using Stream blobStream = await blob.OpenWriteAsync(true, default, ct); using MemoryStream payloadStream = new(payloadBuffer, writable: false); - await payloadStream.CopyToAsync(blobStream, bufferSize: 81920, ct); + await payloadStream.CopyToAsync(blobStream, bufferSize: DefaultCopyBufferSize, ct); await blobStream.FlushAsync(ct); } @@ -115,7 +121,7 @@ public async Task DownloadAsync(string token, CancellationToken cancella using BlobDownloadStreamingResult result = await blob.DownloadStreamingAsync(cancellationToken: ct); Stream contentStream = result.Content; bool isGzip = string.Equals( - result.Details.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase); + result.Details.ContentEncoding, ContentEncodingGzip, StringComparison.OrdinalIgnoreCase); if (isGzip) { @@ -162,8 +168,8 @@ public bool IsKnownPayloadToken(string value) static async Task WithTransientRetryAsync(Func> operation, CancellationToken cancellationToken) { - const int maxAttempts = 8; - TimeSpan baseDelay = TimeSpan.FromMilliseconds(250); + const int maxAttempts = MaxRetryAttempts; + TimeSpan baseDelay = TimeSpan.FromMilliseconds(BaseDelayMs); int attempt = 0; while (true) @@ -201,9 +207,9 @@ static TimeSpan ComputeBackoff(TimeSpan baseDelay, int attempt) int jitterMs; lock (RandomLock) { - jitterMs = SharedRandom.Next(0, 100); + jitterMs = SharedRandom.Next(0, MaxJitterMs); } - return TimeSpan.FromMilliseconds(Math.Min((baseDelay.TotalMilliseconds * factor) + jitterMs, 10_000)); - } + return TimeSpan.FromMilliseconds(Math.Min((baseDelay.TotalMilliseconds * factor) + jitterMs, MaxDelayMs)); + } } diff --git a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs index edcf1a6fa..03a2c3e7c 100644 --- a/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs +++ b/src/Extensions/AzureBlobPayloads/Converters/LargePayloadDataConverter.cs @@ -37,14 +37,14 @@ public sealed class LargePayloadDataConverter( [return: NotNullIfNotNull("value")] public override string? Serialize(object? value) { - throw new NotImplementedException(); + throw new NotSupportedException(); } /// [return: NotNullIfNotNull("data")] public override object? Deserialize(string? data, Type targetType) { - throw new NotImplementedException(); + throw new NotSupportedException(); } /// diff --git a/src/Worker/Core/Shims/DurableTaskShimFactory.cs b/src/Worker/Core/Shims/DurableTaskShimFactory.cs index e222c5ec3..7ff3a73ca 100644 --- a/src/Worker/Core/Shims/DurableTaskShimFactory.cs +++ b/src/Worker/Core/Shims/DurableTaskShimFactory.cs @@ -50,7 +50,12 @@ public TaskActivity CreateActivity(TaskName name, ITaskActivity activity) { Check.NotDefault(name); Check.NotNull(activity); - return new TaskActivityShim(this.loggerFactory, this.options.DataConverter, name, activity, this.options.EnableLargePayloadSupport); + return new TaskActivityShim( + this.loggerFactory, + this.options.DataConverter, + name, + activity, + this.options.EnableLargePayloadSupport); } /// @@ -151,6 +156,11 @@ public TaskEntity CreateEntity(TaskName name, ITaskEntity entity, EntityId entit // In the future we may consider caching those shims and reusing them, which can reduce // deserialization and allocation overheads. ILogger logger = this.loggerFactory.CreateLogger(entity.GetType()); - return new TaskEntityShim(this.options.DataConverter, entity, entityId, logger, this.options.EnableLargePayloadSupport); + return new TaskEntityShim( + this.options.DataConverter, + entity, + entityId, + logger, + enableLargePayloadSupport: this.options.EnableLargePayloadSupport); } } From 1064329e3e4fa758f67fbc2d2cc91a7022ac2b5d Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 14 Sep 2025 13:00:45 -0700 Subject: [PATCH 36/38] more --- .../Entities/TaskEntityContext.cs | 18 ++++++ src/Worker/Core/Shims/TaskEntityShim.cs | 26 +++++++- .../Shims/TaskOrchestrationContextWrapper.cs | 60 ++++++++++++++++--- .../Shims/TaskOrchestrationEntityContext.cs | 9 ++- .../Core/Shims/TaskOrchestrationShim.cs | 19 +++--- 5 files changed, 109 insertions(+), 23 deletions(-) diff --git a/src/Abstractions/Entities/TaskEntityContext.cs b/src/Abstractions/Entities/TaskEntityContext.cs index a638c0fbd..6ec68e5fd 100644 --- a/src/Abstractions/Entities/TaskEntityContext.cs +++ b/src/Abstractions/Entities/TaskEntityContext.cs @@ -68,4 +68,22 @@ public virtual ValueTask ScheduleNewOrchestrationAsync( { return new ValueTask(this.ScheduleNewOrchestration(name, input, options)); } + + /// + /// Signals an entity operation asynchronously. + /// + /// The entity to signal. + /// The operation name. + /// The operation input. + /// The options to signal with. + /// A task representing the asynchronous operation. + public virtual ValueTask SignalEntityAsync( + EntityInstanceId id, + string operationName, + object? input = null, + SignalEntityOptions? options = null) + { + this.SignalEntity(id, operationName, input, options); + return default; + } } diff --git a/src/Worker/Core/Shims/TaskEntityShim.cs b/src/Worker/Core/Shims/TaskEntityShim.cs index ed36bf8c9..9724558b3 100644 --- a/src/Worker/Core/Shims/TaskEntityShim.cs +++ b/src/Worker/Core/Shims/TaskEntityShim.cs @@ -253,7 +253,26 @@ public void Reset() this.checkpointPosition = 0; } - public override async void SignalEntity(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) + public override void SignalEntity(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) + { + Check.NotDefault(id); + + this.operationActions.Add(new SendSignalOperationAction() + { + InstanceId = id.ToString(), + Name = operationName, + Input = this.dataConverter.Serialize(input), + ScheduledTime = options?.SignalTime?.UtcDateTime, + RequestTime = DateTimeOffset.UtcNow, + ParentTraceContext = this.parentTraceContext, + }); + } + + public override async ValueTask SignalEntityAsync( + EntityInstanceId id, + string operationName, + object? input = null, + SignalEntityOptions? options = null) { Check.NotDefault(id); @@ -270,7 +289,10 @@ public override async void SignalEntity(EntityInstanceId id, string operationNam }); } - public override string ScheduleNewOrchestration(TaskName name, object? input = null, StartOrchestrationOptions? options = null) + public override string ScheduleNewOrchestration( + TaskName name, + object? input = null, + StartOrchestrationOptions? options = null) { Check.NotEntity(true, options?.InstanceId); Check.ThrowIfLargePayloadEnabled(this.enableLargePayloadSupport, nameof(this.ScheduleNewOrchestration)); diff --git a/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs b/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs index 852315b9a..d818541f6 100644 --- a/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs +++ b/src/Worker/Core/Shims/TaskOrchestrationContextWrapper.cs @@ -113,11 +113,6 @@ public override TaskOrchestrationEntityFeature Entities /// internal DataConverter DataConverter => this.invocationContext.Options.DataConverter; - /// - /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). - /// - bool SupportsAsyncSerialization => this.invocationContext.Options.EnableLargePayloadSupport; - /// protected override ILoggerFactory LoggerFactory => this.invocationContext.LoggerFactory; @@ -288,7 +283,7 @@ public override Task WaitForExternalEvent(string eventName, CancellationTo // Return immediately if this external event has already arrived. if (this.externalEventBuffer.TryTake(eventName, out string? bufferedEventPayload)) { - return this.SupportsAsyncSerialization + return this.invocationContext.Options.EnableLargePayloadSupport ? this.DataConverter.DeserializeAsync(bufferedEventPayload, cancellationToken).AsTask() : Task.FromResult(this.DataConverter.Deserialize(bufferedEventPayload)); } @@ -421,7 +416,50 @@ internal void ExitCriticalSectionIfNeeded() /// /// The name of the event to complete. /// The serialized event payload. - internal async Task CompleteExternalEvent(string eventName, string rawEventPayload) + internal void CompleteExternalEvent(string eventName, string rawEventPayload) + { + Check.ThrowIfLargePayloadEnabled( + this.invocationContext.Options.EnableLargePayloadSupport, + nameof(this.CompleteExternalEvent)); + if (this.externalEventSources.TryGetValue(eventName, out Queue? waiters)) + { + object? value; + + IEventSource waiter = waiters.Dequeue(); + if (waiter.EventType == typeof(OperationResult)) + { + // use the framework-defined deserialization for entity responses, not the application-defined data converter, + // because we are just unwrapping the entity response without yet deserializing any application-defined data. + value = this.entityFeature!.EntityContext.DeserializeEntityResponseEvent(rawEventPayload); + } + else + { + value = this.DataConverter.Deserialize(rawEventPayload, waiter.EventType); + } + + // Events are completed in FIFO order. Remove the key if the last event was delivered. + if (waiters.Count == 0) + { + this.externalEventSources.Remove(eventName); + } + + waiter.TrySetResult(value); + } + else + { + // The orchestrator isn't waiting for this event (yet?). Save it in case + // the orchestrator wants it later. + this.externalEventBuffer.Add(eventName, rawEventPayload); + } + } + + /// + /// Completes the external event by name, allowing the orchestration to continue if it is waiting on this event. + /// + /// The name of the event to complete. + /// The serialized event payload. + /// A task representing the asynchronous operation. + internal async Task CompleteExternalEventAsync(string eventName, string rawEventPayload) { if (this.externalEventSources.TryGetValue(eventName, out Queue? waiters)) { @@ -436,7 +474,7 @@ internal async Task CompleteExternalEvent(string eventName, string rawEventPaylo } else { - value = this.SupportsAsyncSerialization + value = this.invocationContext.Options.EnableLargePayloadSupport ? await this.DataConverter.DeserializeAsync(rawEventPayload, waiter.EventType, CancellationToken.None) : this.DataConverter.Deserialize(rawEventPayload, waiter.EventType); } @@ -460,10 +498,11 @@ internal async Task CompleteExternalEvent(string eventName, string rawEventPaylo /// /// Gets the serialized custom status. /// + /// The cancellation token. /// The custom status serialized to a string, or null if there is not custom status. internal async ValueTask GetSerializedCustomStatusAsync(CancellationToken cancellationToken = default) { - return this.SupportsAsyncSerialization + return this.invocationContext.Options.EnableLargePayloadSupport ? await this.DataConverter.SerializeAsync(this.customStatus, cancellationToken) : this.DataConverter.Serialize(this.customStatus); } @@ -474,6 +513,9 @@ internal async Task CompleteExternalEvent(string eventName, string rawEventPaylo /// The custom status serialized to a string, or null if there is not custom status. internal string? GetSerializedCustomStatus() { + Check.ThrowIfLargePayloadEnabled( + this.invocationContext.Options.EnableLargePayloadSupport, + nameof(this.GetSerializedCustomStatus)); return this.DataConverter.Serialize(this.customStatus); } diff --git a/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs b/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs index 0934b955a..a62514318 100644 --- a/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs +++ b/src/Worker/Core/Shims/TaskOrchestrationEntityContext.cs @@ -98,7 +98,7 @@ public override async Task CallEntityAsync(EntityInstanceId id } else { - return this.wrapper.SupportsAsyncSerialization + return this.wrapper.invocationContext.Options.EnableLargePayloadSupport ? await this.wrapper.DataConverter.DeserializeAsync(operationResult.Result) : this.wrapper.DataConverter.Deserialize(operationResult.Result); } @@ -117,11 +117,10 @@ public override async Task CallEntityAsync(EntityInstanceId id, string operation } /// - public override Task SignalEntityAsync(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) + public override async Task SignalEntityAsync(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) { Check.NotDefault(id); - this.SendOperationMessage(id.ToString(), operationName, input, oneWay: true, scheduledTime: options?.SignalTime); - return Task.CompletedTask; + await this.SendOperationMessage(id.ToString(), operationName, input, oneWay: true, scheduledTime: options?.SignalTime); } /// @@ -196,7 +195,7 @@ async Task SendOperationMessage(string instanceId, string operationName, o } Guid guid = this.wrapper.NewGuid(); // deterministically replayable unique id for this request - string? serializedInput = this.wrapper.SupportsAsyncSerialization + string? serializedInput = this.wrapper.invocationContext.Options.EnableLargePayloadSupport ? await this.wrapper.DataConverter.SerializeAsync(input, CancellationToken.None) : this.wrapper.DataConverter.Serialize(input); var target = new OrchestrationInstance() { InstanceId = instanceId }; diff --git a/src/Worker/Core/Shims/TaskOrchestrationShim.cs b/src/Worker/Core/Shims/TaskOrchestrationShim.cs index 3d4fdc042..b1f2e8c2f 100644 --- a/src/Worker/Core/Shims/TaskOrchestrationShim.cs +++ b/src/Worker/Core/Shims/TaskOrchestrationShim.cs @@ -55,11 +55,6 @@ public TaskOrchestrationShim( DataConverter DataConverter => this.invocationContext.Options.DataConverter; - /// - /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). - /// - bool SupportsAsyncSerialization => this.invocationContext.Options.EnableLargePayloadSupport; - /// public override async Task Execute(OrchestrationContext innerContext, string rawInput) { @@ -68,7 +63,7 @@ public TaskOrchestrationShim( innerContext.MessageDataConverter = converterShim; innerContext.ErrorDataConverter = converterShim; - object? input = this.SupportsAsyncSerialization + object? input = this.invocationContext.Options.EnableLargePayloadSupport ? await this.DataConverter.DeserializeAsync(rawInput, this.implementation.InputType) : this.DataConverter.Deserialize(rawInput, this.implementation.InputType); this.wrapperContext = new(innerContext, this.invocationContext, input, this.properties); @@ -89,7 +84,7 @@ public TaskOrchestrationShim( } // Return the output (if any) as a serialized string. - return this.SupportsAsyncSerialization + return this.invocationContext.Options.EnableLargePayloadSupport ? await this.DataConverter.SerializeAsync(output) : this.DataConverter.Serialize(output); } @@ -128,6 +123,7 @@ public TaskOrchestrationShim( /// public override string? GetStatus() { + Check.ThrowIfLargePayloadEnabled(this.invocationContext.Options.EnableLargePayloadSupport, nameof(this.GetStatus)); return this.wrapperContext?.GetSerializedCustomStatus(); } @@ -136,4 +132,13 @@ public override void RaiseEvent(OrchestrationContext context, string name, strin { this.wrapperContext?.CompleteExternalEvent(name, input); } + + /// + public override async Task RaiseEventAsync(OrchestrationContext context, string name, string input) + { + if (this.wrapperContext != null) + { + await this.wrapperContext.CompleteExternalEventAsync(name, input); + } + } } From cd05d6aedea5b28b15767c3e9ecf637956e26785 Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 14 Sep 2025 13:07:59 -0700 Subject: [PATCH 37/38] more --- src/Worker/Core/Shims/TaskEntityShim.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Worker/Core/Shims/TaskEntityShim.cs b/src/Worker/Core/Shims/TaskEntityShim.cs index 9724558b3..9d9516174 100644 --- a/src/Worker/Core/Shims/TaskEntityShim.cs +++ b/src/Worker/Core/Shims/TaskEntityShim.cs @@ -193,6 +193,7 @@ public void Reset() public override void SetState(object? state) { + Check.ThrowIfLargePayloadEnabled(this.enableLargePayloadSupport, nameof(this.SetState)); this.value = this.dataConverter.Serialize(state); this.cachedValue = state; } @@ -256,7 +257,7 @@ public void Reset() public override void SignalEntity(EntityInstanceId id, string operationName, object? input = null, SignalEntityOptions? options = null) { Check.NotDefault(id); - + Check.ThrowIfLargePayloadEnabled(this.enableLargePayloadSupport, nameof(this.SignalEntity)); this.operationActions.Add(new SendSignalOperationAction() { InstanceId = id.ToString(), From b3cbb43abf4d57c7b41b1df474323096aa52f3cd Mon Sep 17 00:00:00 2001 From: peterstone2017 <12449837+YunchuWang@users.noreply.github.com> Date: Sun, 14 Sep 2025 14:36:03 -0700 Subject: [PATCH 38/38] more --- samples/LargePayloadConsoleApp/Program.cs | 4 ++-- src/Client/Grpc/GrpcDurableTaskClient.cs | 13 ++++--------- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/samples/LargePayloadConsoleApp/Program.cs b/samples/LargePayloadConsoleApp/Program.cs index 0cef4a601..87258e1ea 100644 --- a/samples/LargePayloadConsoleApp/Program.cs +++ b/samples/LargePayloadConsoleApp/Program.cs @@ -25,7 +25,7 @@ { b.UseDurableTaskScheduler(schedulerConnectionString); // Ensure entity APIs are enabled for the client - b.Configure(o => o.EnableEntitySupport = true); + b.Configure(o => { o.EnableEntitySupport = true; o.EnableLargePayloadSupport = true; }); b.UseExternalizedPayloads(opts => { // Keep threshold small to force externalization for demo purposes @@ -104,7 +104,7 @@ await ctx.Entities.CallEntityAsync( opts.ContainerName = builder.Configuration.GetValue("DURABLETASK_PAYLOAD_CONTAINER"); }); // Ensure entity APIs are enabled for the worker - b.Configure(o => o.EnableEntitySupport = true); + b.Configure(o => { o.EnableEntitySupport = true; o.EnableLargePayloadSupport = true; }); }); IHost host = builder.Build(); diff --git a/src/Client/Grpc/GrpcDurableTaskClient.cs b/src/Client/Grpc/GrpcDurableTaskClient.cs index 5706eb658..6aab2efd2 100644 --- a/src/Client/Grpc/GrpcDurableTaskClient.cs +++ b/src/Client/Grpc/GrpcDurableTaskClient.cs @@ -64,11 +64,6 @@ public GrpcDurableTaskClient(string name, GrpcDurableTaskClientOptions options, DataConverter DataConverter => this.options.DataConverter; - /// - /// Gets a value indicating whether the DataConverter supports async operations (LargePayload enabled). - /// - bool SupportsAsyncSerialization => this.options.EnableLargePayloadSupport; - /// public override ValueTask DisposeAsync() { @@ -102,7 +97,7 @@ public override async Task ScheduleNewOrchestrationInstanceAsync( Name = orchestratorName.Name, Version = version, InstanceId = instanceId, - Input = this.SupportsAsyncSerialization + Input = this.options.EnableLargePayloadSupport ? await this.DataConverter.SerializeAsync(input, cancellation) : this.DataConverter.Serialize(input), RequestTime = DateTimeOffset.UtcNow.ToTimestamp(), @@ -156,7 +151,7 @@ public override async Task RaiseEventAsync( { InstanceId = instanceId, Name = eventName, - Input = this.SupportsAsyncSerialization + Input = this.options.EnableLargePayloadSupport ? await this.DataConverter.SerializeAsync(eventPayload, cancellation) : this.DataConverter.Serialize(eventPayload), }; @@ -178,7 +173,7 @@ public override async Task TerminateInstanceAsync( this.logger.TerminatingInstance(instanceId); - string? serializedOutput = this.SupportsAsyncSerialization + string? serializedOutput = this.options.EnableLargePayloadSupport ? await this.DataConverter.SerializeAsync(output, cancellation) : this.DataConverter.Serialize(output); await this.sidecarClient.TerminateInstanceAsync( @@ -521,7 +516,7 @@ OrchestrationMetadata CreateMetadata(P.OrchestrationState state, bool includeInp SerializedCustomStatus = state.CustomStatus, FailureDetails = state.FailureDetails.ToTaskFailureDetails(), DataConverter = includeInputsAndOutputs ? this.DataConverter : null, - EnableLargePayloadSupport = this.SupportsAsyncSerialization, + EnableLargePayloadSupport = this.options.EnableLargePayloadSupport, Tags = new Dictionary(state.Tags), };