Introduce a Lightweight Scaling Package for Multi Durable Backends

.github/workflows/scale-tests.yml

@@ -0,0 +1,201 @@
+name: DurableTask Scale Tests
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ main, dev ]
+    paths:
+      - 'src/WebJobs.Extensions.DurableTask.Scale/**'
+      - 'test/ScaleTests/**'
+      - '.github/workflows/scale-tests.yml'
+  pull_request:
+    branches: [ main, dev ]
+    paths:
+      - 'src/WebJobs.Extensions.DurableTask.Scale/**'
+      - 'test/ScaleTests/**'
+      - '.github/workflows/scale-tests.yml'
+
+jobs:
+  scale-tests-azurestorage:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET 8.0
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 8.0.x
+
+      - name: Set up Node.js (needed for Azurite)
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Install Azurite
+        run: npm install -g azurite
+
+      - name: Start Azurite
+        run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
+
+      - name: Restore and Build Scale Tests
+        working-directory: test/ScaleTests
+        run: dotnet build --configuration Release
+
+      - name: Run Azure Storage Scale Tests
+        working-directory: test/ScaleTests
+        env:
+          AzureWebJobsStorage: UseDevelopmentStorage=true
+        run: dotnet test --configuration Release --no-build --verbosity normal --filter "FullyQualifiedName~AzureStorage&FullyQualifiedName!~Sql&FullyQualifiedName!~DurableTaskTriggersScaleProviderSqlServer"
+
+  scale-tests-sql:
+    runs-on: ubuntu-latest
+    env:
+      MSSQL_SA_PASSWORD: "Strong!Passw0rd123"
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET 8.0
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 8.0.x
+
+      - name: Set up Node.js (needed for Azurite)
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Install Azurite
+        run: npm install -g azurite
+
+      - name: Start Azurite
+        run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
+
+      - name: Pull SQL Server Docker Image
+        run: docker pull mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Start SQL Server Container
+        run: |
+          docker run --name mssql-server \
+            -e ACCEPT_EULA=Y \
+            -e "MSSQL_SA_PASSWORD=${{ env.MSSQL_SA_PASSWORD }}" \
+            -e "MSSQL_PID=Express" \
+            -p 1433:1433 \
+            -d mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Wait for SQL Server to be ready
+        run: |
+          echo "Waiting for SQL Server to be ready..."
+          for i in {1..30}; do
+            # Try to connect using sqlcmd (try multiple possible paths for different SQL Server versions)
+            if docker exec mssql-server /opt/mssql-tools18/bin/sqlcmd -S . -U sa -P "${{ env.MSSQL_SA_PASSWORD }}" -Q "SELECT 1" -C > /dev/null 2>&1 || \
+               docker exec mssql-server /opt/mssql-tools/bin/sqlcmd -S . -U sa -P "${{ env.MSSQL_SA_PASSWORD }}" -Q "SELECT 1" -C > /dev/null 2>&1; then
+              echo "SQL Server is ready!"
+              break
+            fi
+            echo "Waiting for SQL Server... ($i/30)"
+            sleep 2
+          done
+          docker ps
+
+      - name: Create database
+        run: |
+          echo "Creating TestDurableDB database..."
+          # Try multiple possible sqlcmd paths for different SQL Server versions
+          if docker exec mssql-server /opt/mssql-tools18/bin/sqlcmd -S . -U sa -P "${{ env.MSSQL_SA_PASSWORD }}" -Q "CREATE DATABASE [TestDurableDB] COLLATE Latin1_General_100_BIN2_UTF8" -C 2>/dev/null || \
+             docker exec mssql-server /opt/mssql-tools/bin/sqlcmd -S . -U sa -P "${{ env.MSSQL_SA_PASSWORD }}" -Q "CREATE DATABASE [TestDurableDB] COLLATE Latin1_General_100_BIN2_UTF8" -C; then
+            echo "Database created successfully"
+          else
+            echo "Failed to create database"
+            exit 1
+          fi
+
+      - name: Restore and Build Scale Tests
+        working-directory: test/ScaleTests
+        run: dotnet build --configuration Release
+
+      - name: Run SQL Server Scale Tests
+        working-directory: test/ScaleTests
+        env:
+          AzureWebJobsStorage: UseDevelopmentStorage=true
+          SQLDB_Connection: "Server=localhost,1433;Database=TestDurableDB;User Id=sa;Password=${{ env.MSSQL_SA_PASSWORD }};TrustServerCertificate=True;Encrypt=False;"
+        run: dotnet test --configuration Release --no-build --verbosity normal --filter "FullyQualifiedName~Sql|FullyQualifiedName~DurableTaskTriggersScaleProviderSqlServer"
+
+  scale-tests-azuremanaged:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET 8.0
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 8.0.x
+
+      - name: Set up Node.js (needed for Azurite)
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Install Azurite
+        run: npm install -g azurite
+
+      - name: Start Azurite
+        run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
+
+      - name: Pull DTS Emulator Docker Image
+        run: docker pull mcr.microsoft.com/dts/dts-emulator:latest
+
+      - name: Start DTS Container
+        run: |
+          docker run -i \
+            -p 8080:8080 \
+            -p 8082:8082 \
+            -d mcr.microsoft.com/dts/dts-emulator:latest
+
+      - name: Wait for DTS to be ready
+        run: |
+          echo "Waiting for DTS to be ready..."
+          sleep 30
+          docker ps
+
+      - name: Restore and Build Scale Tests
+        working-directory: test/ScaleTests
+        run: dotnet build --configuration Release
+
+      - name: Run Azure Managed Scale Tests
+        working-directory: test/ScaleTests
+        env:
+          AzureWebJobsStorage: UseDevelopmentStorage=true
+          DURABLE_TASK_SCHEDULER_CONNECTION_STRING: "Endpoint=http://localhost:8080;Authentication=None"
+        run: dotnet test --configuration Release --no-build --verbosity normal --filter "FullyQualifiedName~AzureManaged&FullyQualifiedName!~Sql&FullyQualifiedName!~DurableTaskTriggersScaleProviderSqlServer"
+
+  scale-tests-configuration:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup .NET 8.0
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: 8.0.x
+
+      - name: Set up Node.js (needed for Azurite)
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Install Azurite
+        run: npm install -g azurite
+
+      - name: Start Azurite
+        run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
+
+      - name: Restore and Build Scale Tests
+        working-directory: test/ScaleTests
+        run: dotnet build --configuration Release
+
+      - name: Run Configuration Extension Tests
+        working-directory: test/ScaleTests
+        env:
+          AzureWebJobsStorage: UseDevelopmentStorage=true
+        run: dotnet test --configuration Release --no-build --verbosity normal --filter "FullyQualifiedName~DurableTaskJobHostConfigurationExtensionsTests"
+

Directory.Packages.props

@@ -10,7 +10,7 @@
   <ItemGroup>
     <PackageVersion Include="Azure.Identity" Version="1.17.0" />
     <PackageVersion Include="Grpc.Tools" Version="2.49.0" />
-    <PackageVersion Include="Grpc.Net.Client" Version="2.70.0" />
+    <PackageVersion Include="Grpc.Net.Client" Version="2.71.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Http.Abstractions" Version="2.3.0" />
     <PackageVersion Include="Microsoft.AspNetCore.Mvc.WebApiCompatShim" Version="2.2.0" />
     <PackageVersion Include="Microsoft.Azure.DurableTask.ApplicationInsights" Version="0.8.0" />
@@ -31,6 +31,9 @@
     <PackageVersion Include="Microsoft.DurableTask.Worker.Grpc" Version="1.17.1" />
     <PackageVersion Include="Microsoft.DurableTask.Abstractions" Version="1.17.1" />
     <PackageVersion Include="Microsoft.DurableTask.Analyzers" Version="0.1.0" />
+    <PackageVersion Include="Microsoft.DurableTask.AzureManagedBackend" Version="1.0.0" />
+    <PackageVersion Include="Microsoft.DurableTask.SqlServer" Version="1.5.2" />
+    <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.1.2" />
     <PackageVersion Include="Microsoft.Extensions.Azure" Version="1.7.0" />
     <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="6.0.3" />
     <PackageVersion Include="Microsoft.Extensions.Http" Version="6.0.0" />
@@ -56,7 +59,7 @@
     <PackageVersion Include="Microsoft.Azure.DurableTask.Redis" Version="0.1.9-alpha" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker" Version="2.2.0" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="2.0.0" />
-    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" Version="0.4.2-alpha" />
+    <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.AzureManaged" Version="1.0.0" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.DurableTask.SqlServer" Version="1.5.2" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.3.0" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Http.AspNetCore" Version="2.0.2" />

WebJobs.Extensions.DurableTask.sln

@@ -62,6 +62,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotNetIsolated", "test\Smok
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Worker.Extensions.DurableTask", "src\Worker.Extensions.DurableTask\Worker.Extensions.DurableTask.csproj", "{5F5FAF27-D6B8-4A60-ACF2-F63D13F89CA2}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Extensions.DurableTask.Scale", "src\WebJobs.Extensions.DurableTask.Scale\WebJobs.Extensions.DurableTask.Scale.csproj", "{A2E5E1B2-2B4B-4B6F-A0A7-6D7E0E68F1C9}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Extensions.DurableTask.Scale.Tests", "test\ScaleTests\WebJobs.Extensions.DurableTask.Scale.Tests.csproj", "{B3F6E2C3-3C5C-5C7F-B1B8-7E8F1F79F2D0}"
+EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipelines", "pipelines", "{B7FBBE6E-9AC7-4CEB-9B80-6FD9AE74415A}"
 	ProjectSection(SolutionItems) = preProject
 		azure-pipelines.yml = azure-pipelines.yml
@@ -126,6 +130,14 @@ Global
 		{5F5FAF27-D6B8-4A60-ACF2-F63D13F89CA2}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{5F5FAF27-D6B8-4A60-ACF2-F63D13F89CA2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{5F5FAF27-D6B8-4A60-ACF2-F63D13F89CA2}.Release|Any CPU.Build.0 = Release|Any CPU
+		{A2E5E1B2-2B4B-4B6F-A0A7-6D7E0E68F1C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A2E5E1B2-2B4B-4B6F-A0A7-6D7E0E68F1C9}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A2E5E1B2-2B4B-4B6F-A0A7-6D7E0E68F1C9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A2E5E1B2-2B4B-4B6F-A0A7-6D7E0E68F1C9}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B3F6E2C3-3C5C-5C7F-B1B8-7E8F1F79F2D0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B3F6E2C3-3C5C-5C7F-B1B8-7E8F1F79F2D0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B3F6E2C3-3C5C-5C7F-B1B8-7E8F1F79F2D0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B3F6E2C3-3C5C-5C7F-B1B8-7E8F1F79F2D0}.Release|Any CPU.Build.0 = Release|Any CPU
 		{FC8AD123-F949-4D21-B817-E5A4BBF7F69B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{FC8AD123-F949-4D21-B817-E5A4BBF7F69B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FC8AD123-F949-4D21-B817-E5A4BBF7F69B}.Release|Any CPU.ActiveCfg = Release|Any CPU
@@ -162,6 +174,8 @@ Global
 		{9B0F4A0A-1B18-4E98-850A-14A2F76F673D} = {A8CF6993-258A-484A-AF6D-6CC88D36AF93}
 		{FF6CD07A-A4BF-43C5-B14E-213328DEB835} = {9B0F4A0A-1B18-4E98-850A-14A2F76F673D}
 		{5F5FAF27-D6B8-4A60-ACF2-F63D13F89CA2} = {7EC858EE-3481-4A82-AED4-CB00C34F42D0}
+		{A2E5E1B2-2B4B-4B6F-A0A7-6D7E0E68F1C9} = {7EC858EE-3481-4A82-AED4-CB00C34F42D0}
+		{B3F6E2C3-3C5C-5C7F-B1B8-7E8F1F79F2D0} = {78BCF152-C22C-408F-9FB1-0F8C99B154B5}
 		{7387E723-E153-4B7A-B105-8C67BFBD48CF} = {78BCF152-C22C-408F-9FB1-0F8C99B154B5}
 		{FC8AD123-F949-4D21-B817-E5A4BBF7F69B} = {7387E723-E153-4B7A-B105-8C67BFBD48CF}
 		{76DEC17C-BF6A-498A-8E8A-7D6CB2E03284} = {78BCF152-C22C-408F-9FB1-0F8C99B154B5}

src/WebJobs.Extensions.DurableTask.Scale/AzureManaged/AzureManagedConnectionString.cs

@@ -0,0 +1,60 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System.Data.Common;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Scale.AzureManaged
+{
+    /// <summary>
+    /// Connection string to conenct to AzureManaged backend service.
+    /// </summary>
+    public sealed class AzureManagedConnectionString
+    {
+        private readonly DbConnectionStringBuilder builder;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="AzureManagedConnectionString"/> class.
+        /// </summary>
+        /// <param name="connectionString">A connection string for an Azure-managed durable task service.</param>
+        public AzureManagedConnectionString(string connectionString)
+        {
+            this.builder = new DbConnectionStringBuilder { ConnectionString = connectionString };
+        }
+
+        /// <summary>
+        /// Gets the authentication method specified in the connection string (if any).
+        /// </summary>
+        public string Authentication => this.GetValue("Authentication");
+
+        /// <summary>
+        /// Gets the managed identity or workload identity client ID specified in the connection string (if any).
+        /// </summary>
+        public string ClientId => this.GetValue("ClientID");
+
+        /// <summary>
+        /// Gets the endpoint specified in the connection string (if any).
+        /// </summary>
+        public string Endpoint => this.GetValue("Endpoint");
+
+        /// <summary>
+        /// Gets the task hub name specified in the connection string (if any).
+        /// </summary>
+        public string TaskHubName => this.GetValue("TaskHub");
+
+        private string GetValue(string name)
+        {
+            // Case-insensitive lookup
+            foreach (string key in this.builder.Keys)
+            {
+                if (string.Equals(key, name, System.StringComparison.OrdinalIgnoreCase))
+                {
+                    return this.builder.TryGetValue(key, out object value)
+                        ? value as string
+                        : null;
+                }
+            }
+
+            return null;
+        }
+    }
+}