Lightweight scale controller package for Mssql backend

.github/workflows/validate-build-scale.yml

@@ -1,4 +1,4 @@
-name: Functions Scale Tests Azure Storage 
+name: Functions Scale Tests
 permissions:
   contents: read
 
@@ -19,7 +19,8 @@ env:
   AzureWebJobsStorage: UseDevelopmentStorage=true
 
 jobs:
-  build:
+  azure-storage:
+    name: Functions Scale Tests - Azure Storage
     runs-on: ubuntu-latest
 
     steps:
@@ -62,5 +63,71 @@ jobs:
           sleep 1
         done
 
-        # Run tests
-        dotnet test ./test/ScaleTests/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.Tests.csproj -c $config --no-build --verbosity normal
+        # Run tests (exclude AzureManaged and SqlServer tests)
+        dotnet test ./test/ScaleTests/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.Tests.csproj -c $config --no-build --verbosity normal --filter "FullyQualifiedName!~AzureManaged&FullyQualifiedName!~SqlServer"
+
+  mssql:
+    name: Functions Scale Tests - MSSQL
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v3
+        with:
+          global-json-file: global.json
+
+      - name: Set up Node.js (needed for Azurite)
+        uses: actions/setup-node@v3
+        with:
+          node-version: '18.x'
+
+      - name: Install Azurite
+        run: npm install -g azurite
+
+      - name: Start Azurite
+        run: azurite --silent --blobPort 10000 --queuePort 10001 --tablePort 10002 &
+
+      - name: Initialize Environment Variables
+        run: |
+          MSSQL_SA_PASSWORD="TEST12_${RANDOM}!"
+          echo "MSSQL_SA_PASSWORD=$MSSQL_SA_PASSWORD" >> $GITHUB_ENV
+          echo "SQLDB_Connection=Server=localhost,1433;Database=DurableDB;User Id=sa;Password=$MSSQL_SA_PASSWORD;TrustServerCertificate=True;Encrypt=False;" >> $GITHUB_ENV
+
+      - name: Pull SQL Server Docker Image
+        run: docker pull mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Start SQL Server Container
+        run: |
+          docker run --name mssql-server \
+            -e ACCEPT_EULA=Y \
+            -e "MSSQL_SA_PASSWORD=$MSSQL_SA_PASSWORD" \
+            -e "MSSQL_PID=Express" \
+            -p 1433:1433 \
+            -d mcr.microsoft.com/mssql/server:2022-latest
+
+      - name: Wait for SQL Server to be ready
+        run: |
+          echo "Waiting for SQL Server to be ready..."
+          for i in {1..30}; do
+            if docker exec mssql-server /opt/mssql-tools*/bin/sqlcmd -S localhost -U sa -P "$MSSQL_SA_PASSWORD" -Q "SELECT 1" > /dev/null 2>&1; then
+              echo "SQL Server is ready"
+              break
+            fi
+            echo "SQL Server is not ready, waiting ($i/30)"
+            sleep 2
+          done
+          docker ps
+
+      - name: Restore and Build Scale Tests
+        working-directory: test/ScaleTests
+        run: dotnet build --configuration Release
+
+      - name: Run SQL Server Scale Tests
+        working-directory: test/ScaleTests
+        env:
+          AzureWebJobsStorage: UseDevelopmentStorage=true
+        run: dotnet test --configuration Release --no-build --verbosity normal --filter "FullyQualifiedName~SqlServer"

src/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale/DurableTaskScaleConfigurationExtensions.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Linq;
 using Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.AzureStorage;
+using Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.Sql;
 using Microsoft.Azure.WebJobs.Host.Scale;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
@@ -42,7 +43,7 @@ public static Microsoft.Azure.WebJobs.IWebJobsBuilder AddDurableTask(this IWebJo
             serviceCollection.TryAddSingleton<IStorageServiceClientProviderFactory, StorageServiceClientProviderFactory>();
 
             serviceCollection.AddSingleton<IScalabilityProviderFactory, AzureStorageScalabilityProviderFactory>();
-
+            serviceCollection.AddSingleton<IScalabilityProviderFactory, SqlServerScalabilityProviderFactory>();
             return builder;
         }
 

src/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.csproj

@@ -51,6 +51,7 @@
   <ItemGroup>
     <PackageReference Include="Microsoft.Azure.WebJobs" />
     <PackageReference Include="Microsoft.Azure.DurableTask.AzureStorage" />
+	<PackageReference Include="Microsoft.DurableTask.SqlServer" />
     <PackageReference Include="Azure.Identity" />
     <PackageReference Include="Microsoft.Extensions.Azure" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" PrivateAssets="All" />

src/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale/Sql/SqlServerMetricsProvider.cs

@@ -0,0 +1,61 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using DurableTask.SqlServer;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.Sql
+{
+    /// <summary>
+    /// Metrics provider for SQL Server backend scaling.
+    /// Provides recommended replica count based on SQL Server orchestration service metrics.
+    /// </summary>
+    public class SqlServerMetricsProvider
+    {
+        private readonly SqlOrchestrationService service;
+        private DateTime metricsTimeStamp = DateTime.MinValue;
+        private SqlServerScaleMetric metrics;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SqlServerMetricsProvider"/> class that
+        /// retrieves scaling metrics from the specified SQL orchestration service.
+        /// </summary>
+        /// <param name="service">The SQL orchestration service used to get metrics.</param>
+        /// <exception cref="ArgumentNullException">
+        /// Thrown if <paramref name="service"/> is null.
+        /// </exception>
+        public SqlServerMetricsProvider(SqlOrchestrationService service)
+        {
+            this.service = service ?? throw new ArgumentNullException(nameof(service));
+        }
+
+        /// <summary>
+        /// Gets the latest SQL Server scaling metrics, including the recommended worker count. Results are cached for 5 seconds to reduce query load.
+        /// </summary>
+        /// <param name="previousWorkerCount">
+        /// The previous number of workers, used to compare scaling recommendations (optional).
+        /// </param>
+        /// <returns>
+        /// A <see cref="SqlServerScaleMetric"/> containing the recommended worker count.
+        /// </returns>
+        public virtual async Task<SqlServerScaleMetric> GetMetricsAsync(int? previousWorkerCount = null)
+        {
+            // We only want to query the metrics every 5 seconds to avoid excessive SQL queries.
+            if (this.metrics == null || DateTime.UtcNow >= this.metricsTimeStamp.AddSeconds(5))
+            {
+                // GetRecommendedReplicaCountAsync will write a trace if the recommendation results
+                // in a worker count that is different from the worker count we pass in as an argument.
+                int recommendedReplicaCount = await this.service.GetRecommendedReplicaCountAsync(
+                    previousWorkerCount,
+                    CancellationToken.None);
+
+                this.metricsTimeStamp = DateTime.UtcNow;
+                this.metrics = new SqlServerScaleMetric { RecommendedReplicaCount = recommendedReplicaCount };
+            }
+
+            return this.metrics;
+        }
+    }
+}

src/Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale/Sql/SqlServerScalabilityProvider.cs

@@ -0,0 +1,105 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See LICENSE in the project root for license information.
+
+using System;
+using DurableTask.SqlServer;
+using Microsoft.Azure.WebJobs.Host.Scale;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.FunctionsScale.Sql
+{
+    /// <summary>
+    /// The SQL Server implementation of ScalabilityProvider.
+    /// Provides scale monitoring and target-based scaling for SQL Server backend.
+    /// </summary>
+    public class SqlServerScalabilityProvider : ScalabilityProvider
+    {
+        private readonly SqlOrchestrationService service;
+        private readonly string connectionName;
+        private readonly ILogger logger;
+
+        private readonly object initLock = new object();
+        private SqlServerMetricsProvider singletonSqlMetricsProvider;
+
+        /// <summary>
+        /// Initializes a new instance of the <see cref="SqlServerScalabilityProvider"/> class.
+        /// for managing scaling operations using a SQL Server–based orchestration service.
+        /// </summary>
+        /// <param name="service">The SQL orchestration service instance.</param>
+        /// <param name="connectionName">The name of the SQL connection.</param>
+        /// <param name="logger">The logger used for diagnostic output.</param>
+        /// <exception cref="ArgumentNullException">
+        /// Thrown if <paramref name="service"/> is null.
+        /// </exception>
+        public SqlServerScalabilityProvider(
+            SqlOrchestrationService service,
+            string connectionName,
+            ILogger logger)
+            : base("mssql", connectionName)
+        {
+            this.service = service ?? throw new ArgumentNullException(nameof(service));
+            this.connectionName = connectionName;
+            this.logger = logger;
+        }
+
+        /// <summary>
+        /// Gets the app setting containing the SQL Server connection string.
+        /// </summary>
+        public override string ConnectionName => this.connectionName;
+
+        /// <inheritdoc/>
+        public override bool TryGetScaleMonitor(
+            string functionId,
+            string functionName,
+            string hubName,
+            string targetConnectionName,
+            out IScaleMonitor scaleMonitor)
+        {
+            lock (this.initLock)
+            {
+                if (this.singletonSqlMetricsProvider == null)
+                {
+                    // This is only called by the ScaleController, it doesn't run in the Functions Host process.
+                    this.singletonSqlMetricsProvider = this.GetMetricsProvider(
+                        hubName,
+                        this.service,
+                        this.logger);
+                }
+
+                scaleMonitor = new SqlServerScaleMonitor(functionId, hubName, this.singletonSqlMetricsProvider);
+                return true;
+            }
+        }
+
+        /// <inheritdoc/>
+        public override bool TryGetTargetScaler(
+            string functionId,
+            string functionName,
+            string hubName,
+            string targetConnectionName,
+            out ITargetScaler targetScaler)
+        {
+            lock (this.initLock)
+            {
+                if (this.singletonSqlMetricsProvider == null)
+                {
+                    this.singletonSqlMetricsProvider = this.GetMetricsProvider(
+                        hubName,
+                        this.service,
+                        this.logger);
+                }
+
+                targetScaler = new SqlServerTargetScaler(functionId, this.singletonSqlMetricsProvider);
+                return true;
+            }
+        }
+
+        internal SqlServerMetricsProvider GetMetricsProvider(
+            string hubName,
+            SqlOrchestrationService sqlOrchestrationService,
+            ILogger logger)
+        {
+            return new SqlServerMetricsProvider(sqlOrchestrationService);
+        }
+    }
+}