Remove sidecar dependency and add netcoreapp3.1 TFM (#2157)

* Replaced net60 target with netcoreapp3.1
* Added Functions V2 validation workflows for .NET and Python

Author: cgillum

.github/workflows/smoketest-dotnet-isolated-v4.yml

@@ -1,4 +1,4 @@
-name: Smoke Test - .NET Isolated
+name: Smoke Test - .NET Isolated on Functions V4
 
 on:
   push:
@@ -19,6 +19,6 @@ jobs:
     - uses: actions/checkout@v2
 
       # Validation is blocked on https://github.com/Azure/azure-functions-host/issues/7995
-    - name: Run .NET Isolated Smoke Test
+    - name: Run V4 .NET Isolated Smoke Test
       run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/OOProcSmokeTests/DotNetIsolated/Dockerfile -HttpStartPath api/StartHelloCitiesTyped -NoValidation
       shell: pwsh

.github/workflows/smoketest-dotnet-v2.yml

@@ -0,0 +1,22 @@
+name: Smoke Test - .NET on Functions V2
+
+on:
+  push:
+    branches: [ main, dev ]
+    paths:
+      - 'src/**'
+      - 'test/SmokeTests/SmokeTestsV2/**'
+  pull_request:
+    branches: [ main, dev ]
+    paths:
+      - 'src/**'
+      - 'test/SmokeTests/SmokeTestsV2/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run V2 .NET Smoke Test
+      run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/SmokeTestsV2/Dockerfile -HttpStartPath api/HttpStart
+      shell: pwsh

.github/workflows/smoketest-dotnet-v3.yml

@@ -1,4 +1,4 @@
-name: Smoke Test - .NET V3
+name: Smoke Test - .NET on Functions V3
 
 on:
   push:
@@ -17,6 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Run .NET V3 Smoke Test
+    - name: Run V3 .NET Smoke Test
       run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/SmokeTestsV3/Dockerfile -HttpStartPath api/HttpStart
       shell: pwsh

.github/workflows/smoketest-node14-v4.yml

@@ -1,4 +1,4 @@
-name: Smoke Test - Node14 V4
+name: Smoke Test - Node 14 on Functions V4
 
 on:
   push:
@@ -17,6 +17,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Run Node14 V4 Smoke Test
+    - name: Run V4 Node 14 Smoke Test
       run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/OOProcSmokeTests/durableJS/Dockerfile -HttpStartPath api/DurableFunctionsHttpStart
       shell: pwsh

.github/workflows/smoketest-python37-v2.yml

@@ -0,0 +1,22 @@
+name: Smoke Test - Python 3.7 on Functions V2
+
+on:
+  push:
+    branches: [ main, dev ]
+    paths:
+      - 'src/**'
+      - 'test/SmokeTests/OOProcSmokeTests/durablePy/**'
+  pull_request:
+    branches: [ main, dev ]
+    paths:
+      - 'src/**'
+      - 'test/SmokeTests/OOProcSmokeTests/durablePy/**'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Run V2 Python 3.7 Smoke Test
+      run: test/SmokeTests/e2e-test.ps1 -DockerfilePath test/SmokeTests/OOProcSmokeTests/durablePy/Dockerfile -HttpStartPath api/DurableFunctionsHttpStart -ContainerName pyApp
+      shell: pwsh

WebJobs.Extensions.DurableTask.sln

@@ -24,6 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		azure-pipelines-release.yml = azure-pipelines-release.yml
 		README.md = README.md
 		release_notes.md = release_notes.md
+		.stylecop\stylecop.json = .stylecop\stylecop.json
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VSSampleV1", "test\SmokeTests\SmokeTestsV1\VSSampleV1.csproj", "{78141F66-4DDD-4ABB-9DE3-D0B426FE8110}"
@@ -38,10 +39,17 @@ EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VSSampleV3", "test\SmokeTests\SmokeTestsV3\VSSampleV3.csproj", "{F1611EF4-457B-47CB-94E9-B611EE488984}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "JavaScript", "JavaScript", "{7B2EDAEC-CFD0-404A-8E52-133FFF595C2F}"
+	ProjectSection(SolutionItems) = preProject
+		test\SmokeTests\OOProcSmokeTests\durableJS\Dockerfile = test\SmokeTests\OOProcSmokeTests\durableJS\Dockerfile
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "extensions", "test\SmokeTests\OOProcSmokeTests\durableJS\extensions.csproj", "{9BA6093A-F71A-49F9-A159-B98ACBE41636}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Python", "Python", "{FA53EBC4-8265-4063-B9F8-0D5EA0366B90}"
+	ProjectSection(SolutionItems) = preProject
+		test\SmokeTests\OOProcSmokeTests\durablePy\Dockerfile = test\SmokeTests\OOProcSmokeTests\durablePy\Dockerfile
+		test\SmokeTests\OOProcSmokeTests\durablePy\requirements.txt = test\SmokeTests\OOProcSmokeTests\durablePy\requirements.txt
+	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "extensions", "test\SmokeTests\OOProcSmokeTests\durablePy\extensions.csproj", "{CB481383-8B54-4FCB-8DCB-F348E08C2A9C}"
 EndProject
@@ -77,8 +85,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipelines", "pipelines", "{
 	ProjectSection(SolutionItems) = preProject
 		azure-pipelines.yml = azure-pipelines.yml
 		.github\workflows\smoketest-dotnet-isolated-v4.yml = .github\workflows\smoketest-dotnet-isolated-v4.yml
+		.github\workflows\smoketest-dotnet-v2.yml = .github\workflows\smoketest-dotnet-v2.yml
 		.github\workflows\smoketest-dotnet-v3.yml = .github\workflows\smoketest-dotnet-v3.yml
 		.github\workflows\smoketest-node14-v4.yml = .github\workflows\smoketest-node14-v4.yml
+		.github\workflows\smoketest-python37-v2.yml = .github\workflows\smoketest-python37-v2.yml
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DFPerfScenariosV1", "test\DFPerfScenariosV1\DFPerfScenariosV1.csproj", "{65F904AA-0F6F-48CB-BE19-593B7D68152A}"

release_notes.md

@@ -1,35 +1,16 @@
 ## New Features
-- Azure Durable Functions now supports identity-based connections. See [here](https://docs.microsoft.com/azure/azure-functions/functions-reference?tabs=blob#connecting-to-host-storage-with-an-identity-preview) for details on how to configure these connections ([#2014](https://github.com/Azure/azure-functions-durable-extension/pull/2014)) - contributed by [@wsugarman](https://github.com/wsugarman)
-- `IConnectionStringResolver` has been deprecated in favor of `IConnectionInfoResolver`
-  - Similarly, both `StandardConnectionStringProvider` and `WebJobsConnectionStringProvider` have been deprecated in favor of `StandardConnectionInfoProvider` and `WebJobsConnectionInfoProvider`
-- Initial support for .NET Isolated
-- Added support for long timers for Out-of-Proc SDKs
-- Updated the Replay Schema version to V3 for Out-of-Proc SDKs
 
-## Bug fixes
+None
 
-All bug fixes are coming in transitively via our DTFx dependency updates:
+## Bug fixes
 
-- Fix for stuck orchestration issue caused by incorrect de-dupe flagging (https://github.com/Azure/durabletask/pull/708)
-- Fix for stuck orchestration issue caused by certain partition movement issues (https://github.com/Azure/durabletask/pull/710)
-- Fix for noisy error message in DTFx logs (ArgumentException: A lease ID must be specified when changing a lease) (https://github.com/Azure/durabletask/issues/406)
-- Fix some false positives in deadlock detection that resulted in unnecessary ExecutionEngineExceptions (https://github.com/Azure/durabletask/pull/678)
-- Fix issue related to in-order delivery guarantees for Durable Entities (https://github.com/Azure/durabletask/pull/680)
-- Improved logging for lease/partition management (https://github.com/Azure/durabletask/pull/699)
-- Reduce GC impact of lease blob operations (https://github.com/Azure/durabletask/pull/673)
+None
 
 ## Breaking Changes
 
-See dependency updates below, which can be breaking if there are hard dependency conflicts (like .NET Framework 4.6.1)
+None
 
 ## Dependency Updates
 
-- Added .NET 6 target
-- Added DurableTask.Sidecar v0.3.0 dependency (.NET 6 only), with transitive dependency on Grpc.AspNetCore.Server v2.38
-- Updated minimum C# compiler version to 9.0
-- Added [Microsoft.Extensions.Azure](https://www.nuget.org/packages/Microsoft.Extensions.Azure/1.1.1) v1.1.1 as a dependency for Azure Functions 2.0 and beyond
-- Azure.Identity 1.1.1 -> 1.5.0 for Azure Functions 2.0 and beyond
-- Microsoft.Azure.WebJobs 3.0.14 -> 3.0.31 for for Azure Functions 2.0 and beyond
-- .NET Framework v4.6.1 -> v4.6.2 to stay within official support window
-- [DurableTask.AzureStorage](https://www.nuget.org/packages/Microsoft.Azure.DurableTask.AzureStorage/) 1.10.1 -> 1.11.0
-- [DurableTask.Core](https://www.nuget.org/packages/Microsoft.Azure.DurableTask.Core/) 2.7.0 -> 2.9.0
+- Removed .NET 6 target and added .NET Core 3.1 target
+- Removed DurableTask.Sidecar v0.3.0 dependency

src/WebJobs.Extensions.DurableTask/Bindings/OrchestrationTriggerAttributeBindingProvider.cs

@@ -141,21 +141,21 @@ public Task<ITriggerData> BindAsync(object? value, ValueBindingContext context)
                     var triggerData = new TriggerData(contextValueProvider, bindingData);
                     return Task.FromResult<ITriggerData>(triggerData);
                 }
-#if NET6_0_OR_GREATER
+#if FUNCTIONS_V3_OR_GREATER
                 else if (value is RemoteOrchestratorContext remoteContext)
                 {
                     // Generate a byte array which is the serialized protobuf payload
                     // https://developers.google.com/protocol-buffers/docs/csharptutorial#parsing_and_serialization
                     var orchestratorRequest = new Microsoft.DurableTask.Protobuf.OrchestratorRequest()
                     {
                         InstanceId = remoteContext.InstanceId,
-                        PastEvents = { remoteContext.PastEvents.Select(Microsoft.DurableTask.Sidecar.Grpc.ProtobufUtils.ToHistoryEventProto) },
-                        NewEvents = { remoteContext.NewEvents.Select(Microsoft.DurableTask.Sidecar.Grpc.ProtobufUtils.ToHistoryEventProto) },
+                        PastEvents = { remoteContext.PastEvents.Select(ProtobufUtils.ToHistoryEventProto) },
+                        NewEvents = { remoteContext.NewEvents.Select(ProtobufUtils.ToHistoryEventProto) },
                     };
 
                     // We convert the binary payload into a base64 string because that seems to be the most commonly supported
                     // format for Azure Functions language workers. Attempts to send unencoded byte[] payloads were unsuccessful.
-                    string encodedRequest = Microsoft.DurableTask.Sidecar.Grpc.ProtobufUtils.Base64Encode(orchestratorRequest);
+                    string encodedRequest = ProtobufUtils.Base64Encode(orchestratorRequest);
                     var contextValueProvider = new ObjectValueProvider(encodedRequest, typeof(string));
                     var triggerData = new TriggerData(contextValueProvider, EmptyBindingData);
                     return Task.FromResult<ITriggerData>(triggerData);

src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs

@@ -66,12 +66,12 @@ public class DurableTaskExtension :
             new ConcurrentDictionary<FunctionName, RegisteredFunctionInfo>();
 
         private readonly AsyncLock taskHubLock = new AsyncLock();
-#if !FUNCTIONS_V1
+#if FUNCTIONS_V2_OR_GREATER
 #pragma warning disable CS0169
         private readonly ITelemetryActivator telemetryActivator;
 #pragma warning restore CS0169
 #endif
-#if NET6_0_OR_GREATER
+#if FUNCTIONS_V3_OR_GREATER
         private readonly LocalGrpcListener localGrpcListener;
 #endif
         private readonly bool isOptionsConfigured;
@@ -104,7 +104,6 @@ public DurableTaskExtension()
         }
 #endif
 
-#pragma warning disable CS1572 // XML comment has a param tag for 'XXX', but there is no parameter by that name
         /// <summary>
         /// Initializes a new instance of the <see cref="DurableTaskExtension"/>.
         /// </summary>
@@ -132,7 +131,7 @@ public DurableTaskExtension(
 #pragma warning disable CS0612 // Type or member is obsolete
             IPlatformInformation platformInformationService = null,
 #pragma warning restore CS0612 // Type or member is obsolete
-#if !FUNCTIONS_V1
+#if FUNCTIONS_V2_OR_GREATER
             IErrorSerializerSettingsFactory errorSerializerSettingsFactory = null,
 #pragma warning disable CS0618 // Type or member is obsolete
             IWebHookProvider webhookProvider = null,
@@ -141,7 +140,6 @@ public DurableTaskExtension(
 #else
             IErrorSerializerSettingsFactory errorSerializerSettingsFactory = null)
 #endif
-#pragma warning restore CS1572 // XML comment has a param tag for 'XXX', but there is no parameter by that name
         {
             // Options will be null in Functions v1 runtime - populated later.
             this.Options = options?.Value ?? new DurableTaskOptions();
@@ -198,10 +196,9 @@ public DurableTaskExtension(
             if (runtimeType == WorkerRuntimeType.DotNetIsolated || runtimeType == WorkerRuntimeType.Java)
             {
                 this.OutOfProcProtocol = OutOfProcOrchestrationProtocol.MiddlewarePassthrough;
-#if NET6_0_OR_GREATER
+#if FUNCTIONS_V3_OR_GREATER
                 this.localGrpcListener = new LocalGrpcListener(
                     this,
-                    this.loggerFactory,
                     this.defaultDurabilityProvider,
                     this.defaultDurabilityProvider);
                 this.HostLifetimeService.OnStopped.Register(this.StopLocalGrpcServer);
@@ -435,11 +432,21 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context)
             // Note that the order of the middleware added determines the order in which it executes.
             if (this.OutOfProcProtocol == OutOfProcOrchestrationProtocol.MiddlewarePassthrough)
             {
+#if FUNCTIONS_V3_OR_GREATER
                 // This is a newer, more performant flavor of orchestration/activity middleware that is being
                 // enabled for newer language runtimes. Support for entities in this model is TBD.
                 var ooprocMiddleware = new OutOfProcMiddleware(this);
                 this.taskHubWorker.AddActivityDispatcherMiddleware(ooprocMiddleware.CallActivityAsync);
                 this.taskHubWorker.AddOrchestrationDispatcherMiddleware(ooprocMiddleware.CallOrchestratorAsync);
+#else
+                // This can happen if, for example, a Java user tries to use Durable Functions while targeting V2 or V3 extension bundles
+                // because those bundles target .NET Core 2.2, which doesn't support the gRPC libraries used in the modern out-of-proc implementation.
+                throw new PlatformNotSupportedException(
+                    "This project type is not supported on this version of the Azure Functions runtime. Please upgrade to Azure Functions V3 or higher. " +
+                    "If you are using a language that supports extension bundles, please use extension bundles V4 or higher. " +
+                    "For more information on Azure Functions versions, see https://docs.microsoft.com/azure/azure-functions/functions-versions. " +
+                    "For more information on extension bundles, see https://docs.microsoft.com/azure/azure-functions/functions-bindings-register#extension-bundles.");
+#endif
             }
             else
             {
@@ -450,22 +457,25 @@ void IExtensionConfigProvider.Initialize(ExtensionConfigContext context)
                 this.taskHubWorker.AddOrchestrationDispatcherMiddleware(this.OrchestrationMiddleware);
             }
 
-#if !FUNCTIONS_V1
             // The RPC server needs to be started sometime before any functions can be triggered
             // and this is the latest point in the pipeline available to us.
-            this.StartLocalHttpServer();
-#endif
-#if NET6_0_OR_GREATER
+#if FUNCTIONS_V3_OR_GREATER
             if (this.OutOfProcProtocol == OutOfProcOrchestrationProtocol.MiddlewarePassthrough)
             {
                 this.StartLocalGrpcServer();
             }
+#endif
+#if FUNCTIONS_V2_OR_GREATER
+            if (this.OutOfProcProtocol == OutOfProcOrchestrationProtocol.OrchestratorShim)
+            {
+                this.StartLocalHttpServer();
+            }
 #endif
         }
 
         internal string GetLocalRpcAddress()
         {
-#if NET6_0_OR_GREATER
+#if FUNCTIONS_V3_OR_GREATER
             if (this.OutOfProcProtocol == OutOfProcOrchestrationProtocol.MiddlewarePassthrough)
             {
                 return this.localGrpcListener.ListenAddress;
@@ -520,7 +530,7 @@ public void Dispose()
             this.eventSourceListener?.Dispose();
         }
 
-#if !FUNCTIONS_V1
+#if FUNCTIONS_V2_OR_GREATER
         private void StartLocalHttpServer()
         {
             bool? shouldEnable = this.Options.LocalRpcEndpointEnabled;
@@ -557,7 +567,7 @@ private void StopLocalHttpServer()
         }
 #endif
 
-#if NET6_0_OR_GREATER
+#if FUNCTIONS_V3_OR_GREATER
         private void StartLocalGrpcServer()
         {
             this.localGrpcListener.StartAsync().GetAwaiter().GetResult();

src/WebJobs.Extensions.DurableTask/DurableTaskJobHostConfigurationExtensions.cs

@@ -12,7 +12,6 @@
 using Microsoft.Extensions.Azure;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.DependencyInjection.Extensions;
-using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Options;
 #else
 using Microsoft.Azure.WebJobs.Extensions.DurableTask.ContextImplementations;
@@ -181,5 +180,5 @@ public static void UseDurableTask(
             extensions.RegisterExtension<IExtensionConfigProvider>(listenerConfig);
         }
 #endif
-        }
     }
+}