Add Function Execution Retry Support (#6706)

pragnagopa · web-flow · commit 4b292b3371b8 · 2020-09-30T14:57:23.000-07:00
diff --git a/WebJobs.Script.sln b/WebJobs.Script.sln
@@ -320,6 +320,29 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger", "HttpTrigger"
 		sample\CustomHandler\HttpTrigger\function.json = sample\CustomHandler\HttpTrigger\function.json
 	EndProjectSection
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger-Retry", "HttpTrigger-Retry", "{821D5B92-2C3E-44F0-AA92-8B996DCB8E6C}"
+	ProjectSection(SolutionItems) = preProject
+		sample\Node\HttpTrigger-Retry\function.json = sample\Node\HttpTrigger-Retry\function.json
+		sample\Node\HttpTrigger-Retry\index.js = sample\Node\HttpTrigger-Retry\index.js
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "NodeRetry", "NodeRetry", "{EEBAC197-FAD8-4214-9A12-76334BB3021A}"
+	ProjectSection(SolutionItems) = preProject
+		sample\NodeRetry\host.json = sample\NodeRetry\host.json
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger-FunctionJsonRetry", "HttpTrigger-FunctionJsonRetry", "{7935A7A4-191A-4A9B-B8DD-173968309EBF}"
+	ProjectSection(SolutionItems) = preProject
+		sample\NodeRetry\HttpTrigger-RetryFunctionJson\function.json = sample\NodeRetry\HttpTrigger-RetryFunctionJson\function.json
+		sample\NodeRetry\HttpTrigger-RetryFunctionJson\index.js = sample\NodeRetry\HttpTrigger-RetryFunctionJson\index.js
+	EndProjectSection
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger-HostJsonRetry", "HttpTrigger-HostJsonRetry", "{CF6D9CDA-2290-46DF-B162-2D422477288A}"
+	ProjectSection(SolutionItems) = preProject
+		sample\NodeRetry\HttpTrigger-RetryHostJson\function.json = sample\NodeRetry\HttpTrigger-RetryHostJson\function.json
+		sample\NodeRetry\HttpTrigger-RetryHostJson\index.js = sample\NodeRetry\HttpTrigger-RetryHostJson\index.js
+	EndProjectSection
+EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
 		test\WebJobs.Script.Tests.Shared\WebJobs.Script.Tests.Shared.projitems*{35c9ccb7-d8b6-4161-bb0d-bcfa7c6dcffb}*SharedItemsImports = 13
@@ -414,6 +437,10 @@ Global
 		{BA45A727-34B7-484F-9B93-B1755AF09A2A} = {0AE3CE25-4CD9-4769-AE58-399FC59CF70F}
 		{A59D3F65-53E5-4666-AEFE-882987A39A49} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
 		{209DC34B-762E-4B1B-B094-EBD7C4B972C2} = {A59D3F65-53E5-4666-AEFE-882987A39A49}
+		{821D5B92-2C3E-44F0-AA92-8B996DCB8E6C} = {9D87C796-7914-4A43-B843-579562393E10}
+		{EEBAC197-FAD8-4214-9A12-76334BB3021A} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
+		{7935A7A4-191A-4A9B-B8DD-173968309EBF} = {EEBAC197-FAD8-4214-9A12-76334BB3021A}
+		{CF6D9CDA-2290-46DF-B162-2D422477288A} = {EEBAC197-FAD8-4214-9A12-76334BB3021A}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {85400884-5FFD-4C27-A571-58CB3C8CAAC5}
diff --git a/release_notes.md b/release_notes.md
@@ -8,6 +8,7 @@
 - Add a new host.json property "watchFiles" for restarting the Host when files are modified.
 - Update Java Worker to 1.8.0 [Release Note](https://github.com/Azure/azure-functions-java-worker/releases/tag/1.8.0)
 - Update Java Library to 1.4.0 [Release Note](https://github.com/Azure/azure-functions-java-library)
+- Added support for function execution retry on invocation failures [#6664](https://github.com/Azure/azure-functions-host/issues/6664)
 - **[BreakingChange]** Fixes [#400](https://github.com/Azure/azure-functions-java-worker/issues/400) which was a regression from the 1.7.1 release.
    There is potential of impact if the function code has taken a dependency on a feature in gson 2.8.6 as the dependency `gson-2.8.5.jar` is now included in the class path of the worker and will take precedence over the function's lib folder.
 - **Breaking Changes in CustomHandler**
@@ -30,4 +31,7 @@
     If you are using these properties, please ensure your app is able to detect and handle the new schema.
 
 **Release sprint:** Sprint 84
-[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Afeature+is%3Aclosed) ]
+[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+84%22+label%3Afeature+is%3Aclosed) ]
+
+**Release sprint:** Sprint 85
+[ [bugs](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+85%22+label%3Abug+is%3Aclosed) | [features](https://github.com/Azure/azure-functions-host/issues?q=is%3Aissue+milestone%3A%22Functions+Sprint+85%22+label%3Afeature+is%3Aclosed) ]
diff --git a/sample/NodeRetry/HttpTrigger-RetryFunctionJson/function.json b/sample/NodeRetry/HttpTrigger-RetryFunctionJson/function.json
@@ -0,0 +1,23 @@
+﻿{
+  "bindings": [
+    {
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ],
+  "retry": {
+    "strategy": "fixedDelay",
+    "maxRetryCount": 4,
+    "delayInterval": "00:00:03"
+  }
+}
diff --git a/sample/NodeRetry/HttpTrigger-RetryFunctionJson/index.js b/sample/NodeRetry/HttpTrigger-RetryFunctionJson/index.js
@@ -0,0 +1,18 @@
+﻿var invocationCount = 0;
+
+module.exports = async function (context, req) {
+    const reset = req.query.reset;
+    invocationCount = reset ? 0 : invocationCount
+
+    context.log('JavaScript HTTP trigger function processed a request.invocationCount: ' + invocationCount);
+
+    invocationCount = invocationCount + 1;
+    const responseMessage = "invocationCount: " + invocationCount;
+    if (invocationCount < 4) {
+        throw new Error('An error occurred');
+    }
+    context.res = {
+        // status: 200, /* Defaults to 200 */
+        body: responseMessage
+    };
+}
diff --git a/sample/NodeRetry/HttpTrigger-RetryHostJson/function.json b/sample/NodeRetry/HttpTrigger-RetryHostJson/function.json
@@ -0,0 +1,18 @@
+{
+  "bindings": [
+    {
+      "type": "httpTrigger",
+      "direction": "in",
+      "name": "req",
+      "methods": [
+        "get",
+        "post"
+      ]
+    },
+    {
+      "type": "http",
+      "direction": "out",
+      "name": "res"
+    }
+  ]
+}
diff --git a/sample/NodeRetry/HttpTrigger-RetryHostJson/index.js b/sample/NodeRetry/HttpTrigger-RetryHostJson/index.js
@@ -0,0 +1,18 @@
+﻿var invocationCount = 0;
+
+module.exports = async function (context, req) {
+    const reset = req.query.reset;
+    invocationCount = reset ? 0 : invocationCount
+
+    context.log('JavaScript HTTP trigger function processed a request.invocationCount: ' + invocationCount);
+    
+    invocationCount = invocationCount + 1;
+    const responseMessage = "invocationCount: " + invocationCount;
+    if (invocationCount < 2) {
+        throw new Error('An error occurred');
+    }
+    context.res = {
+        // status: 200, /* Defaults to 200 */
+        body: responseMessage
+    };
+}
diff --git a/sample/NodeRetry/host.json b/sample/NodeRetry/host.json
@@ -0,0 +1,8 @@
+{
+  "version": "2.0",
+  "retry": {
+    "strategy": "fixedDelay",
+    "maxRetryCount": 2,
+    "delayInterval": "00:00:03"
+  }
+}
diff --git a/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj b/src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
@@ -59,8 +59,8 @@
     <PackageReference Include="Microsoft.Azure.AppService.Proxy.Client" Version="2.0.8100001-0126c21e" />
     <PackageReference Include="Microsoft.Azure.Services.AppAuthentication" Version="1.0.3" />
     <PackageReference Include="Microsoft.Azure.Storage.File" Version="11.1.7" />
-    <PackageReference Include="Microsoft.Azure.WebJobs.Host.Storage" Version="4.0.1" />    
-    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.22" />
+    <PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.23-11785" />
+    <PackageReference Include="Microsoft.Azure.WebJobs.Host.Storage" Version="4.0.1" />
     <PackageReference Include="Microsoft.Azure.KeyVault" Version="3.0.3" />
     <PackageReference Include="Microsoft.Azure.WebJobs.Logging" Version="4.0.1" />
     <PackageReference Include="Microsoft.Azure.WebSites.DataProtection" Version="2.1.91-alpha" />
diff --git a/src/WebJobs.Script/Config/ConfigurationSectionNames.cs b/src/WebJobs.Script/Config/ConfigurationSectionNames.cs
@@ -21,5 +21,6 @@ public static class ConfigurationSectionNames
         public const string Hsts = Http + ":hsts";
         public const string CustomHttpHeaders = Http + ":customHeaders";
         public const string EasyAuth = "easyauth";
+        public const string Retry = "retry";
     }
 }
diff --git a/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs b/src/WebJobs.Script/Config/HostJsonFileConfigurationSource.cs
@@ -49,7 +49,7 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider
         {
             private static readonly string[] WellKnownHostJsonProperties = new[]
             {
-                "version", "functionTimeout", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",
+                "version", "functionTimeout", "retry", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",
                 "eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies",
                 "customHandler", "httpWorker"
             };
diff --git a/src/WebJobs.Script/Config/ScriptHostOptionsSetup.cs b/src/WebJobs.Script/Config/ScriptHostOptionsSetup.cs
@@ -51,6 +51,7 @@ public void Configure(ScriptJobHostOptions options)
                 {
                     options.FileLoggingMode = fileLoggingMode.Value;
                 }
+                Utility.ValidateRetryOptions(options.Retry);
             }
 
             // FunctionTimeout
diff --git a/src/WebJobs.Script/Config/ScriptJobHostOptions.cs b/src/WebJobs.Script/Config/ScriptJobHostOptions.cs
@@ -6,6 +6,7 @@
 using System.Collections.Immutable;
 using System.Collections.ObjectModel;
 using System.IO;
+using Microsoft.Azure.WebJobs.Script.Description;
 
 namespace Microsoft.Azure.WebJobs.Script
 {
@@ -107,5 +108,10 @@ public string RootScriptPath
         /// locally or via CLI.
         /// </summary>
         public bool IsSelfHost { get; set; }
+
+        /// <summary>
+        /// Gets or sets retry options to use on function executions on function invocation failures.
+        /// </summary>
+        public RetryOptions Retry { get; set; }
     }
 }
diff --git a/src/WebJobs.Script/CustomAttributeBuilderUtility.cs b/src/WebJobs.Script/CustomAttributeBuilderUtility.cs
@@ -0,0 +1,61 @@
+﻿// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Reflection;
+using System.Reflection.Emit;
+using Microsoft.Azure.WebJobs.Host;
+using Microsoft.Azure.WebJobs.Script.Description;
+
+namespace Microsoft.Azure.WebJobs.Script
+{
+    internal static class CustomAttributeBuilderUtility
+    {
+        internal static CustomAttributeBuilder GetRetryCustomAttributeBuilder(RetryOptions functionRetry)
+        {
+            switch (functionRetry.Strategy)
+            {
+                case RetryStrategy.FixedDelay:
+                    Type fixedDelayRetryType = typeof(FixedDelayRetryAttribute);
+                    ConstructorInfo fixedDelayRetryCtorInfo = fixedDelayRetryType.GetConstructor(new[] { typeof(int), typeof(string) });
+                    CustomAttributeBuilder fixedDelayRetryBuilder = new CustomAttributeBuilder(
+                    fixedDelayRetryCtorInfo,
+                    new object[] { functionRetry.MaxRetryCount.Value, functionRetry.DelayInterval.ToString() });
+                    return fixedDelayRetryBuilder;
+                case RetryStrategy.ExponentialBackoff:
+                    Type exponentialBackoffRetryType = typeof(ExponentialBackoffRetryAttribute);
+                    ConstructorInfo exponentialBackoffDelayRetryCtorInfo = exponentialBackoffRetryType.GetConstructor(new[] { typeof(int), typeof(string), typeof(string) });
+                    CustomAttributeBuilder exponentialBackoffRetryBuilder = new CustomAttributeBuilder(
+                    exponentialBackoffDelayRetryCtorInfo,
+                    new object[] { functionRetry.MaxRetryCount.Value, functionRetry.MinimumInterval.ToString(), functionRetry.MaximumInterval.ToString() });
+                    return exponentialBackoffRetryBuilder;
+            }
+            return null;
+        }
+
+        internal static CustomAttributeBuilder GetTimeoutCustomAttributeBuilder(TimeSpan functionTimeout)
+        {
+            Type timeoutType = typeof(TimeoutAttribute);
+            ConstructorInfo ctorInfo = timeoutType.GetConstructor(new[] { typeof(string) });
+
+            PropertyInfo[] propertyInfos = new[]
+            {
+                timeoutType.GetProperty("ThrowOnTimeout"),
+                timeoutType.GetProperty("TimeoutWhileDebugging")
+            };
+
+            // Hard-code these for now. Eventually elevate to config
+            object[] propertyValues = new object[]
+            {
+                true,
+                true
+            };
+
+            return new CustomAttributeBuilder(
+                ctorInfo,
+                new object[] { functionTimeout.ToString() },
+                propertyInfos,
+                propertyValues);
+        }
+    }
+}
diff --git a/src/WebJobs.Script/Description/FunctionDescriptorProvider.cs b/src/WebJobs.Script/Description/FunctionDescriptorProvider.cs
@@ -11,7 +11,6 @@
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Script.Binding;
-using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Extensibility;
 using Microsoft.Extensions.Logging;
 using Newtonsoft.Json.Linq;
@@ -226,6 +225,16 @@ protected static void ApplyMethodLevelAttributes(FunctionMetadata functionMetada
                 CustomAttributeBuilder attributeBuilder = new CustomAttributeBuilder(ctorInfo, new object[0]);
                 methodAttributes.Add(attributeBuilder);
             }
+
+            // apply the retry settings from function.json
+            if (functionMetadata.Retry != null)
+            {
+                CustomAttributeBuilder retryCustomAttributeBuilder = CustomAttributeBuilderUtility.GetRetryCustomAttributeBuilder(functionMetadata.Retry);
+                if (retryCustomAttributeBuilder != null)
+                {
+                    methodAttributes.Add(retryCustomAttributeBuilder);
+                }
+            }
         }
 
         protected abstract IFunctionInvoker CreateFunctionInvoker(string scriptFilePath, BindingMetadata triggerMetadata, FunctionMetadata functionMetadata, Collection<FunctionBinding> inputBindings, Collection<FunctionBinding> outputBindings);
diff --git a/src/WebJobs.Script/Host/FunctionMetadataProvider.cs b/src/WebJobs.Script/Host/FunctionMetadataProvider.cs
@@ -8,6 +8,7 @@
 using System.IO;
 using System.IO.Abstractions;
 using System.Linq;
+using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Diagnostics.Extensions;
@@ -159,6 +160,11 @@ private FunctionMetadata ParseFunctionMetadata(string functionName, JObject conf
                 functionMetadata.Language = ParseLanguage(functionMetadata.ScriptFile, _workerConfigs);
             }
             functionMetadata.EntryPoint = (string)configMetadata["entryPoint"];
+
+            //Retry
+            functionMetadata.Retry = configMetadata.Property(ConfigurationSectionNames.Retry)?.Value?.ToObject<RetryOptions>();
+            Utility.ValidateRetryOptions(functionMetadata.Retry);
+
             return functionMetadata;
         }
 
diff --git a/src/WebJobs.Script/Host/ScriptHost.cs b/src/WebJobs.Script/Host/ScriptHost.cs
@@ -356,30 +356,19 @@ internal static Collection<CustomAttributeBuilder> CreateTypeAttributes(ScriptJo
             // apply the timeout settings to our type
             if (scriptConfig.FunctionTimeout != null)
             {
-                Type timeoutType = typeof(TimeoutAttribute);
-                ConstructorInfo ctorInfo = timeoutType.GetConstructor(new[] { typeof(string) });
-
-                PropertyInfo[] propertyInfos = new[]
-                {
-                    timeoutType.GetProperty("ThrowOnTimeout"),
-                    timeoutType.GetProperty("TimeoutWhileDebugging")
-                };
-
-                // Hard-code these for now. Eventually elevate to config
-                object[] propertyValues = new object[]
-                {
-                    true,
-                    true
-                };
-
-                CustomAttributeBuilder timeoutBuilder = new CustomAttributeBuilder(
-                    ctorInfo,
-                    new object[] { scriptConfig.FunctionTimeout.ToString() },
-                    propertyInfos,
-                    propertyValues);
-
+                var timeoutBuilder = CustomAttributeBuilderUtility.GetTimeoutCustomAttributeBuilder(scriptConfig.FunctionTimeout.Value);
                 customAttributes.Add(timeoutBuilder);
             }
+            // apply retry settings for function execution
+            if (scriptConfig.Retry != null)
+            {
+                // apply the retry settings from host.json
+                var retryCustomAttributeBuilder = CustomAttributeBuilderUtility.GetRetryCustomAttributeBuilder(scriptConfig.Retry);
+                if (retryCustomAttributeBuilder != null)
+                {
+                    customAttributes.Add(retryCustomAttributeBuilder);
+                }
+            }
 
             return customAttributes;
         }
diff --git a/src/WebJobs.Script/Utility.cs b/src/WebJobs.Script/Utility.cs
diff --git a/src/WebJobs.Script/WebJobs.Script.csproj b/src/WebJobs.Script/WebJobs.Script.csproj
diff --git a/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_Node_Retry.cs b/test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_Node_Retry.cs
diff --git a/test/WebJobs.Script.Tests/Configuration/ScriptHostOptionsSetupTests.cs b/test/WebJobs.Script.Tests/Configuration/ScriptHostOptionsSetupTests.cs
diff --git a/test/WebJobs.Script.Tests/FunctionMetadataProviderTests.cs b/test/WebJobs.Script.Tests/FunctionMetadataProviderTests.cs

Original file line number	Diff line number	Diff line change
`@@ -21,5 +21,6 @@ public static class ConfigurationSectionNames`
`21`	`21`	`public const string Hsts = Http + ":hsts";`
`22`	`22`	`public const string CustomHttpHeaders = Http + ":customHeaders";`
`23`	`23`	`public const string EasyAuth = "easyauth";`
	`24`	`+ public const string Retry = "retry";`
`24`	`25`	`}`
`25`	`26`	`}`
Original file line number	Diff line number	Diff line change
`@@ -49,7 +49,7 @@ public class HostJsonFileConfigurationProvider : ConfigurationProvider`
`49`	`49`	`{`
`50`	`50`	`private static readonly string[] WellKnownHostJsonProperties = new[]`
`51`	`51`	`{`
`52`		`- "version", "functionTimeout", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",`
	`52`	`+ "version", "functionTimeout", "retry", "functions", "http", "watchDirectories", "watchFiles", "queues", "serviceBus",`
`53`	`53`	`"eventHub", "singleton", "logging", "aggregator", "healthMonitor", "extensionBundle", "managedDependencies",`
`54`	`54`	`"customHandler", "httpWorker"`
`55`	`55`	`};`