[AC-220] - Implements retry feature with Polly

docker/.env.template.mssql

@@ -4,6 +4,7 @@ PATHBASE=""
 OVERRIDE_EXISTING_DATABASE=false
 IGNORES_CERTIFICATE_ERRORS=true
 SQLSERVER_BAK_FILE="./Ods_Minimal_Template.bak"
+MAX_RETRY_ATTEMPTS=5
 EDFI_MASTER="Data Source=host.docker.internal;Initial Catalog=master;User ID=sa;Password=1StrongPwd!!;Trusted_Connection=false;Encrypt=True;TrustServerCertificate=True;Persist Security Info=True;"
 EDFI_ODS="Data Source=host.docker.internal;Initial Catalog={0};User ID=sa;Password=1StrongPwd!!;Trusted_Connection=false;Encrypt=True;TrustServerCertificate=True;Persist Security Info=True;"
 ACCESS_TOKEN_URL="https://host.docker.internal/auth/realms/edfi-admin-console/protocol/openid-connect/token"

docker/.env.template.pgsql

@@ -3,6 +3,7 @@ DATABASE_ENGINE=PostgreSQL
 PATHBASE=""
 OVERRIDE_EXISTING_DATABASE=false
 IGNORES_CERTIFICATE_ERRORS=true
+MAX_RETRY_ATTEMPTS=5
 EDFI_MASTER="host=host.docker.internal;port=5432;username=postgres;password=admin;database=postgres;pooling=false"
 EDFI_ODS="host=host.docker.internal;port=5432;username=postgres;password=admin;database={0};pooling=false"
 ACCESS_TOKEN_URL="https://host.docker.internal/auth/realms/edfi-admin-console/protocol/openid-connect/token"

docker/mssql/instance-management-svc.yml

@@ -17,6 +17,7 @@ services:
       EdFi_AdminConsole_AppSettings__OverrideExistingDatabase: ${OVERRIDE_EXISTING_DATABASE:-false}
       EdFi_AdminConsole_AppSettings__IgnoresCertificateErrors: ${IGNORES_CERTIFICATE_ERRORS:-true}
       EdFi_AdminConsole_AppSettings__SqlServerBakFile: ${SQLSERVER_BAK_FILE:-"./Ods_Minimal_Template.bak"}
+      EdFi_AdminConsole_AppSettings__MaxRetryAttempts: ${MAX_RETRY_ATTEMPTS:-5}
       EdFi_AdminConsole_ConnectionStrings__EdFi_Master: ${EDFI_MASTER:-"Data Source=host.docker.internal;Initial Catalog=master;User ID=sa;Password=1StrongPwd!!;Trusted_Connection=false;Encrypt=True;TrustServerCertificate=True;Persist Security Info=True;"}
       EdFi_AdminConsole_ConnectionStrings__EdFi_Ods: ${EDFI_ODS:-"Data Source=host.docker.internal;Initial Catalog={0};User ID=sa;Password=1StrongPwd!!;Trusted_Connection=false;Encrypt=True;TrustServerCertificate=True;Persist Security Info=True;"}
       EdFi_AdminConsole_AdminApiSettings__AccessTokenUrl: ${ACCESS_TOKEN_URL:-"https://host.docker.internal/auth/realms/edfi-admin-console/protocol/openid-connect/token"}

docker/pgsql/instance-management-svc.yml

@@ -15,6 +15,7 @@ services:
       EdFi_AdminConsole_AppSettings__DefaultPageSizeLimit: 25
       EdFi_AdminConsole_AppSettings__OverrideExistingDatabase: ${OVERRIDE_EXISTING_DATABASE:-false}
       EdFi_AdminConsole_AppSettings__IgnoresCertificateErrors: ${IGNORES_CERTIFICATE_ERRORS:-true}
+      EdFi_AdminConsole_AppSettings__MaxRetryAttempts: ${MAX_RETRY_ATTEMPTS:-5}
       EdFi_AdminConsole_ConnectionStrings__EdFi_Master: ${EDFI_MASTER:-host=host.docker.internal;port=5432;username=postgres;password=admin;database=postgres;pooling=false}
       EdFi_AdminConsole_ConnectionStrings__EdFi_Ods: ${EDFI_ODS:-host=host.docker.internal;port=5432;username=postgres;password=admin;database={0};pooling=false}
       EdFi_AdminConsole_AdminApiSettings__AccessTokenUrl: ${ACCESS_TOKEN_URL:-https://host.docker.internal/auth/realms/edfi-admin-console/protocol/openid-connect/token}

src/EdFi.AdminConsole.InstanceManagementWorker/Startup.cs

@@ -3,7 +3,6 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
-using EdFi.Admin.DataAccess.Utils;
 using EdFi.AdminConsole.InstanceMgrWorker.Configuration.Provisioners;
 using EdFi.AdminConsole.InstanceMgrWorker.Core;
 using EdFi.AdminConsole.InstanceMgrWorker.Core.Features.AdminApi;
@@ -69,7 +68,6 @@ public static void ConfigureServices(IServiceCollection services, IConfiguration
                 services.AddTransient<IInstanceProvisioner, PostgresInstanceProvisioner>();
             }
 
-            services.AddTransient<IHttpRequestMessageBuilder, HttpRequestMessageBuilder>();
             services.AddTransient<IAdminApiClient, AdminApiClient>();
             services.AddTransient<IAdminApiCaller, AdminApiCaller>();
             services.AddTransient<IApplication, Application>();

src/EdFi.AdminConsole.InstanceManagementWorker/appsettings.json

@@ -6,7 +6,8 @@
         "DefaultPageSizeLimit": 25,
         "OverrideExistingDatabase": false,
         "IgnoresCertificateErrors": true,
-        "SqlServerBakFile": "/tmp/Ods_Minimal_Template.bak"
+        "SqlServerBakFile": "/tmp/Ods_Minimal_Template.bak",
+        "MaxRetryAttempts": 5
     },
     "AdminApiSettings": {
         "AdminConsoleTenantsURL": "https://localhost/adminapi/adminconsole/tenants",

src/EdFi.AdminConsole.InstanceMgrWorker.Core/AppSettings.cs

@@ -11,6 +11,7 @@ public interface IAppSettings
         bool IgnoresCertificateErrors { get; set; }
         bool OverrideExistingDatabase { get; set; }
         string SqlServerBakFile { get; set; }
+        int MaxRetryAttempts { get; set; }
     }
 
     public sealed class AppSettings : IAppSettings
@@ -19,5 +20,6 @@ public sealed class AppSettings : IAppSettings
         public bool IgnoresCertificateErrors { get; set; } = false;
         public bool OverrideExistingDatabase { get; set; } = false;
         public string SqlServerBakFile { get; set; } = string.Empty;
+        public int MaxRetryAttempts { get; set; } = 5;
     }
 }

src/EdFi.AdminConsole.InstanceMgrWorker.Core/EdFi.AdminConsole.InstanceMgrWorker.Core.csproj

@@ -21,6 +21,8 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="9.0.1" />
     <PackageReference Include="Microsoft.Extensions.Options" Version="9.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageReference Include="Polly" Version="8.5.2" />
+    <PackageReference Include="Polly.Contrib.WaitAndRetry" Version="1.1.1" />
     <PackageReference Include="SonarAnalyzer.CSharp" Version="10.6.0.109712">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

src/EdFi.AdminConsole.InstanceMgrWorker.Core/Extensions/HttpStatusCodeExtensions.cs

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: Apache-2.0
+// Licensed to the Ed-Fi Alliance under one or more agreements.
+// The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
+// See the LICENSE and NOTICES files in the project root for more information.
+
+using System.Net;
+
+namespace EdFi.AdminConsole.InstanceMgrWorker.Core.Extensions;
+public static class HttpStatusCodeExtensions
+{
+    public static bool IsPotentiallyTransientFailure(this HttpStatusCode httpStatusCode)
+    {
+        switch (httpStatusCode)
+        {
+            case HttpStatusCode.InternalServerError:
+            case HttpStatusCode.GatewayTimeout:
+            case HttpStatusCode.ServiceUnavailable:
+            case HttpStatusCode.RequestTimeout:
+                return true;
+            default:
+                return false;
+        }
+    }
+}

src/EdFi.AdminConsole.InstanceMgrWorker.Core/Features/AdminApi/AdminApiClient.cs

@@ -39,34 +39,18 @@ public async Task<ApiResponse> AdminApiGet(string url, string? tenant)
 
         if (!string.IsNullOrEmpty(_accessToken))
         {
-            const int RetryAttempts = 3;
-            var currentAttempt = 0;
-
             StringContent? content = null;
             if (!string.IsNullOrEmpty(tenant))
             {
                 content = new StringContent(string.Empty, Encoding.UTF8, "application/json");
                 content.Headers.Add("tenant", tenant);
             }
 
-            while (RetryAttempts > currentAttempt)
-            {
-                response = await _appHttpClient.SendAsync(url,
-                    HttpMethod.Get,
-                    content,
-                    new AuthenticationHeaderValue("bearer", _accessToken)
-                );
-
-                currentAttempt++;
-
-                if (response.StatusCode == HttpStatusCode.OK)
-                    break;
-
-                if (currentAttempt == RetryAttempts)
-                {
-                    _logger.LogError("Error calling {0}. Status Code: {1}. Response: {3}", url, response.StatusCode.ToString(), response.Content);
-                }
-            }
+            response = await _appHttpClient.SendAsync(url,
+                HttpMethod.Get,
+                content,
+                new AuthenticationHeaderValue("bearer", _accessToken)
+            );
         }
 
         return response;
@@ -75,36 +59,23 @@ public async Task<ApiResponse> AdminApiGet(string url, string? tenant)
     public async Task<ApiResponse> AdminApiPost(string url, string? tenant, object? body = null)
     {
         ApiResponse response = new ApiResponse(HttpStatusCode.InternalServerError, "Unknown error.");
-        await GetAccessToken();
-
-        const int RetryAttempts = 3;
-        var currentAttempt = 0;
-
-        StringContent? content = new StringContent(body != null ? JsonConvert.SerializeObject(body) : string.Empty, Encoding.UTF8, "application/json");
-
-        if (!string.IsNullOrEmpty(tenant))
-        {
-            content.Headers.Add("tenant", tenant);
-        }
-
-        while (RetryAttempts > currentAttempt)
-        {
-            response = await _appHttpClient.SendAsync(
-                url,
-                HttpMethod.Post,
-                content,
-                new AuthenticationHeaderValue("bearer", _accessToken)
-            );
-
-            currentAttempt++;
-
-            if (response.StatusCode is HttpStatusCode.Created or HttpStatusCode.OK or HttpStatusCode.NoContent)
-                break;
+        await GetAccessToken();
 
-            if (currentAttempt == RetryAttempts)
+        if (!string.IsNullOrEmpty(_accessToken))
+        {
+            StringContent? content = new StringContent(body != null ? JsonConvert.SerializeObject(body) : string.Empty, Encoding.UTF8, "application/json");
+
+            if (!string.IsNullOrEmpty(tenant))
             {
-                _logger.LogError("Error calling {0}. Status Code: {1}. Response: {3}", url, response.StatusCode.ToString(), response.Content);
-            }
+                content.Headers.Add("tenant", tenant);
+            }
+
+            response = await _appHttpClient.SendAsync(
+                url,
+                HttpMethod.Post,
+                content,
+                new AuthenticationHeaderValue("bearer", _accessToken)
+            );
         }
 
         return response;

src/EdFi.AdminConsole.InstanceMgrWorker.Core/Helpers/Constants.cs

@@ -14,5 +14,7 @@ public struct Constants
         public const string TenantHeader = "tenant";
 
         public const string CompletedInstances = "?status=Completed";
+
+        public const int RetryStartingDelayMilliseconds = 500;
     }
 }

src/EdFi.AdminConsole.InstanceMgrWorker.Core/Infrastructure/AppHttpClient.cs

@@ -3,9 +3,14 @@
 // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0.
 // See the LICENSE and NOTICES files in the project root for more information.
 
+using System.Net;
 using System.Net.Http.Headers;
+using EdFi.AdminConsole.InstanceMgrWorker.Core.Extensions;
+using EdFi.AdminConsole.InstanceMgrWorker.Core.Helpers;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
+using Polly;
+using Polly.Contrib.WaitAndRetry;
 
 namespace EdFi.AdminConsole.InstanceMgrWorker.Core.Infrastructure
 {
@@ -16,45 +21,81 @@ public interface IAppHttpClient
         Task<ApiResponse> SendAsync(string uriString, HttpMethod method, FormUrlEncodedContent content, AuthenticationHeaderValue? authenticationHeaderValue);
     }
 
-    public class AppHttpClient : IAppHttpClient
+    public class AppHttpClient(HttpClient httpClient, ILogger logger, IOptions<AppSettings> options) : IAppHttpClient
     {
-        private readonly HttpClient _httpClient;
-        protected readonly ILogger _logger;
-        protected readonly IOptions<AppSettings> _options;
-        private readonly IHttpRequestMessageBuilder _httpRequestMessageBuilder;
-
-        public AppHttpClient(HttpClient httpClient, ILogger logger, IOptions<AppSettings> options, IHttpRequestMessageBuilder httpRequestMessageBuilder)
-        {
-            _httpClient = httpClient;
-            _logger = logger;
-            _options = options;
-            _httpRequestMessageBuilder = httpRequestMessageBuilder;
-        }
+        private readonly HttpClient _httpClient = httpClient;
+        protected readonly ILogger _logger = logger;
+        protected readonly AppSettings _options = options.Value;
 
         public async Task<ApiResponse> SendAsync(string uriString, HttpMethod method, StringContent? content, AuthenticationHeaderValue? authenticationHeaderValue)
         {
-            var request = _httpRequestMessageBuilder.GetHttpRequestMessage(uriString, method, content);
+            var getByIdDelay = Backoff.ExponentialBackoff(
+            TimeSpan.FromMilliseconds(Constants.RetryStartingDelayMilliseconds),
+            _options.MaxRetryAttempts);
 
-            if (authenticationHeaderValue != null)
+            int attempts = 0;
+
+            var retryPolicy = Policy
+                .HandleResult<HttpResponseMessage>(r => r.StatusCode.IsPotentiallyTransientFailure())
+                .WaitAndRetryAsync(
+                    getByIdDelay,
+                    (result, ts, retryAttempt, ctx) =>
+                    {
+                        _logger.LogWarning("Retrying GET for resource '{UriString}'. Failed with status '{StatusCode}'. Retrying... (retry #{RetryAttempt} of {MaxRetryAttempts} with {TotalSeconds:N1}s delay)",
+                            uriString, result.Result.StatusCode, retryAttempt, _options.MaxRetryAttempts, ts.TotalSeconds);
+                    });
+
+            var response = await retryPolicy.ExecuteAsync(
+                async (ctx, ct) =>
+                {
+                    attempts++;
+
+                    if (attempts > 1)
+                    {
+                        _logger.LogDebug("GET for resource '{UriString}'. Attempt #{GetByIdAttempts}.",
+                            uriString, attempts);
+                    }
+
+                    var requestMessage = new HttpRequestMessage(method, uriString)
+                    {
+                        Content = content
+                    };
+
+                    if (authenticationHeaderValue != null)
+                    {
+                        requestMessage.Headers.Authorization = authenticationHeaderValue;
+                    }
+
+                    return await _httpClient.SendAsync(requestMessage, ct);
+                },
+                [],
+                CancellationToken.None);
+
+            string responseContent = await response.Content.ReadAsStringAsync().ConfigureAwait(false);
+
+            if (response.StatusCode != HttpStatusCode.OK)
             {
-                _httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
+                var message = $"GET request for '{uriString}' reference failed with status '{response.StatusCode}': {responseContent}";
+                _logger.LogWarning(message);
             }
-            var response = await _httpClient.SendAsync(request);
-            var responseContent = await response.Content.ReadAsStringAsync();
+
             return new ApiResponse(response.StatusCode, responseContent, response.Headers);
         }
 
         /// Access Token
         public async Task<ApiResponse> SendAsync(string uriString, HttpMethod method, FormUrlEncodedContent content, AuthenticationHeaderValue? authenticationHeaderValue)
         {
-            var request = _httpRequestMessageBuilder.GetHttpRequestMessage(uriString, method, content);
+            using var requestMessage = new HttpRequestMessage(method, uriString)
+            {
+                Content = content
+            };
 
             if (authenticationHeaderValue != null)
             {
                 _httpClient.DefaultRequestHeaders.Authorization = authenticationHeaderValue;
             }
 
-            var response = await _httpClient.SendAsync(request);
+            var response = await _httpClient.SendAsync(requestMessage);
             var responseContent = await response.Content.ReadAsStringAsync();
             return new ApiResponse(response.StatusCode, responseContent, response.Headers);
         }