Retry GitHub anonymously (#860)

Author: BMurri

src/deploy-cromwell-on-azure/Deployer.cs

@@ -83,6 +83,7 @@ var x when StringComparisonOrdinalIgnoreCase(x, "OperationNotAllowed") => true,
         private static bool AsyncRetryExceptionPolicy(Exception ex)
         {
             var dontRetry = ex is InvalidOperationException
+                || (ex is Microsoft.Kiota.Abstractions.ApiException ae && (int)HttpStatusCode.Unauthorized == ae.ResponseStatusCode)
                 || (ex is GitHub.Models.ValidationError ve && (int)HttpStatusCode.UnprocessableContent == ve.ResponseStatusCode)
                 || (ex is GitHub.Models.BasicError be &&
                     ((int)HttpStatusCode.Forbidden == be.ResponseStatusCode
@@ -1223,46 +1224,61 @@ private async Task BuildPushAcrAsync(Dictionary<string, string> settings, string
                 var build = await Execute($"Building TES and TriggerService images on {acr.Id.Name}...",
                 () => buildPushAcrRetryPolicy.ExecuteAsync(async () =>
                 {
-                    AcrBuild build;
-                    {
-                        IAsyncDisposable tarDisposable = default;
+                    AcrBuild build = default;
+                    await Policy.Handle<Microsoft.Kiota.Abstractions.ApiException>(ae => (int)HttpStatusCode.Unauthorized == ae.ResponseStatusCode)
+                        .WaitAndRetryAsync([TimeSpan.FromSeconds(1)], (ae, _) =>
+                        {
+                            if (string.IsNullOrWhiteSpace(Environment.GetEnvironmentVariable("GITHUB_TOKEN")))
+                            {
+                                throw new InvalidOperationException("GitHub returned an authentication error.", ae);
+                            }
 
-                        try
+                            Console.WriteLine("GitHub returned an authentication error. Retrying anonymously.");
+                            Environment.SetEnvironmentVariable("GITHUB_TOKEN", null);
+                        })
+                        .ExecuteAsync(async token =>
                         {
-                            IArchive tar;
+                            IAsyncDisposable tarDisposable = default;
 
-                            if (string.IsNullOrWhiteSpace(configuration.SolutionDir))
+                            try
                             {
-                                tar = AcrBuild.GetGitHubArchive(BuildType.CoA, string.IsNullOrWhiteSpace(configuration.GitHubCommit) ? new Version(targetVersion).ToString(3) : configuration.GitHubCommit, GitHubArchive.GetAccessTokenProvider());
-                                tarDisposable = tar as IAsyncDisposable;
+                                IArchive tar;
+
+                                if (string.IsNullOrWhiteSpace(configuration.SolutionDir))
+                                {
+                                    tar = GitHubArchive.Create(BuildType.CoA, string.IsNullOrWhiteSpace(configuration.GitHubCommit) ? new Version(targetVersion).ToString(3) : configuration.GitHubCommit, GitHubArchive.GetAccessTokenProvider());
+                                    tarDisposable = tar as IAsyncDisposable;
+                                }
+                                else
+                                {
+                                    tar = LocalGitArchive.Create(new(configuration.SolutionDir));
+                                }
+
+                                build = new(BuildType.CoA, await tar.GetTagAsync(token), acr.Id, tokenCredential, new Azure.Containers.ContainerRegistry.ContainerRegistryAudience(azureCloudConfig.ArmEnvironment.Value.Endpoint.AbsoluteUri));
+                                await build.LoadAsync(tar, azureCloudConfig.ArmEnvironment.Value, token);
                             }
-                            else
+                            finally
                             {
-                                tar = AcrBuild.GetLocalGitArchiveAsync(new(configuration.SolutionDir));
+                                await (tarDisposable?.DisposeAsync() ?? ValueTask.CompletedTask);
                             }
+                        },
+                        cts.Token);
 
-                            build = new(BuildType.CoA, await tar.GetTagAsync(cts.Token), acr.Id, tokenCredential, new Azure.Containers.ContainerRegistry.ContainerRegistryAudience(azureCloudConfig.ArmEnvironment.Value.Endpoint.AbsoluteUri));
-                            await build.LoadAsync(tar, azureCloudConfig.ArmEnvironment.Value, cts.Token);
-                        }
-                        finally
+                    if (!await Policy
+                        .HandleResult(false)
+                        .WaitAndRetryAsync(Enumerable.Repeat(TimeSpan.FromSeconds(1), 2), (_, _) => Console.WriteLine("Retrying build."))
+                        .ExecuteAsync(async token =>
                         {
-                            await (tarDisposable?.DisposeAsync() ?? ValueTask.CompletedTask);
-                        }
-                    }
-
-                    var buildSuccess = false;
-
-                    for (var i = 3; i > 0 && !buildSuccess; --i, await Task.Delay(TimeSpan.FromSeconds(1), cts.Token))
-                    {
-                        (buildSuccess, var buildLog) = await build.BuildAsync(configuration.DebugLogging ? LogType.Interactive : LogType.CapturedOnError, cts.Token);
+                            var (buildSuccess, buildLog) = await build.BuildAsync(configuration.DebugLogging ? LogType.Interactive : LogType.CapturedOnError, token);
 
-                        if (!buildSuccess && !string.IsNullOrWhiteSpace(buildLog))
-                        {
-                            ConsoleEx.WriteLine(buildLog);
-                        }
-                    }
+                            if (!buildSuccess && !string.IsNullOrWhiteSpace(buildLog))
+                            {
+                                ConsoleEx.WriteLine(buildLog);
+                            }
 
-                    if (!buildSuccess)
+                            return buildSuccess;
+                        },
+                        cts.Token))
                     {
                         throw new InvalidOperationException("Build failed.");
                     }