Add exponential backoff retries for extension bundle download and reuse HttpClient (#11315)

* Implement exponential backoff retry

* white space

* update the test for fix

* suppresing warning

* introduce primary constructor

* fix white space

* Improve the UserAgent and AzureRef header.

* Improve UserAgent and AzureRef header

* fix user agent issue.

* Fixing runtimeassemblies.json and minor code cleanup

* Updating Microsoft.Extensions.Diagnostics resolution policy

* reflect the comments

* rollback bom

---------

Co-authored-by: Fabio Cavalcante <[email protected]>

Author: TsuyoshiUshio

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -14,6 +14,7 @@
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks;
+using Microsoft.Azure.WebJobs.Script.ExtensionBundle;
 using Microsoft.Azure.WebJobs.Script.Grpc;
 using Microsoft.Azure.WebJobs.Script.Metrics;
 using Microsoft.Azure.WebJobs.Script.Middleware;
@@ -143,6 +144,8 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
             services.AddSingleton<IFunctionMetadataManager, FunctionMetadataManager>();
             services.AddSingleton<IWebFunctionsManager, WebFunctionsManager>();
             services.AddHttpClient();
+            services.AddBundlesHttpClient();
+
             services.AddSingleton<StartupContextProvider>();
             services.AddSingleton<IFileSystem>(_ => FileUtility.Instance);
             services.AddTransient<VirtualFileSystem>();

src/WebJobs.Script/Diagnostics/Extensions/ExtensionBundleLoggerExtension.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Net;
 using System.Net.Http;
 using Microsoft.Extensions.Logging;
 
@@ -83,6 +84,26 @@ internal static class ExtensionBundleLoggerExtension
             new EventId(111, nameof(MatchingBundleNotFound)),
             "Bundle version matching the {version} was not found");
 
+        private static readonly Action<ILogger, Uri, HttpStatusCode?, HttpRequestError?, string, string, string, Exception> _errorDownloadingExtensionBundleZipHttpRequest =
+            LoggerMessage.Define<Uri, HttpStatusCode?, HttpRequestError?, string, string, string>(
+            LogLevel.Error,
+            new EventId(112, nameof(ErrorDownloadingExtensionBundleZipHttpRequest)),
+            "Error downloading extension bundle zip content {zip}. Status Code:{statusCode}, RequestError:{requestError}, FilePath:{filePath}, Disk:{diskUsage}, AzureRef:{azureRef}");
+
+        // Logs unexpected (non-HttpRequestException) failures during bundle download. No HTTP status/request error data.
+        private static readonly Action<ILogger, Uri, string, string, string, Exception> _errorDownloadingExtensionBundleZipUnexpected =
+            LoggerMessage.Define<Uri, string, string, string>(
+            LogLevel.Error,
+            new EventId(113, nameof(ErrorDownloadingExtensionBundleZipUnexpected)),
+            "Unexpected error downloading extension bundle Zip content {zip}. FilePath:{filePath}, Disk:{diskUsage}, AzureRef:{azureRef}");
+
+        // Logs IO-specific failures (e.g., disk full, device I/O issues) during bundle download.
+        private static readonly Action<ILogger, Uri, string, string, string, Exception> _errorDownloadingExtensionBundleZipIO =
+            LoggerMessage.Define<Uri, string, string, string>(
+            LogLevel.Error,
+            new EventId(114, nameof(ErrorDownloadingExtensionBundleZipIO)),
+            "IO error downloading extension bundle Zip content {zip}. FilePath:{filePath}, Disk:{diskUsage}, AzureRef:{azureRef}");
+
         public static void ContentProviderNotConfigured(this ILogger logger, string path)
         {
             _contentProviderNotConfigured(logger, path, null);
@@ -127,6 +148,23 @@ public static void ErrorDownloadingZip(this ILogger logger, Uri zipUri, HttpResp
             _errorDownloadingZip(logger, zip, statusCode, reasonPhrase, null);
         }
 
+        public static void ErrorDownloadingExtensionBundleZipHttpRequest(this ILogger logger, Exception ex, Uri zipUri, HttpStatusCode? statusCode, HttpRequestError? httpRequestError, string filePath, string diskUsage, string azureRef)
+        {
+            // Avoid premature ToString allocations; LoggerMessage will format only if enabled.
+            _errorDownloadingExtensionBundleZipHttpRequest(logger, zipUri, statusCode, httpRequestError, filePath, diskUsage, azureRef, ex);
+        }
+
+        public static void ErrorDownloadingExtensionBundleZipUnexpected(this ILogger logger, Exception ex, Uri zipUri, string filePath, string diskUsage, string azureRef)
+        {
+            // Generic non-HTTP failure (e.g. IO, disk full, zip extraction issues).
+            _errorDownloadingExtensionBundleZipUnexpected(logger, zipUri, filePath, diskUsage, azureRef, ex);
+        }
+
+        public static void ErrorDownloadingExtensionBundleZipIO(this ILogger logger, Exception ex, Uri zipUri, string filePath, string diskUsage, string azureRef)
+        {
+            _errorDownloadingExtensionBundleZipIO(logger, zipUri, filePath, diskUsage, azureRef, ex);
+        }
+
         public static void DownloadComplete(this ILogger logger, Uri zipUri, string filePath)
         {
             string zip = zipUri.ToString();

src/WebJobs.Script/ExtensionBundle/BundlesServiceCollectionExtensions.cs

@@ -0,0 +1,54 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Runtime.InteropServices;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using Polly;
+using Polly.Extensions.Http;
+
+namespace Microsoft.Azure.WebJobs.Script.ExtensionBundle
+{
+    public static class BundlesServiceCollectionExtensions
+    {
+        public static IServiceCollection AddBundlesHttpClient(this IServiceCollection services)
+        {
+            services.AddHttpClient(nameof(ExtensionBundleManager), client =>
+            {
+                var hostVersion = ScriptHost.Version;
+                client.DefaultRequestHeaders.UserAgent.ParseAdd($"AzureFunctionsHost/{hostVersion}");
+            })
+                .AddPolicyHandler((sp, request) =>
+                {
+                    var logger = sp.GetRequiredService<ILogger<ExtensionBundleManager>>();
+
+                    var policyBuilder = HttpPolicyExtensions
+                    .HandleTransientHttpError()
+                    .OrResult(resp => resp.StatusCode == HttpStatusCode.TooManyRequests);
+
+                    return policyBuilder.WaitAndRetryAsync(
+                        retryCount: 4,
+                        sleepDurationProvider: attempt => TimeSpan.FromSeconds(Math.Pow(2, attempt)),
+                        onRetry: (outcome, delay, attempt, ctx) =>
+                        {
+                            var statusCode = outcome.Result?.StatusCode;
+                            var statusCodeDisplay = statusCode.HasValue ? ((int)statusCode.Value).ToString() : "None";
+                            string azureRef = null;
+                            outcome.Result?.TryGetAzureRef(out azureRef);
+                            logger.LogWarning(
+                                outcome.Exception,
+                                "Extension bundle download failure. Status: {StatusCode}, Attempt: {Attempt}, Uri: {Uri}, AzureRef: {AzureRef}. Retrying after {DelayMs}ms.",
+                                statusCodeDisplay,
+                                attempt,
+                                request?.RequestUri,
+                                azureRef,
+                                delay.TotalMilliseconds);
+                        });
+                });
+            return services;
+        }
+    }
+}

src/WebJobs.Script/ExtensionBundle/ExtensionBundleHttpExtensions.cs

@@ -0,0 +1,40 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Net.Http;
+
+namespace Microsoft.Azure.WebJobs.Script.ExtensionBundle
+{
+    internal static class ExtensionBundleHttpExtensions
+    {
+        // The X-Azure-Ref header is added by Azure Front Door for requests that traverse Front Door to the origin.
+        // Documentation: https://learn.microsoft.com/en-us/azure/frontdoor/front-door-http-headers-protocol#from-the-front-door-to-the-backend
+        // We capture and log this value (sanitized + length bounded) with extension bundle download failures
+        // to enable end-to-end correlation and root cause analysis on the Front Door side when diagnosing
+        // CDN/edge or networking issues impacting bundle retrieval.
+        internal const string AzureRefHeaderName = "X-Azure-Ref";
+
+        internal static bool TryGetAzureRef(this HttpResponseMessage response, out string azureRef)
+        {
+            ArgumentNullException.ThrowIfNull(response);
+
+            try
+            {
+                if (response.Headers is not null &&
+                    response.Headers.TryGetValues(AzureRefHeaderName, out var values))
+                {
+                    azureRef = values.FirstOrDefault();
+                    return !string.IsNullOrEmpty(azureRef);
+                }
+            }
+            catch (Exception)
+            {
+            }
+
+            azureRef = null;
+            return false;
+        }
+    }
+}

src/WebJobs.Script/ExtensionBundle/ExtensionBundleManager.cs

@@ -7,6 +7,7 @@
 using System.IO.Compression;
 using System.Linq;
 using System.Net.Http;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
@@ -22,22 +23,25 @@ namespace Microsoft.Azure.WebJobs.Script.ExtensionBundle
 {
     public class ExtensionBundleManager : IExtensionBundleManager
     {
+        private const string ExtensionBundleClientName = nameof(ExtensionBundleManager);
         private readonly IEnvironment _environment;
         private readonly ExtensionBundleOptions _options;
         private readonly FunctionsHostingConfigOptions _configOption;
         private readonly ILogger _logger;
         private readonly string _cdnUri;
         private readonly string _platformReleaseChannel;
+        private readonly IHttpClientFactory _httpClientFactory;
         private string _extensionBundleVersion;
 
-        public ExtensionBundleManager(ExtensionBundleOptions options, IEnvironment environment, ILoggerFactory loggerFactory, FunctionsHostingConfigOptions configOption)
+        public ExtensionBundleManager(ExtensionBundleOptions options, IEnvironment environment, ILoggerFactory loggerFactory, FunctionsHostingConfigOptions configOption, IHttpClientFactory httpClientFactory)
         {
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
             _logger = loggerFactory.CreateLogger<ExtensionBundleManager>() ?? throw new ArgumentNullException(nameof(loggerFactory));
             _options = options ?? throw new ArgumentNullException(nameof(options));
             _configOption = configOption ?? throw new ArgumentNullException(nameof(configOption));
             _cdnUri = _environment.GetEnvironmentVariable(EnvironmentSettingNames.ExtensionBundleSourceUri) ?? ScriptConstants.ExtensionBundleDefaultSourceUri;
             _platformReleaseChannel = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper;
+            _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
         }
 
         public async Task<ExtensionBundleDetails> GetExtensionBundleDetails()
@@ -79,10 +83,8 @@ public bool IsLegacyExtensionBundle()
         /// <returns>Path of the extension bundle.</returns>
         public async Task<string> GetExtensionBundlePath()
         {
-            using (var httpClient = new HttpClient())
-            {
-                return await GetBundle(httpClient);
-            }
+            var client = _httpClientFactory.CreateClient(ExtensionBundleClientName);
+            return await GetBundle(client);
         }
 
         /// <summary>
@@ -199,34 +201,103 @@ private string GetBundleFlavorForCurrentEnvironment()
             return ScriptConstants.ExtensionBundleForNonAppServiceEnvironment;
         }
 
-        private async Task<bool> TryDownloadZipFileAsync(Uri zipUri, string filePath, HttpClient httpClient)
+        private async Task<bool> TryDownloadZipFileAsync(Uri zipUri, string filePath, HttpClient httpClient, CancellationToken cancellationToken = default)
         {
-            _logger.DownloadingZip(zipUri, filePath);
-            var response = await httpClient.GetAsync(zipUri);
-            if (!response.IsSuccessStatusCode)
+            string azureRef = string.Empty;
+            try
+            {
+                _logger.DownloadingZip(zipUri, filePath);
+
+                using var response = await httpClient.GetAsync(zipUri, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+
+                // Log AzureRef header if present (debug level to avoid noise in normal operations)
+                response.TryGetAzureRef(out azureRef);
+
+                response.EnsureSuccessStatusCode();
+
+                using var fileStream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true);
+                await response.Content.CopyToAsync(fileStream, cancellationToken);
+                await fileStream.FlushAsync(cancellationToken);
+
+                _logger.DownloadComplete(zipUri, filePath);
+
+                return true;
+            }
+            catch (HttpRequestException ex)
             {
-                _logger.ErrorDownloadingZip(zipUri, response);
+                var statusCode = ex.StatusCode;
+                _logger.ErrorDownloadingExtensionBundleZipHttpRequest(
+                    ex,
+                    zipUri,
+                    statusCode,
+                    ex.HttpRequestError,
+                    filePath,
+                    GetDiskUsageSafe(filePath),
+                    azureRef);
                 return false;
             }
-
-            using (var content = await response.Content.ReadAsStreamAsync())
-            using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true))
+            catch (IOException ex)
             {
-                await content.CopyToAsync(stream);
+                _logger.ErrorDownloadingExtensionBundleZipIO(
+                    ex,
+                    zipUri,
+                    filePath,
+                    GetDiskUsageSafe(filePath),
+                    azureRef);
+                return false;
             }
+            catch (Exception ex)
+            {
+                // Non-HttpRequestException path: log as unexpected without HTTP-specific fields.
+                _logger.ErrorDownloadingExtensionBundleZipUnexpected(
+                    ex,
+                    zipUri,
+                    filePath,
+                    GetDiskUsageSafe(filePath),
+                    azureRef);
 
-            _logger.DownloadComplete(zipUri, filePath);
-            return true;
+                return false;
+            }
         }
 
-        private async Task<string> GetLatestMatchingBundleVersionAsync()
+        private string GetDiskUsageSafe(string path)
         {
-            using (var httpClient = new HttpClient())
+            try
+            {
+                var root = Path.GetPathRoot(path);
+                if (string.IsNullOrEmpty(root))
+                {
+                    return "error=RootPathNotFound";
+                }
+
+                var di = new DriveInfo(root);
+                const double BytesPerMB = 1024d * 1024d;
+                double freeMb = di.AvailableFreeSpace / BytesPerMB;
+                double totalMb = di.TotalSize / BytesPerMB;
+                return $"free={freeMb:F2}MB total={totalMb:F2}MB";
+            }
+            catch (Exception ex)
             {
-                return await GetLatestMatchingBundleVersionAsync(httpClient);
+                return FormatDiskError(ex);
             }
         }
 
+        private static string FormatDiskError(Exception ex)
+        {
+            var msg = ex.Message?.Replace(Environment.NewLine, " ").Trim();
+            if (!string.IsNullOrEmpty(msg) && msg.Length > 200)
+            {
+                msg = msg.Substring(0, 200) + "...";
+            }
+            return $"error={ex.GetType().Name}: {msg}";
+        }
+
+        private async Task<string> GetLatestMatchingBundleVersionAsync()
+        {
+            var client = _httpClientFactory.CreateClient(ExtensionBundleClientName);
+            return await GetLatestMatchingBundleVersionAsync(client);
+        }
+
         private async Task<string> GetLatestMatchingBundleVersionAsync(HttpClient httpClient)
         {
             var uri = new Uri($"{_cdnUri}/{ScriptConstants.ExtensionBundleDirectory}/{_options.Id}/{ScriptConstants.ExtensionBundleVersionIndexFile}");

src/WebJobs.Script/ScriptHostBuilderExtensions.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Diagnostics.Tracing;
+using System.Net.Http;
 using System.Runtime.InteropServices;
 using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.ApplicationInsights.WindowsServer.TelemetryChannel;
@@ -137,7 +138,8 @@ public static IHostBuilder AddScriptHost(this IHostBuilder builder,
 
                 var extensionRequirementOptions = applicationOptions.RootServiceProvider.GetService<IOptions<ExtensionRequirementOptions>>();
 
-                ExtensionBundleManager bundleManager = new(extensionBundleOptions, SystemEnvironment.Instance, loggerFactory, configOption);
+                var httpClientFactory = applicationOptions.RootServiceProvider.GetService<IHttpClientFactory>();
+                var bundleManager = new ExtensionBundleManager(extensionBundleOptions, SystemEnvironment.Instance, loggerFactory, configOption, httpClientFactory);
                 var metadataServiceManager = applicationOptions.RootServiceProvider.GetService<IFunctionMetadataManager>();
 
                 ScriptStartupTypeLocator locator = new(

src/WebJobs.Script/WebJobs.Script.csproj

@@ -50,6 +50,8 @@
     <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="9.0.6" />
     <PackageReference Include="Microsoft.Extensions.Telemetry.Abstractions" Version="9.8.0" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.7" />
     <PackageReference Include="Mono.Posix.NETStandard" Version="1.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="NuGet.ProjectModel" Version="5.11.6" />