Fixing WEBSITE_HOSTNAME issue (#4162)

Author: mathewc

src/WebJobs.Script.WebHost/Diagnostics/AzureMonitorDiagnosticLogger.cs

@@ -18,30 +18,30 @@ public class AzureMonitorDiagnosticLogger : ILogger
         private readonly string _regionName;
         private readonly string _category;
         private readonly string _hostInstanceId;
-        private readonly string _websiteHostName;
 
+        private readonly HostNameProvider _hostNameProvider;
         private readonly IEventGenerator _eventGenerator;
         private readonly IEnvironment _environment;
         private readonly IExternalScopeProvider _scopeProvider;
 
-        public AzureMonitorDiagnosticLogger(string category, string hostInstanceId, IEventGenerator eventGenerator, IEnvironment environment, IExternalScopeProvider scopeProvider)
+        public AzureMonitorDiagnosticLogger(string category, string hostInstanceId, IEventGenerator eventGenerator, IEnvironment environment, IExternalScopeProvider scopeProvider, HostNameProvider hostNameProvider)
         {
             _category = category ?? throw new ArgumentNullException(nameof(category));
             _hostInstanceId = hostInstanceId ?? throw new ArgumentNullException(nameof(hostInstanceId));
             _eventGenerator = eventGenerator ?? throw new ArgumentNullException(nameof(eventGenerator));
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
             _scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider));
+            _hostNameProvider = hostNameProvider ?? throw new ArgumentNullException(nameof(hostNameProvider));
 
             _regionName = _environment.GetEnvironmentVariable(EnvironmentSettingNames.RegionName) ?? string.Empty;
-            _websiteHostName = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName);
         }
 
         public IDisposable BeginScope<TState>(TState state) => _scopeProvider.Push(state);
 
         public bool IsEnabled(LogLevel logLevel)
         {
             // We want to instantiate this Logger in placeholder mode to warm it up, but do not want to log anything.
-            return !string.IsNullOrEmpty(_websiteHostName) && !_environment.IsPlaceholderModeEnabled();
+            return !string.IsNullOrEmpty(_hostNameProvider.Value) && !_environment.IsPlaceholderModeEnabled();
         }
 
         public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
@@ -82,7 +82,7 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except
                 writer.WriteEndObject();
             }
 
-            _eventGenerator.LogAzureMonitorDiagnosticLogEvent(logLevel, _websiteHostName, AzureMonitorOperationName, AzureMonitorCategoryName, _regionName, sw.ToString());
+            _eventGenerator.LogAzureMonitorDiagnosticLogEvent(logLevel, _hostNameProvider.Value, AzureMonitorOperationName, AzureMonitorCategoryName, _regionName, sw.ToString());
         }
 
         private static void WritePropertyIfNotNull(JsonTextWriter writer, string propertyName, string propertyValue)

src/WebJobs.Script.WebHost/Diagnostics/AzureMonitorDiagnosticLoggerProvider.cs

@@ -12,23 +12,25 @@ public class AzureMonitorDiagnosticLoggerProvider : ILoggerProvider, ISupportExt
         private readonly string _hostInstanceId;
         private readonly IEventGenerator _eventGenerator;
         private readonly IEnvironment _environment;
+        private readonly HostNameProvider _hostNameProvider;
         private IExternalScopeProvider _scopeProvider;
 
-        public AzureMonitorDiagnosticLoggerProvider(IOptions<ScriptJobHostOptions> scriptOptions, IEventGenerator eventGenerator, IEnvironment environment)
-            : this(scriptOptions.Value.InstanceId, eventGenerator, environment)
+        public AzureMonitorDiagnosticLoggerProvider(IOptions<ScriptJobHostOptions> scriptOptions, IEventGenerator eventGenerator, IEnvironment environment, HostNameProvider hostNameProvider)
+            : this(scriptOptions.Value.InstanceId, eventGenerator, environment, hostNameProvider)
         {
         }
 
-        public AzureMonitorDiagnosticLoggerProvider(string hostInstanceId, IEventGenerator eventGenerator, IEnvironment environment)
+        public AzureMonitorDiagnosticLoggerProvider(string hostInstanceId, IEventGenerator eventGenerator, IEnvironment environment, HostNameProvider hostNameProvider)
         {
             _hostInstanceId = hostInstanceId ?? throw new ArgumentNullException(hostInstanceId);
             _eventGenerator = eventGenerator ?? throw new ArgumentNullException(nameof(eventGenerator));
             _environment = environment ?? throw new ArgumentException(nameof(environment));
+            _hostNameProvider = hostNameProvider ?? throw new ArgumentException(nameof(hostNameProvider));
         }
 
         public ILogger CreateLogger(string categoryName)
         {
-            return new AzureMonitorDiagnosticLogger(categoryName, _hostInstanceId, _eventGenerator, _environment, _scopeProvider);
+            return new AzureMonitorDiagnosticLogger(categoryName, _hostInstanceId, _eventGenerator, _environment, _scopeProvider, _hostNameProvider);
         }
 
         public void Dispose()

src/WebJobs.Script.WebHost/HostNameProvider.cs

@@ -0,0 +1,72 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost
+{
+    /// <summary>
+    /// Provides the current HostName for the Function App.
+    /// <remarks>
+    /// The environment value for WEBSITE_HOSTNAME is unreliable and shouldn't be used directly. AppService site swaps change
+    /// the site’s hostname under the covers, and the worker process is NOT recycled (for performance reasons). That means the
+    /// site will continue to run with the same hostname environment variable, leading to an incorrect host name.
+    ///
+    /// WAS_DEFAULT_HOSTNAME is a header injected by front end on every request which provides the correct hostname. We check
+    /// this header on all http requests, and updated the cached hostname value as needed.
+    /// </remarks>
+    /// </summary>
+    public class HostNameProvider
+    {
+        private readonly IEnvironment _environment;
+        private readonly ILogger _logger;
+        private string _hostName;
+
+        public HostNameProvider(IEnvironment environment, ILogger<HostNameProvider> logger)
+        {
+            _environment = environment ?? throw new ArgumentNullException(nameof(environment));
+            _logger = logger;
+        }
+
+        public virtual string Value
+        {
+            get
+            {
+                if (string.IsNullOrEmpty(_hostName))
+                {
+                    // default to the the value specified in environment
+                    _hostName = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName);
+                    if (string.IsNullOrEmpty(_hostName))
+                    {
+                        // Linux Dedicated on AppService doesn't have WEBSITE_HOSTNAME
+                        string websiteName = _environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName);
+                        if (!string.IsNullOrEmpty(websiteName))
+                        {
+                            _hostName = $"{websiteName}.azurewebsites.net";
+                        }
+                    }
+                }
+                return _hostName;
+            }
+        }
+
+        public virtual void Synchronize(HttpRequest request)
+        {
+            string hostNameHeaderValue = request.Headers[ScriptConstants.AntaresDefaultHostNameHeader];
+            if (!string.IsNullOrEmpty(hostNameHeaderValue) &&
+                string.Compare(Value, hostNameHeaderValue) != 0)
+            {
+                _logger.LogInformation("HostName updated from '{0}' to '{1}'", Value, hostNameHeaderValue);
+                _hostName = hostNameHeaderValue;
+            }
+        }
+
+        // for testing only
+        internal void Reset()
+        {
+            _hostName = null;
+        }
+    }
+}

src/WebJobs.Script.WebHost/Management/FunctionsSyncManager.cs

@@ -42,11 +42,12 @@ public class FunctionsSyncManager : IFunctionsSyncManager, IDisposable
         private readonly IHostIdProvider _hostIdProvider;
         private readonly IScriptWebHostEnvironment _webHostEnvironment;
         private readonly IEnvironment _environment;
+        private readonly HostNameProvider _hostNameProvider;
         private readonly SemaphoreSlim _syncSemaphore = new SemaphoreSlim(1, 1);
 
         private CloudBlockBlob _hashBlob;
 
-        public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IOptions<LanguageWorkerOptions> languageWorkerOptions, ILoggerFactory loggerFactory, HttpClient httpClient, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment)
+        public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostIdProvider, IOptionsMonitor<ScriptApplicationHostOptions> applicationHostOptions, IOptions<LanguageWorkerOptions> languageWorkerOptions, ILoggerFactory loggerFactory, HttpClient httpClient, ISecretManagerProvider secretManagerProvider, IScriptWebHostEnvironment webHostEnvironment, IEnvironment environment, HostNameProvider hostNameProvider)
         {
             _applicationHostOptions = applicationHostOptions;
             _logger = loggerFactory?.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
@@ -57,6 +58,7 @@ public FunctionsSyncManager(IConfiguration configuration, IHostIdProvider hostId
             _hostIdProvider = hostIdProvider;
             _webHostEnvironment = webHostEnvironment;
             _environment = environment;
+            _hostNameProvider = hostNameProvider;
         }
 
         public async Task<SyncTriggersResult> TrySyncTriggersAsync(bool checkHash = false)
@@ -329,21 +331,16 @@ private async Task<Dictionary<string, string>> ReadDurableTaskConfig()
             return config;
         }
 
-        internal static HttpRequestMessage BuildSetTriggersRequest()
+        internal HttpRequestMessage BuildSetTriggersRequest()
         {
             var protocol = "https";
-            // On private stamps with no ssl certificate use http instead.
-            if (Environment.GetEnvironmentVariable(EnvironmentSettingNames.SkipSslValidation) == "1")
+            if (_environment.GetEnvironmentVariable(EnvironmentSettingNames.SkipSslValidation) == "1")
             {
+                // On private stamps with no ssl certificate use http instead.
                 protocol = "http";
             }
 
-            var hostname = Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteHostName);
-            // Linux Dedicated on AppService doesn't have WEBSITE_HOSTNAME
-            hostname = string.IsNullOrWhiteSpace(hostname)
-                ? $"{Environment.GetEnvironmentVariable(EnvironmentSettingNames.AzureWebsiteName)}.azurewebsites.net"
-                : hostname;
-
+            var hostname = _hostNameProvider.Value;
             var url = $"{protocol}://{hostname}/operations/settriggers";
 
             return new HttpRequestMessage(HttpMethod.Post, url);

src/WebJobs.Script.WebHost/Middleware/SystemTraceMiddleware.cs

@@ -18,16 +18,19 @@ internal class SystemTraceMiddleware
     {
         private readonly ILogger _logger;
         private readonly RequestDelegate _next;
+        private readonly HostNameProvider _hostNameProvider;
 
-        public SystemTraceMiddleware(RequestDelegate next, ILogger<SystemTraceMiddleware> logger)
+        public SystemTraceMiddleware(RequestDelegate next, ILogger<SystemTraceMiddleware> logger, HostNameProvider hostNameProvider)
         {
             _logger = logger;
             _next = next;
+            _hostNameProvider = hostNameProvider;
         }
 
         public async Task Invoke(HttpContext context)
         {
             SetRequestId(context.Request);
+            _hostNameProvider.Synchronize(context.Request);
 
             var sw = new Stopwatch();
             sw.Start();

src/WebJobs.Script.WebHost/Security/KeyManagement/DefaultSecretManagerProvider.cs

@@ -21,10 +21,11 @@ public sealed class DefaultSecretManagerProvider : ISecretManagerProvider
         private readonly IHostIdProvider _hostIdProvider;
         private readonly IConfiguration _configuration;
         private readonly IEnvironment _environment;
+        private readonly HostNameProvider _hostNameProvider;
         private Lazy<ISecretManager> _secretManagerLazy;
 
         public DefaultSecretManagerProvider(IOptionsMonitor<ScriptApplicationHostOptions> options, IHostIdProvider hostIdProvider,
-            IConfiguration configuration, IEnvironment environment, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger)
+            IConfiguration configuration, IEnvironment environment, ILoggerFactory loggerFactory, IMetricsLogger metricsLogger, HostNameProvider hostNameProvider)
         {
             if (loggerFactory == null)
             {
@@ -35,6 +36,7 @@ public DefaultSecretManagerProvider(IOptionsMonitor<ScriptApplicationHostOptions
             _hostIdProvider = hostIdProvider ?? throw new ArgumentNullException(nameof(hostIdProvider));
             _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration));
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
+            _hostNameProvider = hostNameProvider ?? throw new ArgumentNullException(nameof(hostNameProvider));
 
             _logger = loggerFactory.CreateLogger(ScriptConstants.LogCategoryHostGeneral);
             _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
@@ -48,7 +50,7 @@ public DefaultSecretManagerProvider(IOptionsMonitor<ScriptApplicationHostOptions
 
         private void ResetSecretManager() => Interlocked.Exchange(ref _secretManagerLazy, new Lazy<ISecretManager>(Create));
 
-        private ISecretManager Create() => new SecretManager(CreateSecretsRepository(), _logger, _metricsLogger);
+        private ISecretManager Create() => new SecretManager(CreateSecretsRepository(), _logger, _metricsLogger, _hostNameProvider);
 
         internal ISecretsRepository CreateSecretsRepository()
         {

src/WebJobs.Script.WebHost/Security/KeyManagement/ScriptSecrets.cs

@@ -13,7 +13,6 @@ public abstract class ScriptSecrets
     {
         protected ScriptSecrets()
         {
-            HostName = ScriptSettingsManager.Instance.GetSetting(EnvironmentSettingNames.AzureWebsiteHostName);
             InstanceId = ScriptSettingsManager.Instance.AzureWebsiteInstanceId;
             Source = ScriptConstants.Runtime;
         }

src/WebJobs.Script.WebHost/Security/KeyManagement/SecretManager.cs

@@ -25,6 +25,7 @@ public class SecretManager : IDisposable, ISecretManager
         private readonly IKeyValueConverterFactory _keyValueConverterFactory;
         private readonly ILogger _logger;
         private readonly ISecretsRepository _repository;
+        private readonly HostNameProvider _hostNameProvider;
         private HostSecretsInfo _hostSecrets;
         private SemaphoreSlim _hostSecretsLock = new SemaphoreSlim(1, 1);
         private IMetricsLogger _metricsLogger;
@@ -35,24 +36,24 @@ public SecretManager()
         {
         }
 
-        public SecretManager(ISecretsRepository repository, ILogger logger, IMetricsLogger metricsLogger, bool createHostSecretsIfMissing = false)
-            : this(repository, new DefaultKeyValueConverterFactory(repository.IsEncryptionSupported), logger, metricsLogger, createHostSecretsIfMissing)
+        public SecretManager(ISecretsRepository repository, ILogger logger, IMetricsLogger metricsLogger, HostNameProvider hostNameProvider, bool createHostSecretsIfMissing = false)
+            : this(repository, new DefaultKeyValueConverterFactory(repository.IsEncryptionSupported), logger, metricsLogger, hostNameProvider, createHostSecretsIfMissing)
         {
         }
 
-        public SecretManager(ISecretsRepository repository, IKeyValueConverterFactory keyValueConverterFactory, ILogger logger, IMetricsLogger metricsLogger, bool createHostSecretsIfMissing = false)
+        public SecretManager(ISecretsRepository repository, IKeyValueConverterFactory keyValueConverterFactory, ILogger logger, IMetricsLogger metricsLogger, HostNameProvider hostNameProvider, bool createHostSecretsIfMissing = false)
         {
             _repository = repository;
             _keyValueConverterFactory = keyValueConverterFactory;
             _repository.SecretsChanged += OnSecretsChanged;
             _logger = logger;
             _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
             _repositoryClassName = _repository.GetType().Name.ToLower();
+            _hostNameProvider = hostNameProvider;
 
             if (createHostSecretsIfMissing)
             {
-                // The SecretManager implementation of GetHostSecrets will
-                // create a host secret if one is not present.
+                // GetHostSecrets will create host secrets if not present
                 GetHostSecretsAsync().GetAwaiter().GetResult();
             }
         }
@@ -450,6 +451,11 @@ private Task RefreshSecretsAsync<T>(T secrets, string keyScope = null) where T :
 
         private async Task PersistSecretsAsync<T>(T secrets, string keyScope = null, bool isNonDecryptable = false) where T : ScriptSecrets
         {
+            if (secrets != null)
+            {
+                secrets.HostName = _hostNameProvider.Value;
+            }
+
             ScriptSecretsType secretsType = secrets.SecretsType;
             if (isNonDecryptable)
             {

src/WebJobs.Script.WebHost/WebHooks/DefaultScriptWebHookProvider.cs

@@ -17,13 +17,15 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost
     internal class DefaultScriptWebHookProvider : IScriptWebHookProvider
     {
         private readonly ISecretManagerProvider _secretManagerProvider;
+        private readonly HostNameProvider _hostNameProvider;
 
         // Map from an extension name to a http handler.
         private IDictionary<string, HttpHandler> _customHttpHandlers = new Dictionary<string, HttpHandler>(StringComparer.OrdinalIgnoreCase);
 
-        public DefaultScriptWebHookProvider(ISecretManagerProvider secretManagerProvider)
+        public DefaultScriptWebHookProvider(ISecretManagerProvider secretManagerProvider, HostNameProvider hostNameProvider)
         {
             _secretManagerProvider = secretManagerProvider;
+            _hostNameProvider = hostNameProvider;
         }
 
         public bool TryGetHandler(string name, out HttpHandler handler)
@@ -53,7 +55,7 @@ public Uri GetUrl(IExtensionConfigProvider extension)
         private Uri GetExtensionWebHookRoute(string extensionName)
         {
             var settings = ScriptSettingsManager.Instance;
-            var hostName = settings.GetSetting(EnvironmentSettingNames.AzureWebsiteHostName);
+            var hostName = _hostNameProvider.Value;
             if (hostName == null)
             {
                 return null;

src/WebJobs.Script.WebHost/WebHostServiceCollectionExtensions.cs

@@ -104,6 +104,7 @@ public static void AddWebJobsScriptHost(this IServiceCollection services, IConfi
             services.AddSingleton<IWebFunctionsManager, WebFunctionsManager>();
             services.AddSingleton<IInstanceManager, InstanceManager>();
             services.AddSingleton(_ => new HttpClient());
+            services.AddSingleton<HostNameProvider>();
             services.AddSingleton<IFileSystem>(_ => FileUtility.Instance);
             services.AddTransient<VirtualFileSystem>();
             services.AddTransient<VirtualFileSystemMiddleware>();