[HealthChecks] Add HealthChecks endpoint, register telemetry publisher (#11341)

* Hook up HealthCheckTelemetryPublisher

* Add /runtime/health/ endpoint

* Update health check tests

* Change HealthCheckMetrics meter name

* Update release_notes.md

* Require admin auth for health checks

* Add HealthCheck E2E tests

* remove host.json

* Add newlines

* Fix empty lines

* Move health API under /admin/health

* Add assertion message for failing test

Author: jviau

release_notes.md

@@ -16,4 +16,5 @@
 - Reduce allocations in `Utility.IsAzureMonitorLoggingEnabled` (#11323)
 - Update PowerShell worker to [4.0.4581](https://github.com/Azure/azure-functions-powershell-worker/releases/tag/v4.0.4581)
 - Bug fix that fails in-flight invocations when a worker channel shuts down (#11159)
+- Adds WebHost and ScriptHost health checks. (#11341, #11183, #11178, #11173, #11161)
 - Update Node.js Worker Version to [3.12.0](https://github.com/Azure/azure-functions-nodejs-worker/releases/tag/v3.12.0)

src/WebJobs.Script.WebHost/Controllers/FunctionsController.cs

@@ -12,7 +12,6 @@
 using Microsoft.AspNetCore.Http.Extensions;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Azure.WebJobs.Extensions.Http;
-using Microsoft.Azure.WebJobs.Host;
 using Microsoft.Azure.WebJobs.Script.Description;
 using Microsoft.Azure.WebJobs.Script.Management.Models;
 using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
@@ -28,7 +27,7 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost.Controllers
 {
     /// <summary>
     /// Controller responsible for administrative and management operations on functions
-    /// example retrieving a list of functions, invoking a function, creating a function, etc
+    /// example retrieving a list of functions, invoking a function, creating a function, etc.
     /// </summary>
     public class FunctionsController : Controller
     {

src/WebJobs.Script.WebHost/Controllers/HostController.cs

@@ -19,7 +19,6 @@
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.ExtensionBundle;
 using Microsoft.Azure.WebJobs.Script.Scale;
-using Microsoft.Azure.WebJobs.Script.WebHost.Extensions;
 using Microsoft.Azure.WebJobs.Script.WebHost.Filters;
 using Microsoft.Azure.WebJobs.Script.WebHost.Management;
 using Microsoft.Azure.WebJobs.Script.WebHost.Models;

src/WebJobs.Script.WebHost/Diagnostics/HealthChecks/HealthCheckAuthMiddleware.cs

@@ -0,0 +1,49 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Authentication;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Authorization.Policy;
+using Microsoft.AspNetCore.Http;
+using Microsoft.Azure.WebJobs.Script.WebHost.Security.Authorization.Policies;
+
+namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.HealthChecks
+{
+    public sealed class HealthCheckAuthMiddleware(
+        RequestDelegate next, IPolicyEvaluator policy, IAuthorizationPolicyProvider provider)
+    {
+        private readonly RequestDelegate _next = next ?? throw new ArgumentNullException(nameof(next));
+        private readonly IPolicyEvaluator _policy = policy ?? throw new ArgumentNullException(nameof(policy));
+        private readonly IAuthorizationPolicyProvider _provider = provider ?? throw new ArgumentNullException(nameof(provider));
+
+        public async Task InvokeAsync(HttpContext context)
+        {
+            ArgumentNullException.ThrowIfNull(context);
+
+            AuthorizationPolicy policy = await _provider.GetPolicyAsync(PolicyNames.AdminAuthLevel)
+                .ConfigureAwait(false);
+
+            AuthenticateResult authentication = await _policy.AuthenticateAsync(policy, context)
+                .ConfigureAwait(false);
+
+            if (!authentication.Succeeded)
+            {
+                context.Response.StatusCode = StatusCodes.Status401Unauthorized;
+                return;
+            }
+
+            PolicyAuthorizationResult authorization = await _policy.AuthorizeAsync(
+                policy, authentication, context, null).ConfigureAwait(false);
+
+            if (!authorization.Succeeded)
+            {
+                context.Response.StatusCode = StatusCodes.Status403Forbidden;
+                return;
+            }
+
+            await _next(context);
+        }
+    }
+}

src/WebJobs.Script.WebHost/WebJobsApplicationBuilderExtension.cs

@@ -3,10 +3,13 @@
 
 using System;
 using Microsoft.AspNetCore.Builder;
+using Microsoft.AspNetCore.Diagnostics.HealthChecks;
 using Microsoft.AspNetCore.Hosting;
+using Microsoft.AspNetCore.Http;
 using Microsoft.Azure.WebJobs.Extensions.Http;
 using Microsoft.Azure.WebJobs.Script.Extensions;
 using Microsoft.Azure.WebJobs.Script.WebHost.Configuration;
+using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics.HealthChecks;
 using Microsoft.Azure.WebJobs.Script.WebHost.Middleware;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Options;
@@ -41,6 +44,10 @@ public static IApplicationBuilder UseWebJobsScriptHost(this IApplicationBuilder
             builder.UseMiddleware<SystemTraceMiddleware>();
             builder.UseMiddleware<HandleCancellationMiddleware>();
             builder.UseMiddleware<HostnameFixupMiddleware>();
+
+            // Health is registered early in the pipeline to ensure it can avoid failures from the rest of the pipeline.
+            builder.UseHealthChecks();
+
             if (environment.IsAnyLinuxConsumption() || environment.IsAnyKubernetesEnvironment())
             {
                 builder.UseMiddleware<EnvironmentReadyCheckMiddleware>();
@@ -116,5 +123,44 @@ public static IApplicationBuilder UseWebJobsScriptHost(this IApplicationBuilder
 
             return builder;
         }
+
+        private static void UseHealthChecks(this IApplicationBuilder app)
+        {
+            // To start we are putting health under 'admin' to:
+            // 1. Avoid conflicts with function routes.
+            // 2. Allow for the same auth model as other admin APIs.
+            // 3. Ensure this is always available to platform callers.
+            // 4. Bypass easy-auth auth.
+            const string healthPrefix = "/admin/health";
+            static bool Predicate(HttpContext context)
+            {
+                return context.Request.Path.StartsWithSegments(healthPrefix);
+            }
+
+            app.MapWhen(Predicate, app =>
+            {
+                app.UseMiddleware<HealthCheckAuthMiddleware>();
+
+                // This supports the ?wait={seconds} query string.
+                app.UseMiddleware<HealthCheckWaitMiddleware>();
+
+                app.UseHealthChecks(healthPrefix, new HealthCheckOptions
+                {
+                    ResponseWriter = HealthCheckResponseWriter.WriteResponseAsync,
+                });
+
+                app.UseHealthChecks($"{healthPrefix}/live", new HealthCheckOptions
+                {
+                    Predicate = r => r.Tags.Contains("az.functions.liveness"),
+                    ResponseWriter = HealthCheckResponseWriter.WriteResponseAsync,
+                });
+
+                app.UseHealthChecks($"{healthPrefix}/ready", new HealthCheckOptions
+                {
+                    Predicate = r => r.Tags.Contains("az.functions.readiness"),
+                    ResponseWriter = HealthCheckResponseWriter.WriteResponseAsync,
+                });
+            });
+        }
     }
 }

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs

@@ -25,7 +25,8 @@ public static IHealthChecksBuilder AddWebJobsScriptHealthChecks(this IHealthChec
             ArgumentNullException.ThrowIfNull(builder);
             builder
                 .AddWebHostHealthCheck()
-                .AddScriptHostHealthCheck();
+                .AddScriptHostHealthCheck()
+                .AddTelemetryPublisher(HealthCheckTags.Liveness, HealthCheckTags.Readiness);
             return builder;
         }
 

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckMetrics.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Diagnostics.Metrics;
+using Microsoft.Azure.WebJobs.Script.Metrics;
 
 namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
 {
@@ -22,7 +23,7 @@ public HealthCheckMetrics(IMeterFactory meterFactory)
             // We don't dispose the meter because IMeterFactory handles that
             // An issue on analyzer side: https://github.com/dotnet/roslyn-analyzers/issues/6912
             // Related documentation: https://github.com/dotnet/docs/pull/37170
-            Meter meter = meterFactory.Create("Microsoft.Azure.WebJobs.Script");
+            Meter meter = meterFactory.Create(HostMetrics.FaasMeterName, HostMetrics.FaasMeterVersion);
 #pragma warning restore CA2000 // Dispose objects before losing scope
 
             HealthCheckReport = HealthCheckMetricsGeneration.CreateHealthCheckReportHistogram(meter);

src/WebJobs.Script/Metrics/HostMetrics.cs

@@ -25,6 +25,8 @@ public class HostMetrics : IHostMetrics
         // FaaS Metrics
         public const string FaasInvokeDuration = "faas.invoke_duration";
 
+        public static readonly string FaasMeterVersion = typeof(HostMetrics).Assembly.GetName().Version?.ToString();
+
         private Counter<long> _appFailureCount;
         private Counter<long> _startedInvocationCount;
         private Histogram<double> _faasInvokeDuration;
@@ -65,7 +67,7 @@ public HostMetrics(IMeterFactory meterFactory, IEnvironment environment, ILogger
             _appFailureCount = meter.CreateCounter<long>(AppFailureCount, "numeric", "Number of times the host has failed to start.");
             _startedInvocationCount = meter.CreateCounter<long>(StartedInvocationCount, "numeric", "Number of function invocations that have started.");
 
-            var faasMeter = meterFactory.Create(new MeterOptions(FaasMeterName));
+            var faasMeter = meterFactory.Create(new MeterOptions(FaasMeterName) { Version = FaasMeterVersion });
             _faasInvokeDuration = faasMeter.CreateHistogram<double>(
                 name: FaasInvokeDuration,
                 unit: "s",

test/WebJobs.Script.Tests.Integration/WebHostEndToEnd/SamplesEndToEndTests_CSharp.cs

@@ -8,6 +8,7 @@
 using System.Net;
 using System.Net.Http;
 using System.Net.Http.Headers;
+using System.Net.Http.Json;
 using System.Reflection;
 using System.Text;
 using System.Threading;
@@ -316,6 +317,35 @@ public async Task HostPing_Succeeds(string method)
             Assert.Equal("no-store, no-cache", cacheHeader);
         }
 
+        [Theory]
+        [InlineData("/admin/health")]
+        [InlineData("/admin/health/live")]
+        [InlineData("/admin/health/ready")]
+        public async Task HealthCheck_AdminToken_Succeeds(string uri)
+        {
+            // token specified as bearer token
+            HttpRequestMessage request = new(HttpMethod.Get, uri);
+            string token = _fixture.Host.GenerateAdminJwtToken();
+            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token);
+            HttpResponseMessage response = await _fixture.Host.HttpClient.SendAsync(request);
+            Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+
+            string body = await response.Content.ReadAsStringAsync();
+            Assert.Equal("{\"status\":\"Healthy\"}", body);
+        }
+
+        [Theory]
+        [InlineData("/admin/health")]
+        [InlineData("/admin/health/live")]
+        [InlineData("/admin/health/ready")]
+        public async Task HealthCheck_NoAdminToken_Fail(string uri)
+        {
+            // token specified as bearer token
+            HttpRequestMessage request = new(HttpMethod.Get, uri);
+            HttpResponseMessage response = await _fixture.Host.HttpClient.SendAsync(request);
+            Assert.Equal(HttpStatusCode.Unauthorized, response.StatusCode);
+        }
+
         [Fact]
         public async Task InstallExtensionsEnsureOldPathReturns404()
         {
@@ -497,8 +527,8 @@ public async Task HostLog_AdminLevel_Succeeds()
         {
             var request = new HttpRequestMessage(HttpMethod.Post, "admin/host/log");
             request.Headers.Add(AuthenticationLevelHandler.FunctionsKeyHeaderName, await _fixture.Host.GetMasterKeyAsync());
-            var logs = new HostLogEntry[]
-            {
+            HostLogEntry[] logs =
+            [
                 new HostLogEntry
                 {
                     Level = System.Diagnostics.TraceLevel.Verbose,
@@ -524,13 +554,9 @@ public async Task HostLog_AdminLevel_Succeeds()
                     FunctionName = "TestFunction",
                     Message = string.Format("Test Error log {0}", Guid.NewGuid().ToString())
                 }
-            };
-            var serializer = new JsonSerializer();
-            var writer = new StringWriter();
-            serializer.Serialize(writer, logs);
-            var json = writer.ToString();
-            request.Content = new StringContent(json);
-            request.Content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
+            ];
+
+            request.Content = JsonContent.Create(logs);
 
             var response = await _fixture.Host.HttpClient.SendAsync(request);
             Assert.Equal(HttpStatusCode.OK, response.StatusCode);
@@ -540,7 +566,9 @@ public async Task HostLog_AdminLevel_Succeeds()
             var hostLogs = _fixture.Host.GetScriptHostLogMessages();
             foreach (var expectedLog in logs.Select(p => p.Message))
             {
-                Assert.Equal(1, hostLogs.Count(p => p.FormattedMessage != null && p.FormattedMessage.Contains(expectedLog)));
+                Assert.True(
+                    1 == hostLogs.Count(p => p.FormattedMessage != null && p.FormattedMessage.Contains(expectedLog)),
+                    $"Expected log message '{expectedLog}' not found. Log count: {hostLogs.Count}");
             }
         }
 

test/WebJobs.Script.Tests/Diagnostics/HealthChecks/HealthCheckExtensionsTests.cs

@@ -50,10 +50,12 @@ public void AddScriptHostHealthCheck_ThrowsOnNullBuilder()
         }
 
         [Fact]
-        public void AddWebJobsScriptHealthChecks_RegistersBothHealthChecks()
+        public void AddWebJobsScriptHealthChecks_RegistersExpectedServices()
         {
             // arrange
+            ServiceCollection services = new();
             Mock<IHealthChecksBuilder> builder = new(MockBehavior.Strict);
+            builder.Setup(b => b.Services).Returns(services);
             builder.Setup(b => b.Add(It.IsAny<HealthCheckRegistration>())).Returns(builder.Object);
 
             // act
@@ -67,7 +69,10 @@ public void AddWebJobsScriptHealthChecks_RegistersBothHealthChecks()
             builder.Verify(b => b.Add(IsRegistration<ScriptHostHealthCheck>(
                 HealthCheckNames.ScriptHostLifeCycle, HealthCheckTags.Readiness)),
                 Times.Once);
+            builder.Verify(b => b.Services, Times.AtLeastOnce);
             builder.VerifyNoOtherCalls();
+
+            VerifyPublishers(services, null, HealthCheckTags.Liveness, HealthCheckTags.Readiness);
         }
 
         [Fact]
@@ -216,17 +221,7 @@ public void AddTelemetryPublisher_RegistersExpected(string[] tags, string[] expe
             builder.AddTelemetryPublisher(tags);
 
             // assert
-            services.Where(x => x.ServiceType == typeof(IHealthCheckPublisher)).Should().HaveCount(expected.Length)
-                .And.AllSatisfy(x => x.Lifetime.Should().Be(ServiceLifetime.Singleton));
-
-            ServiceProvider provider = services.BuildServiceProvider();
-            IEnumerable<IHealthCheckPublisher> publishers = provider.GetServices<IHealthCheckPublisher>();
-
-            publishers.Should().HaveCount(expected.Length);
-            foreach (string tag in expected)
-            {
-                publishers.Should().ContainSingle(p => VerifyPublisher(p, tag));
-            }
+            VerifyPublishers(services, expected);
         }
 
         private static HealthCheckRegistration IsRegistration<T>(string name, string tag)
@@ -248,6 +243,21 @@ static bool IsType(HealthCheckRegistration registration)
             });
         }
 
+        private static void VerifyPublishers(IServiceCollection services, params string[] tags)
+        {
+            services.Where(x => x.ServiceType == typeof(IHealthCheckPublisher)).Should().HaveCount(tags.Length)
+                .And.AllSatisfy(x => x.Lifetime.Should().Be(ServiceLifetime.Singleton));
+
+            ServiceProvider provider = services.BuildServiceProvider();
+            IEnumerable<IHealthCheckPublisher> publishers = provider.GetServices<IHealthCheckPublisher>();
+
+            publishers.Should().HaveCount(tags.Length);
+            foreach (string tag in tags)
+            {
+                publishers.Should().ContainSingle(p => VerifyPublisher(p, tag));
+            }
+        }
+
         private static bool VerifyPublisher(IHealthCheckPublisher publisher, string tag)
         {
             return publisher is TelemetryHealthCheckPublisher telemetryPublisher