[HealthChecks] Add TelemetryHealthCheckPublisher

eng/build/Engineering.props

@@ -16,7 +16,7 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <LangVersion>latest</LangVersion>
+    <LangVersion>preview</LangVersion>
     <AssemblyOriginatorKeyFile>$(EngResourceRoot)key.snk</AssemblyOriginatorKeyFile>
     <CodeAnalysisRuleSet>$(RepoRoot)src.ruleset</CodeAnalysisRuleSet>
     <NoWarn>$(NoWarn);NU1701;NU5104</NoWarn>

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckExtensions.cs

@@ -0,0 +1,70 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    internal static class HealthCheckExtensions
+    {
+        /// <summary>
+        /// Registers the telemetry health check publisher with the specified additional tags.
+        /// NOTE: this is currently not safe to call multiple times.
+        /// </summary>
+        /// <param name="builder">The builder to register to.</param>
+        /// <param name="additionalTags">Registers addition copies of the publisher for these tags.</param>
+        /// <returns>The original health check builder, for call chaining.</returns>
+        public static IHealthChecksBuilder AddTelemetryPublisher(
+            this IHealthChecksBuilder builder, params string[] additionalTags)
+        {
+            ArgumentNullException.ThrowIfNull(builder);
+
+            static void RegisterPublisher(IServiceCollection services, string tag)
+            {
+                services.AddSingleton<IHealthCheckPublisher>(sp =>
+                {
+                    TelemetryHealthCheckPublisherOptions options = new() { Tag = tag };
+                    return ActivatorUtilities.CreateInstance<TelemetryHealthCheckPublisher>(sp, options);
+                });
+            }
+
+            builder.Services.AddLogging();
+            builder.Services.AddMetrics();
+            builder.Services.AddSingleton<HealthCheckMetrics>();
+            RegisterPublisher(builder.Services, null); // always register the default publisher
+
+            additionalTags ??= [];
+            foreach (string tag in additionalTags.Distinct(StringComparer.OrdinalIgnoreCase))
+            {
+                if (!string.IsNullOrEmpty(tag))
+                {
+                    RegisterPublisher(builder.Services, tag);
+                }
+            }
+
+            return builder;
+        }
+
+        /// <summary>
+        /// Filters a health report to include only specified entries.
+        /// </summary>
+        /// <param name="result">The result to filter.</param>
+        /// <param name="filter">The filter predicate to use.</param>
+        /// <returns>The filtered health report.</returns>
+        public static HealthReport Filter(this HealthReport result, Func<string, HealthReportEntry, bool> filter)
+        {
+            ArgumentNullException.ThrowIfNull(result);
+            ArgumentNullException.ThrowIfNull(filter);
+
+            IReadOnlyDictionary<string, HealthReportEntry> newEntries = result.Entries
+                .Where(x => filter(x.Key, x.Value))
+                .ToDictionary();
+
+            return new HealthReport(newEntries, result.TotalDuration);
+        }
+    }
+}

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckMetrics.cs

@@ -0,0 +1,50 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics.Metrics;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    /// <summary>
+    /// Contains metrics for health checks.
+    /// </summary>
+    /// <remarks>
+    /// Code taken from: https://github.com/dotnet/extensions/blob/d32357716a5261509bf7527101b21cb6f94a0f89/src/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.Common/HealthCheckMetrics.cs.
+    /// </remarks>
+    public sealed partial class HealthCheckMetrics
+    {
+        public HealthCheckMetrics(IMeterFactory meterFactory)
+        {
+            ArgumentNullException.ThrowIfNull(meterFactory);
+
+#pragma warning disable CA2000 // Dispose objects before losing scope
+            // We don't dispose the meter because IMeterFactory handles that
+            // An issue on analyzer side: https://github.com/dotnet/roslyn-analyzers/issues/6912
+            // Related documentation: https://github.com/dotnet/docs/pull/37170
+            Meter meter = meterFactory.Create("Microsoft.Azure.WebJobs.Script");
+#pragma warning restore CA2000 // Dispose objects before losing scope
+
+            HealthCheckReport = HealthCheckMetricsGeneration.CreateHealthCheckReportHistogram(meter);
+            UnhealthyHealthCheck = HealthCheckMetricsGeneration.CreateUnhealthyHealthCheckHistogram(meter);
+        }
+
+        /// <summary>
+        /// Gets the health check report histogram.
+        /// </summary>
+        public HealthCheckReportHistogram HealthCheckReport { get; }
+
+        /// <summary>
+        /// Gets the unhealthy health check histogram.
+        /// </summary>
+        public UnhealthyHealthCheckHistogram UnhealthyHealthCheck { get; }
+
+        public static class Constants
+        {
+            public const string ReportMetricName = "az.functions.health_check.reports";
+            public const string UnhealthyMetricName = "az.functions.health_check.unhealthy_checks";
+            public const string HealthCheckTagTag = "az.functions.health_check.tag"; // Yes, tag tag. A metric tag with 'tag' in the name.
+            public const string HealthCheckNameTag = "az.functions.health_check.name";
+        }
+    }
+}

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckMetricsGeneration.cs

@@ -0,0 +1,46 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Diagnostics.Metrics;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Diagnostics.Metrics;
+using static Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks.HealthCheckMetrics;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    /// <summary>
+    /// Health check metrics generation and extensions.
+    /// </summary>
+    /// <remarks>
+    /// Code adapted from: https://github.com/dotnet/extensions/blob/d32357716a5261509bf7527101b21cb6f94a0f89/src/Libraries/Microsoft.Extensions.Diagnostics.HealthChecks.Common/Metric.cs.
+    /// </remarks>
+    public static partial class HealthCheckMetricsGeneration
+    {
+        [Histogram<double>(Constants.HealthCheckTagTag, Name = Constants.ReportMetricName)]
+        public static partial HealthCheckReportHistogram CreateHealthCheckReportHistogram(Meter meter);
+
+        [Histogram<double>(
+            Constants.HealthCheckNameTag, Constants.HealthCheckTagTag,
+            Name = Constants.UnhealthyMetricName)]
+        public static partial UnhealthyHealthCheckHistogram CreateUnhealthyHealthCheckHistogram(Meter meter);
+
+        public static void Record(this HealthCheckReportHistogram histogram, HealthReport report, string tag)
+        {
+            ArgumentNullException.ThrowIfNull(report);
+            histogram.Record(ToMetricValue(report.Status), tag);
+        }
+
+        public static void Record(this UnhealthyHealthCheckHistogram histogram, string name, HealthReportEntry entry, string tag)
+            => histogram.Record(ToMetricValue(entry.Status), name, tag);
+
+        private static double ToMetricValue(HealthStatus status)
+            => status switch
+            {
+                HealthStatus.Unhealthy => 0,
+                HealthStatus.Degraded => 0.5,
+                HealthStatus.Healthy => 1,
+                _ => throw new NotSupportedException($"Unexpected HealthStatus value: {status}"),
+            };
+    }
+}

src/WebJobs.Script/Diagnostics/HealthChecks/HealthCheckTags.cs

@@ -0,0 +1,16 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    internal static class HealthCheckTags
+    {
+        private const string Prefix = "az.functions";
+
+        public const string Liveness = Prefix + ".liveness";
+
+        public const string Readiness = Prefix + ".readiness";
+
+        public const string Configuration = Prefix + ".configuration";
+    }
+}

src/WebJobs.Script/Diagnostics/HealthChecks/TelemetryHealthCheckPublisher.cs

@@ -0,0 +1,124 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Azure.WebJobs.Script.Pools;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.Azure.WebJobs.Script.Diagnostics.HealthChecks
+{
+    public partial class TelemetryHealthCheckPublisher : IHealthCheckPublisher
+    {
+        private readonly HealthCheckMetrics _metrics;
+        private readonly TelemetryHealthCheckPublisherOptions _options;
+        private readonly ILogger _logger;
+
+        public TelemetryHealthCheckPublisher(
+            HealthCheckMetrics metrics,
+            TelemetryHealthCheckPublisherOptions options,
+            ILogger<TelemetryHealthCheckPublisher> logger)
+        {
+            _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
+            _options = options ?? throw new ArgumentNullException(nameof(options));
+            _logger = logger ?? throw new ArgumentNullException(nameof(logger));
+        }
+
+        /// <summary>
+        /// Gets the tag for this health check publisher. For unit test purposes only.
+        /// </summary>
+        internal string Tag => _options.Tag;
+
+        public Task PublishAsync(HealthReport report, CancellationToken cancellationToken)
+        {
+            ArgumentNullException.ThrowIfNull(report);
+
+            if (_options.Tag is string tag)
+            {
+                report = report.Filter(FilterForTag);
+            }
+
+            if (report.Entries.Count == 0)
+            {
+                // No entries to report, so we can skip logging and metrics.
+                return Task.CompletedTask;
+            }
+
+            tag = _options.Tag ?? string.Empty; // for logs/metrics later on.
+            if (report.Status == HealthStatus.Healthy)
+            {
+                if (!_options.LogOnlyUnhealthy)
+                {
+                    Log.Healthy(_logger, tag, report.Status);
+                }
+            }
+            else
+            {
+                // Construct string showing list of all health entries status and description for logs
+                using PoolRental<StringBuilder> rental = PoolFactory.SharedStringBuilderPool.Rent();
+                string separator = string.Empty;
+                foreach (var entry in report.Entries)
+                {
+                    if (entry.Value.Status != HealthStatus.Healthy)
+                    {
+                        _metrics.UnhealthyHealthCheck.Record(entry.Key, entry.Value, tag);
+                    }
+
+                    rental.Value.Append(separator)
+                        .Append(entry.Key)
+                        .Append(": {")
+                        .Append("status: ")
+                        .Append(entry.Value.Status.ToString())
+                        .Append(", description: ")
+                        .Append(entry.Value.Description)
+                        .Append('}');
+
+                    separator = ", ";
+                }
+
+                Log.Unhealthy(_logger, tag, report.Status, rental.Value);
+            }
+
+            _metrics.HealthCheckReport.Record(report, tag);
+            return Task.CompletedTask;
+        }
+
+        private bool FilterForTag(string name, HealthReportEntry entry)
+        {
+            return entry.Tags.Contains(_options.Tag);
+        }
+
+        internal static partial class Log
+        {
+            [LoggerMessage(0, LogLevel.Warning, "[Tag='{Tag}'] Process reporting unhealthy: {Status}. Health check entries are {Entries}")]
+            public static partial void Unhealthy(
+                ILogger logger,
+                string tag,
+                HealthStatus status,
+                StringBuilder entries);
+
+            [LoggerMessage(1, LogLevel.Debug, "[Tag='{Tag}'] Process reporting healthy: {Status}.")]
+            public static partial void Healthy(
+                ILogger logger,
+                string tag,
+                HealthStatus status);
+        }
+    }
+
+    public class TelemetryHealthCheckPublisherOptions
+    {
+        /// <summary>
+        /// Gets or sets a value indicating whether to log for only non-healthy values or not. Default <c>true</c>.
+        /// </summary>
+        public bool LogOnlyUnhealthy { get; set; } = true;
+
+        /// <summary>
+        /// Gets or sets the tag to filter this health check for. <c>null</c> will perform no filtering.
+        /// </summary>
+        public string Tag { get; set; }
+    }
+}

src/WebJobs.Script/Pools/ObjectPoolExtensions.cs

@@ -0,0 +1,28 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the MIT License. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.Extensions.ObjectPool;
+
+namespace Microsoft.Azure.WebJobs.Script.Pools
+{
+    internal static class ObjectPoolExtensions
+    {
+        /// <summary>
+        /// Rents a <see cref="T"/> from the pool. The object is returned on disposal
+        /// of the <see cref="PoolRental"/>.
+        /// </summary>
+        /// <typeparam name="T">The object type the pool holds.</typeparam>
+        /// <param name="pool">The pool to rent from.</param>
+        /// <returns>
+        /// A disposable struct to return the object to the pool.
+        /// DO NOT dispose multiple times.
+        /// </returns>
+        public static PoolRental<T> Rent<T>(this ObjectPool<T> pool)
+            where T : class
+        {
+            ArgumentNullException.ThrowIfNull(pool);
+            return new PoolRental<T>(pool);
+        }
+    }
+}