feat(operations): add healthchecks and metrics.

This adds a prometheus metrics endpoint (configurable via operator
settings) that collects information about the running
system.

Also healthchecks (divided into two categories "readiness"
and "liveness") are added to check up your system.

This closes #4.

Signed-off-by: Christoph Bühler <[email protected]>

Author: Christoph Bühler

src/KubeOps/KubeOps.csproj

@@ -28,7 +28,10 @@
         <PackageReference Include="KubernetesClient" Version="2.0.29" />
         <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="3.0.0" />
         <PackageReference Include="McMaster.Extensions.Hosting.CommandLine" Version="3.0.0" />
+        <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="3.1.7" />
         <PackageReference Include="Namotion.Reflection" Version="1.0.12" />
+        <PackageReference Include="prometheus-net.AspNetCore" Version="3.6.0" />
+        <PackageReference Include="prometheus-net.AspNetCore.HealthChecks" Version="3.6.0" />
         <PackageReference Include="YamlDotNet" Version="8.1.2" />
     </ItemGroup>
 

src/KubeOps/Operator/Caching/ResourceCache.cs

@@ -1,9 +1,10 @@
-using System.Collections.Generic;
+using System.Collections.Concurrent;
+using System.Collections.Generic;
 using System.Linq;
 using k8s;
 using k8s.Models;
 using KellermanSoftware.CompareNetObjects;
-using KubeOps.Operator.Entities;
+using KubeOps.Operator.DevOps;
 using KubeOps.Operator.Entities.Extensions;
 
 namespace KubeOps.Operator.Caching
@@ -23,7 +24,9 @@ internal class ResourceCache<TEntity> : IResourceCache<TEntity>
                 MembersToIgnore = new List<string> { ResourceVersion },
             });
 
-        private readonly IDictionary<string, TEntity> _cache = new Dictionary<string, TEntity>();
+        private readonly IDictionary<string, TEntity> _cache = new ConcurrentDictionary<string, TEntity>();
+
+        private readonly ResourceCacheMetrics<TEntity> _metrics = new ResourceCacheMetrics<TEntity>();
 
         public TEntity Get(string id) => _cache[id];
 
@@ -40,6 +43,8 @@ public TEntity Upsert(TEntity resource, out CacheComparisonResult result)
                 _cache[resource.Metadata.Uid] = resource.DeepClone();
             }
 
+            _metrics.CachedItemsSize.Set(_cache.Count);
+            _metrics.CachedItemsSummary.Observe(_cache.Count);
             return resource;
         }
 
@@ -72,12 +77,22 @@ private CacheComparisonResult CompareCache(TEntity resource)
 
         public void Remove(TEntity resource) => Remove(resource.Metadata.Uid);
 
-        public void Clear() => _cache.Clear();
+        public void Clear()
+        {
+            _cache.Clear();
+            _metrics.CachedItemsSize.Set(_cache.Count);
+            _metrics.CachedItemsSummary.Observe(_cache.Count);
+        }
 
         private bool Exists(TEntity resource) => _cache.ContainsKey(resource.Metadata.Uid);
 
         private bool Exists(string id) => _cache.ContainsKey(id);
 
-        private void Remove(string resourceUid) => _cache.Remove(resourceUid);
+        private void Remove(string resourceUid)
+        {
+            _cache.Remove(resourceUid);
+            _metrics.CachedItemsSize.Set(_cache.Count);
+            _metrics.CachedItemsSummary.Observe(_cache.Count);
+        }
     }
 }

src/KubeOps/Operator/Comparing/ReferenceEqualityComparer.cs

@@ -4,8 +4,8 @@ namespace KubeOps.Operator.Comparing
 {
     internal class ReferenceEqualityComparer : EqualityComparer<object>
     {
-        public override bool Equals(object x, object y) => ReferenceEquals(x, y);
+        public override bool Equals(object? x, object? y) => ReferenceEquals(x, y);
 
-        public override int GetHashCode(object obj) => obj == null ? 0 : obj.GetHashCode();
+        public override int GetHashCode(object? obj) => obj == null ? 0 : obj.GetHashCode();
     }
 }

src/KubeOps/Operator/Controller/IResourceController.cs

@@ -4,7 +4,12 @@
 
 namespace KubeOps.Operator.Controller
 {
-    public interface IResourceController<TEntity> : IHostedService
+    public interface IResourceController : IHostedService
+    {
+        internal bool Running { get; }
+    }
+
+    public interface IResourceController<TEntity> : IResourceController
         where TEntity : IKubernetesObject<V1ObjectMeta>
     {
     }

src/KubeOps/Operator/Controller/ResourceControllerBase.cs

@@ -29,6 +29,7 @@ public abstract class ResourceControllerBase<TEntity> : IResourceController<TEnt
 
         private readonly ILogger<ResourceControllerBase<TEntity>> _logger;
         private readonly IResourceEventQueue<TEntity> _eventQueue;
+        private bool _running;
 
         protected ResourceControllerBase()
             : this(
@@ -48,6 +49,8 @@ protected ResourceControllerBase(
             Client = client;
         }
 
+        bool IResourceController.Running => _running;
+
         protected IKubernetesClient Client { get; }
 
         public async Task StartAsync(CancellationToken cancellationToken)
@@ -56,6 +59,7 @@ public async Task StartAsync(CancellationToken cancellationToken)
 
             _eventQueue.ResourceEvent += OnResourceEvent;
             await _eventQueue.Start();
+            _running = true;
         }
 
         public Task StopAsync(CancellationToken cancellationToken)
@@ -64,7 +68,7 @@ public Task StopAsync(CancellationToken cancellationToken)
 
             _eventQueue.Stop();
             _eventQueue.ResourceEvent -= OnResourceEvent;
-
+            _running = false;
             return Task.CompletedTask;
         }
 

src/KubeOps/Operator/DevOps/ControllerLivenessCheck.cs

@@ -0,0 +1,40 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using KubeOps.Operator.Controller;
+using Microsoft.Extensions.Diagnostics.HealthChecks;
+using Microsoft.Extensions.Hosting;
+
+namespace KubeOps.Operator.DevOps
+{
+    internal class ControllerLivenessCheck : IHealthCheck
+    {
+        private readonly IList<IResourceController> _controller;
+
+        public ControllerLivenessCheck(IEnumerable<IHostedService> services)
+        {
+            _controller = services
+                .Where(s => s is IResourceController)
+                .OfType<IResourceController>()
+                .ToList();
+        }
+
+        public Task<HealthCheckResult> CheckHealthAsync(
+            HealthCheckContext context,
+            CancellationToken cancellationToken = new CancellationToken())
+        {
+            if (_controller.All(c => c.Running))
+            {
+                return Task.FromResult(HealthCheckResult.Healthy("all controllers are running."));
+            }
+
+            return Task.FromResult(
+                HealthCheckResult.Unhealthy(
+                    "some controllers are not running.",
+                    data: _controller.ToDictionary(
+                        c => c.GetType().Name,
+                        c => $"running: {c.Running}" as object)));
+        }
+    }
+}

src/KubeOps/Operator/DevOps/ResourceCacheMetrics.cs

@@ -0,0 +1,54 @@
+using k8s;
+using k8s.Models;
+using KubeOps.Operator.DependencyInjection;
+using KubeOps.Operator.Entities.Extensions;
+using Microsoft.Extensions.DependencyInjection;
+using Prometheus;
+
+namespace KubeOps.Operator.DevOps
+{
+    internal class ResourceCacheMetrics<TEntity>
+        where TEntity : IKubernetesObject<V1ObjectMeta>
+    {
+        private static readonly string[] Labels =
+        {
+            "operator",
+            "kind",
+            "group",
+            "version",
+            "scope"
+        };
+
+        public ResourceCacheMetrics()
+        {
+            var settings = DependencyInjector.Services.GetRequiredService<OperatorSettings>();
+            var crd = CustomEntityDefinitionExtensions.CreateResourceDefinition<TEntity>();
+            var labelValues = new[]
+            {
+                settings.Name,
+                crd.Kind,
+                crd.Group,
+                crd.Version,
+                crd.Scope.ToString()
+            };
+
+            CachedItemsSummary = Metrics
+                .CreateSummary(
+                    "operator_resource_cached_items_size",
+                    "Summary of the cached items count over the last 10 minutes",
+                    Labels)
+                .WithLabels(labelValues);
+
+            CachedItemsSize = Metrics
+                .CreateGauge(
+                    "operator_resource_cached_items_count",
+                    "Total number of cached items in this resource cache",
+                    Labels)
+                .WithLabels(labelValues);
+        }
+
+        public Summary.Child CachedItemsSummary { get; }
+
+        public Gauge.Child CachedItemsSize { get; }
+    }
+}