feat: add preload cache mechanism via settings (#114)

This closes #58. Note that preload is done on leader election
as well as "each start" of the specific controller.

This could lead to "not catched" resources.

Author: buehler

docs/docs/commands.md

@@ -23,7 +23,7 @@ Here is a brief overview over the available commands:
 > all commands assume either the compiled dll or you using
 > `dotnet run -- ` as prepended command.
 
-- ` ` (empty): runs the operator (normal `dotnet run`)
+- `""` (empty): runs the operator (normal `dotnet run`)
 - `version`: prints the version information for the actual connected kubernetes cluster
 - `install`: install the CRDs for the solution into the cluster
 - `uninstall`: uninstall the CRDs for the solution from the cluster

src/KubeOps/Operator/Builder/OperatorBuilder.cs

@@ -137,7 +137,7 @@ internal IOperatorBuilder AddOperatorBase(OperatorSettings settings)
 
             Services.AddTransient<IKubernetesClient, KubernetesClient>();
 
-            Services.AddTransient(typeof(IResourceCache<>), typeof(ResourceCache<>));
+            Services.AddSingleton(typeof(IResourceCache<>), typeof(ResourceCache<>));
             Services.AddTransient(typeof(IResourceWatcher<>), typeof(ResourceWatcher<>));
             Services.AddTransient(typeof(IResourceEventQueue<>), typeof(ResourceEventQueue<>));
             Services.AddTransient(typeof(IResourceServices<>), typeof(ResourceServices<>));

src/KubeOps/Operator/Caching/IResourceCache.cs

@@ -1,17 +1,53 @@
-using k8s;
+using System.Collections.Generic;
+using k8s;
 using k8s.Models;
+using KubeOps.Operator.Queue;
 
 namespace KubeOps.Operator.Caching
 {
+    /// <summary>
+    /// Resource cache for comparing objects and determine
+    /// a <see cref="CacheComparisonResult"/>. This result
+    /// is used to determine the event type for the <see cref="ResourceEventQueue{TEntity}"/>.
+    /// </summary>
+    /// <typeparam name="TEntity">The type of objects that are cached.</typeparam>
     public interface IResourceCache<TEntity>
         where TEntity : IKubernetesObject<V1ObjectMeta>
     {
+        /// <summary>
+        /// Return an object from the cache.
+        /// </summary>
+        /// <param name="id">String id of the object.</param>
+        /// <returns>The found entity.</returns>
+        /// <exception cref="KeyNotFoundException">When the item does not exist.</exception>
         TEntity Get(string id);
 
+        /// <summary>
+        /// Insert or Update a given resource in the cache and determine the
+        /// <see cref="CacheComparisonResult"/>.
+        /// </summary>
+        /// <param name="resource">The resource in question.</param>
+        /// <param name="result"><see cref="CacheComparisonResult"/> for the given resource.</param>
+        /// <returns>The inserted resource.</returns>
         TEntity Upsert(TEntity resource, out CacheComparisonResult result);
 
+        /// <summary>
+        /// Prefill the cache with a list of entities.
+        /// This does not delete other items in the cache.
+        /// </summary>
+        /// <param name="entities">List of entities.</param>
+        void Fill(IEnumerable<TEntity> entities);
+
+        /// <summary>
+        /// Remove an entity from the cache.
+        /// If the resource is not present, this turns to a no-op.
+        /// </summary>
+        /// <param name="resource">The resource in question.</param>
         void Remove(TEntity resource);
 
+        /// <summary>
+        /// Clear the whole cache.
+        /// </summary>
         public void Clear();
     }
 }

src/KubeOps/Operator/Caching/ResourceCache.cs

@@ -18,20 +18,20 @@ internal class ResourceCache<TEntity> : IResourceCache<TEntity>
         private const string Status = "Status";
 
         private readonly CompareLogic _compare = new(
-            new ComparisonConfig
+            new()
             {
                 Caching = true,
                 AutoClearCache = false,
-                MembersToIgnore = new List<string> { ResourceVersion, ManagedFields },
+                MembersToIgnore = new() { ResourceVersion, ManagedFields },
             });
 
-        private readonly IDictionary<string, TEntity> _cache = new ConcurrentDictionary<string, TEntity>();
+        private readonly ConcurrentDictionary<string, TEntity> _cache = new();
 
         private readonly ResourceCacheMetrics<TEntity> _metrics;
 
         public ResourceCache(OperatorSettings settings)
         {
-            _metrics = new ResourceCacheMetrics<TEntity>(settings);
+            _metrics = new(settings);
         }
 
         public TEntity Get(string id) => _cache[id];
@@ -40,18 +40,24 @@ public TEntity Upsert(TEntity resource, out CacheComparisonResult result)
         {
             result = CompareCache(resource);
 
-            if (result == CacheComparisonResult.New)
-            {
-                _cache.Add(resource.Metadata.Uid, resource.DeepClone());
-            }
-            else
+            var clone = resource.DeepClone();
+            _cache.AddOrUpdate(resource.Metadata.Uid, clone, (_, _) => clone);
+
+            _metrics.CachedItemsSize.Set(_cache.Count);
+            _metrics.CachedItemsSummary.Observe(_cache.Count);
+            return resource;
+        }
+
+        public void Fill(IEnumerable<TEntity> entities)
+        {
+            foreach (var entity in entities)
             {
-                _cache[resource.Metadata.Uid] = resource.DeepClone();
+                var clone = entity.DeepClone();
+                _cache.AddOrUpdate(entity.Metadata.Uid, clone, (_, _) => clone);
             }
 
             _metrics.CachedItemsSize.Set(_cache.Count);
             _metrics.CachedItemsSummary.Observe(_cache.Count);
-            return resource;
         }
 
         public void Remove(TEntity resource) => Remove(resource.Metadata.Uid);
@@ -94,7 +100,7 @@ private CacheComparisonResult CompareCache(TEntity resource)
 
         private void Remove(string resourceUid)
         {
-            _cache.Remove(resourceUid);
+            _cache.TryRemove(resourceUid, out _);
             _metrics.CachedItemsSize.Set(_cache.Count);
             _metrics.CachedItemsSummary.Observe(_cache.Count);
         }

src/KubeOps/Operator/Controller/ResourceControllerBase.cs

@@ -130,6 +130,14 @@ private async void LeadershipChanged(object? sender, LeaderState state)
             if (state == LeaderState.Leader)
             {
                 _logger.LogInformation("This instance was elected as leader, starting event queue.");
+
+                if (_services.Settings.PreloadCache)
+                {
+                    _logger.LogInformation("The 'preload cache' setting is set to 'true'.");
+                    var items = await _services.Client.List<TEntity>(_services.Settings.Namespace);
+                    _services.ResourceCache.Fill(items);
+                }
+
                 _services.EventQueue.ResourceEvent += OnResourceEvent;
                 await _services.EventQueue.Start();
 

src/KubeOps/Operator/OperatorSettings.cs

@@ -62,5 +62,16 @@ public sealed class OperatorSettings
         /// The timeout in seconds which the watcher has (after this timeout, the server will close the connection).
         /// </summary>
         public ushort WatcherHttpTimeout { get; set; } = 60;
+
+        /// <summary>
+        /// If set to true, controllers perform a search for already
+        /// existing objects in the cluster and load them into the objects cache.
+        ///
+        /// This bears the risk of not catching elements when they are created
+        /// during downtime of the operator.
+        ///
+        /// The search will be performed on each "Start" of the controller.
+        /// </summary>
+        public bool PreloadCache { get; set; }
     }
 }