Calculate and compare provider configMap cache before reconcile

Signed-off-by: Danil-Grigorev <[email protected]>

Author: Danil-Grigorev

internal/controller/genericprovider_controller.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"hash"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -39,7 +40,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	utilyaml "sigs.k8s.io/cluster-api/util/yaml"
 )
 
 type GenericProviderReconciler struct {
@@ -56,6 +60,7 @@ type GenericProviderReconciler struct {
 
 const (
 	appliedSpecHashAnnotation = "operator.cluster.x-k8s.io/applied-spec-hash"
+	cacheOwner                = "capi-operator"
 )
 
 func (r *GenericProviderReconciler) BuildWithManager(ctx context.Context, mgr ctrl.Manager) (*ctrl.Builder, error) {
@@ -173,8 +178,15 @@ func (r *GenericProviderReconciler) Reconcile(ctx context.Context, req reconcile
 		return ctrl.Result{}, err
 	}
 
-	if r.Provider.GetAnnotations()[appliedSpecHashAnnotation] == specHash {
+	// Check provider config map for changes
+	cacheUsed, err := applyFromCache(ctx, r.Client, r.Provider)
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if r.Provider.GetAnnotations()[appliedSpecHashAnnotation] == specHash || cacheUsed {
 		log.Info("No changes detected, skipping further steps")
+
 		return ctrl.Result{}, nil
 	}
 
@@ -316,17 +328,140 @@ func addObjectToHash(hash hash.Hash, object interface{}) error {
 	return nil
 }
 
+// providerHash calculates hash for provider and referenced objects.
+func providerHash(ctx context.Context, client client.Client, hash hash.Hash, provider genericprovider.GenericProvider) error {
+	log := log.FromContext(ctx)
+
+	err := addObjectToHash(hash, provider.GetSpec())
+	if err != nil {
+		log.Error(err, "failed to calculate provider hash")
+
+		return err
+	}
+
+	if err := addConfigSecretToHash(ctx, client, hash, provider); err != nil {
+		log.Error(err, "failed to calculate secret hash")
+
+		return err
+	}
+
+	return nil
+}
+
 func calculateHash(ctx context.Context, k8sClient client.Client, provider genericprovider.GenericProvider) (string, error) {
 	hash := sha256.New()
 
-	err := addObjectToHash(hash, provider.GetSpec())
+	err := providerHash(ctx, k8sClient, hash, provider)
+
+	return fmt.Sprintf("%x", hash.Sum(nil)), err
+}
+
+// applyFromCache applies provider configuration from cache and returns true if the cache did not change.
+func applyFromCache(ctx context.Context, cl client.Client, provider genericprovider.GenericProvider) (bool, error) {
+	log := log.FromContext(ctx)
+
+	configMap, err := providerConfigMap(ctx, cl, provider)
 	if err != nil {
-		return "", err
+		log.Error(err, "failed to get provider config map")
+
+		return false, err
+	}
+
+	// config map does not exist, nothing to apply
+	if configMap == nil {
+		return false, nil
+	}
+
+	// calculate combined hash for provider and config map cache
+	hash := sha256.New()
+	if err := providerHash(ctx, cl, hash, provider); err != nil {
+		log.Error(err, "failed to calculate provider hash")
+
+		return false, err
+	}
+
+	if err := addObjectToHash(hash, configMap.Data); err != nil {
+		log.Error(err, "failed to calculate config map hash")
+
+		return false, err
 	}
 
-	if err := addConfigSecretToHash(ctx, k8sClient, hash, provider); err != nil {
-		return "", err
+	cacheHash := fmt.Sprintf("%x", hash.Sum(nil))
+	if configMap.GetAnnotations()[appliedSpecHashAnnotation] != cacheHash {
+		log.Info("Provider or cache state has changed", "cacheHash", cacheHash, "providerHash", configMap.GetAnnotations()[appliedSpecHashAnnotation])
+
+		return false, nil
+	}
+
+	components, err := getComponentsData(*configMap)
+	if err != nil {
+		log.Error(err, "failed to get provider components")
+
+		return false, err
 	}
 
-	return fmt.Sprintf("%x", hash.Sum(nil)), nil
+	additionalManifests, err := fetchAdditionalManifests(ctx, cl, provider)
+	if err != nil {
+		log.Error(err, "failed to get additional manifests")
+
+		return false, err
+	}
+
+	if additionalManifests != "" {
+		components = components + "\n---\n" + additionalManifests
+	}
+
+	for _, manifests := range strings.Split(components, "---") {
+		manifests, err := utilyaml.ToUnstructured([]byte(manifests))
+		if err != nil {
+			log.Error(err, "failed to convert yaml to unstructured")
+
+			return false, err
+		}
+
+		if err := cl.Patch(ctx, &manifests[0], client.Apply, client.ForceOwnership, client.FieldOwner(cacheOwner)); err != nil {
+			log.Error(err, "failed to apply object from cache")
+
+			return false, nil
+		}
+	}
+
+	log.Info("Applied all objects from cache")
+
+	return true, nil
+}
+
+// setCacheHash calculates current provider and configMap hash, and updates it on the configMap.
+func setCacheHash(ctx context.Context, cl client.Client, provider genericprovider.GenericProvider) error {
+	configMap, err := providerConfigMap(ctx, cl, provider)
+	if err != nil {
+		return err
+	}
+
+	helper, err := patch.NewHelper(configMap, cl)
+	if err != nil {
+		return err
+	}
+
+	hash := sha256.New()
+
+	if err := providerHash(ctx, cl, hash, provider); err != nil {
+		return err
+	}
+
+	if err := addObjectToHash(hash, configMap.Data); err != nil {
+		return err
+	}
+
+	cacheHash := fmt.Sprintf("%x", hash.Sum(nil))
+
+	annotations := configMap.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+
+	annotations[appliedSpecHashAnnotation] = cacheHash
+	configMap.SetAnnotations(annotations)
+
+	return helper.Patch(ctx, configMap)
 }

internal/controller/manifests_downloader.go

@@ -138,6 +138,16 @@ func (p *PhaseReconciler) checkConfigMapExists(ctx context.Context, labelSelecto
 	return len(configMapList.Items) == 1, nil
 }
 
+// finalize applies combined hash to a configMap, in order to mark provider provisioning completed.
+func (p *phaseReconciler) finalize(ctx context.Context) (reconcile.Result, error) {
+	err := setCacheHash(ctx, p.ctrlClient, p.provider)
+	if err != nil {
+		ctrl.LoggerFrom(ctx).V(5).Error(err, "Failed to update providers ConfigMap hash")
+	}
+
+	return reconcile.Result{}, wrapPhaseError(err, "failed to update providers ConfigMap hash", operatorv1.ProviderInstalledCondition)
+}
+
 // prepareConfigMapLabels returns labels that identify a config map with downloaded manifests.
 func (p *PhaseReconciler) prepareConfigMapLabels() map[string]string {
 	return ProviderLabels(p.provider)
@@ -285,6 +295,27 @@ func providerLabelSelector(provider operatorv1.GenericProvider) *metav1.LabelSel
 	}
 }
 
+// providerConfigMap finds a ConfigMap the given provider label selector.
+func providerConfigMap(ctx context.Context, cl client.Client, provider operatorv1.GenericProvider) (*corev1.ConfigMap, error) {
+	labelSelector := providerLabelSelector(provider)
+	labelSet := labels.Set(labelSelector.MatchLabels)
+	listOpts := []client.ListOption{
+		client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(labelSet)},
+		client.InNamespace(provider.GetNamespace()),
+	}
+
+	configMapList := &corev1.ConfigMapList{}
+	if err := cl.List(ctx, configMapList, listOpts...); err != nil {
+		return nil, fmt.Errorf("failed to list ConfigMaps: %w", err)
+	}
+
+	if len(configMapList.Items) > 1 {
+		return nil, fmt.Errorf("multiple ConfigMaps found for provider %q", provider.GetName())
+	}
+
+	return &configMapList.Items[0], nil
+}
+
 // ProviderLabels returns default set of labels that identify a config map with downloaded manifests.
 func ProviderLabels(provider operatorv1.GenericProvider) map[string]string {
 	labels := map[string]string{

internal/controller/phases.go

@@ -240,7 +240,7 @@ func (p *PhaseReconciler) Load(ctx context.Context) (*Result, error) {
 		labelSelector = p.provider.GetSpec().FetchConfig.Selector
 	}
 
-	additionalManifests, err := p.fetchAdditionalManifests(ctx)
+	additionalManifests, err := fetchAdditionalManifests(ctx, p.ctrlClient, p.provider)
 	if err != nil {
 		return &Result{}, wrapPhaseError(err, "failed to load additional manifests", operatorv1.ProviderInstalledCondition)
 	}
@@ -419,13 +419,13 @@ func (p *PhaseReconciler) configmapRepository(ctx context.Context, labelSelector
 	return mr, nil
 }
 
-func (p *PhaseReconciler) fetchAdditionalManifests(ctx context.Context) (string, error) {
+func fetchAdditionalManifests(ctx context.Context, cl client.Client, provider genericprovider.GenericProvider) (string, error) {
 	cm := &corev1.ConfigMap{}
 
-	if p.provider.GetSpec().AdditionalManifestsRef != nil {
-		key := types.NamespacedName{Namespace: p.provider.GetSpec().AdditionalManifestsRef.Namespace, Name: p.provider.GetSpec().AdditionalManifestsRef.Name}
+	if provider.GetSpec().AdditionalManifestsRef != nil {
+		key := types.NamespacedName{Namespace: provider.GetSpec().AdditionalManifestsRef.Namespace, Name: provider.GetSpec().AdditionalManifestsRef.Name}
 
-		if err := p.ctrlClient.Get(ctx, key, cm); err != nil {
+		if err := cl.Get(ctx, key, cm); err != nil {
 			return "", fmt.Errorf("failed to get ConfigMap %s/%s: %w", key.Namespace, key.Name, err)
 		}
 	}