Add tests for air-gapped upgrade procedure

- Upgrades should modify manifests regularly as well.

Signed-off-by: Danil Grigorev <[email protected]>

Author: Danil-Grigorev

api/v1alpha2/conditions_consts.go

@@ -47,6 +47,9 @@ const (
 	// ComponentsFetchErrorReason documents that an error occurred fetching the components.
 	ComponentsFetchErrorReason = "ComponentsFetchError"
 
+	// ComponentsUpgradeErrorReason documents that an error occurred while upgrading the components.
+	ComponentsUpgradeErrorReason = "ComponentsUpgradeError"
+
 	// OldComponentsDeletionErrorReason documents that an error occurred deleting the old components prior to upgrading.
 	OldComponentsDeletionErrorReason = "OldComponentsDeletionError"
 
@@ -63,4 +66,7 @@ const (
 const (
 	// ProviderInstalledCondition documents a Provider that has been installed.
 	ProviderInstalledCondition clusterv1.ConditionType = "ProviderInstalled"
+
+	// ProviderUpgradedCondition documents a Provider that has been recently upgraded.
+	ProviderUpgradedCondition clusterv1.ConditionType = "ProviderUpgraded"
 )

internal/controller/client_proxy.go

@@ -32,6 +32,7 @@ import (
 	operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2"
 	clusterctlv1 "sigs.k8s.io/cluster-api/cmd/clusterctl/api/v1alpha3"
 	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/cluster"
+	"sigs.k8s.io/cluster-api/cmd/clusterctl/client/repository"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -218,3 +219,25 @@ func listObjByGVK(ctx context.Context, c client.Client, groupVersion, kind strin
 
 	return objList, nil
 }
+
+type repositoryProxy struct {
+	repository.Client
+
+	components repository.Components
+}
+
+type repositoryClient struct {
+	components repository.Components
+}
+
+func (r repositoryClient) Raw(ctx context.Context, options repository.ComponentsOptions) ([]byte, error) {
+	return nil, nil
+}
+
+func (r repositoryClient) Get(ctx context.Context, options repository.ComponentsOptions) (repository.Components, error) {
+	return r.components, nil
+}
+
+func (r repositoryProxy) Components() repository.ComponentsClient {
+	return repositoryClient{r.components}
+}

internal/controller/genericprovider_controller.go

@@ -157,8 +157,9 @@ func (r *GenericProviderReconciler) reconcile(ctx context.Context, provider gene
 		reconciler.downloadManifests,
 		reconciler.load,
 		reconciler.fetch,
-		reconciler.preInstall,
+		reconciler.upgrade,
 		reconciler.install,
+		reconciler.reportStatus,
 	}
 
 	res := reconcile.Result{}

internal/controller/genericprovider_controller_test.go

@@ -18,8 +18,10 @@ package controller
 
 import (
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -39,7 +41,7 @@ apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
 releaseSeries:
   - major: 0
     minor: 4
-    contract: v1alpha4
+    contract: v1beta1
 `
 	testComponents = `
 apiVersion: apps/v1
@@ -215,22 +217,47 @@ func TestReconcilerPreflightConditions(t *testing.T) {
 	}
 }
 
-func TestUpgradeDowngradeProvider(t *testing.T) {
+func TestAirGappedUpgradeDowngradeProvider(t *testing.T) {
+	currentVersion := "v999.9.2"
+	futureMetadata := `
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+releaseSeries:
+  - major: 999
+    minor: 9
+    contract: v1beta1
+`
+
+	dummyFutureConfigMap := func(ns, name string) *corev1.ConfigMap {
+		return &corev1.ConfigMap{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      name,
+				Namespace: ns,
+				Labels: map[string]string{
+					"test": "dummy-config",
+				},
+			},
+			Data: map[string]string{
+				"metadata":   futureMetadata,
+				"components": testComponents,
+			},
+		}
+	}
+
 	testCases := []struct {
 		name       string
 		newVersion string
 	}{
 		{
 			name:       "same provider version",
-			newVersion: "v0.4.2",
+			newVersion: "v999.9.2",
 		},
 		{
 			name:       "upgrade provider version",
-			newVersion: "v0.4.3",
+			newVersion: "v999.9.3",
 		},
 		{
 			name:       "downgrade provider version",
-			newVersion: "v0.4.1",
+			newVersion: "v999.9.1",
 		},
 	}
 
@@ -244,7 +271,7 @@ func TestUpgradeDowngradeProvider(t *testing.T) {
 				},
 				Spec: operatorv1.CoreProviderSpec{
 					ProviderSpec: operatorv1.ProviderSpec{
-						Version: testCurrentVersion,
+						Version: currentVersion,
 					},
 				},
 			}
@@ -254,7 +281,7 @@ func TestUpgradeDowngradeProvider(t *testing.T) {
 			t.Log("Ensure namespace exists", namespace)
 			g.Expect(env.EnsureNamespaceExists(ctx, namespace)).To(Succeed())
 
-			g.Expect(env.CreateAndWait(ctx, dummyConfigMap(namespace, testCurrentVersion))).To(Succeed())
+			g.Expect(env.CreateAndWait(ctx, dummyFutureConfigMap(namespace, currentVersion))).To(Succeed())
 
 			insertDummyConfig(provider)
 			provider.SetNamespace(namespace)
@@ -266,7 +293,7 @@ func TestUpgradeDowngradeProvider(t *testing.T) {
 					return false
 				}
 
-				if provider.GetStatus().InstalledVersion == nil || *provider.GetStatus().InstalledVersion != testCurrentVersion {
+				if provider.GetStatus().InstalledVersion == nil || *provider.GetStatus().InstalledVersion != currentVersion {
 					return false
 				}
 
@@ -283,13 +310,16 @@ func TestUpgradeDowngradeProvider(t *testing.T) {
 			}, timeout).Should(BeEquivalentTo(true))
 
 			// creating another configmap with another version
-			if tc.newVersion != testCurrentVersion {
-				g.Expect(env.CreateAndWait(ctx, dummyConfigMap(namespace, tc.newVersion))).To(Succeed())
+			if tc.newVersion != currentVersion {
+				g.Expect(env.CreateAndWait(ctx, dummyFutureConfigMap(namespace, tc.newVersion))).To(Succeed())
 			}
 
 			// Change provider version
 			providerSpec := provider.GetSpec()
 			providerSpec.Version = tc.newVersion
+			providerSpec.Deployment = &operatorv1.DeploymentSpec{
+				Replicas: pointer.Int(2),
+			}
 			provider.SetSpec(providerSpec)
 
 			// Set label (needed to start a reconciliation of the provider)
@@ -315,23 +345,68 @@ func TestUpgradeDowngradeProvider(t *testing.T) {
 					return false
 				}
 
+				allFound := false
 				for _, cond := range provider.GetStatus().Conditions {
 					if cond.Type == operatorv1.PreflightCheckCondition {
 						t.Log(t.Name(), provider.GetName(), cond)
 						if cond.Status == corev1.ConditionTrue {
-							return true
+							allFound = true
+							break
 						}
 					}
 				}
 
-				return false
+				if !allFound {
+					return false
+				}
+
+				allFound = tc.newVersion == currentVersion
+				for _, cond := range provider.GetStatus().Conditions {
+					if cond.Type == operatorv1.ProviderUpgradedCondition {
+						t.Log(t.Name(), provider.GetName(), cond)
+						if cond.Status == corev1.ConditionTrue {
+							allFound = tc.newVersion != currentVersion
+							break
+						}
+					}
+				}
+
+				if !allFound {
+					return false
+				}
+
+				// Ensure customization occurred
+				dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{
+					Namespace: provider.Namespace,
+					Name:      "capd-controller-manager",
+				}}
+				if err := env.Get(ctx, client.ObjectKeyFromObject(dep), dep); err != nil {
+					return false
+				}
+
+				return dep.Spec.Replicas != nil && *dep.Spec.Replicas == 2
 			}, timeout).Should(BeEquivalentTo(true))
 
+			g.Consistently(func() bool {
+				allSet := tc.newVersion == currentVersion
+				for _, cond := range provider.GetStatus().Conditions {
+					if cond.Type == operatorv1.ProviderUpgradedCondition {
+						t.Log(t.Name(), provider.GetName(), cond)
+						if cond.Status == corev1.ConditionTrue {
+							allSet = tc.newVersion != currentVersion
+							break
+						}
+					}
+				}
+
+				return allSet
+			}, 2*time.Second).Should(BeTrue())
+
 			// Clean up
 			objs := []client.Object{provider}
 			objs = append(objs, &corev1.ConfigMap{
 				ObjectMeta: metav1.ObjectMeta{
-					Name:      testCurrentVersion,
+					Name:      currentVersion,
 					Namespace: namespace,
 				},
 			})

internal/controller/manifests_downloader.go

@@ -65,7 +65,7 @@ func (p *phaseReconciler) downloadManifests(ctx context.Context) (reconcile.Resu
 
 	exists, err := p.checkConfigMapExists(ctx, labelSelector)
 	if err != nil {
-		return reconcile.Result{}, wrapPhaseError(err, "failed to check that config map with manifests exists")
+		return reconcile.Result{}, wrapPhaseError(err, "failed to check that config map with manifests exists", operatorv1.ProviderInstalledCondition)
 	}
 
 	if exists {
@@ -80,7 +80,7 @@ func (p *phaseReconciler) downloadManifests(ctx context.Context) (reconcile.Resu
 	if err != nil {
 		err = fmt.Errorf("failed to create repo from provider url for provider %q: %w", p.provider.GetName(), err)
 
-		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason)
+		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition)
 	}
 
 	spec := p.provider.GetSpec()
@@ -98,22 +98,22 @@ func (p *phaseReconciler) downloadManifests(ctx context.Context) (reconcile.Resu
 	if err != nil {
 		err = fmt.Errorf("failed to read %q from the repository for provider %q: %w", metadataFile, p.provider.GetName(), err)
 
-		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason)
+		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition)
 	}
 
 	componentsFile, err := repo.GetFile(ctx, spec.Version, repo.ComponentsPath())
 	if err != nil {
 		err = fmt.Errorf("failed to read %q from the repository for provider %q: %w", componentsFile, p.provider.GetName(), err)
 
-		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason)
+		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition)
 	}
 
 	withCompression := needToCompress(metadataFile, componentsFile)
 
 	if err := p.createManifestsConfigMap(ctx, metadataFile, componentsFile, withCompression); err != nil {
 		err = fmt.Errorf("failed to create config map for provider %q: %w", p.provider.GetName(), err)
 
-		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason)
+		return reconcile.Result{}, wrapPhaseError(err, operatorv1.ComponentsFetchErrorReason, operatorv1.ProviderInstalledCondition)
 	}
 
 	return reconcile.Result{}, nil