Improve backwards compatibility test (#785)

Author: thegridman

.github/workflows/compatibility-tests.yaml

@@ -37,15 +37,19 @@ jobs:
       fail-fast: false
       matrix:
         compatibilityVersion:
+        - 3.5.2
         - 3.5.0
         - 3.4.3
         - 3.4.2
         - 3.4.1
         - 3.4.0
         - 3.3.5
         - 3.3.4
-        - 3.3.3
         include:
+        - compatibilityVersion: 3.5.2
+          coherence-image: "ghcr.io/oracle/coherence-ce:14.1.2-0-2"
+          compatibilitySelector: control-plane=coherence
+          k8s: kindest/node:v1.33.1@sha256:050072256b9a903bd914c0b2866828150cb229cea0efe5892e2b644d5dd3b34f
         - compatibilityVersion: 3.5.0
           coherence-image: "ghcr.io/oracle/coherence-ce:14.1.2-0-2"
           compatibilitySelector: control-plane=coherence
@@ -74,10 +78,6 @@ jobs:
           coherence-image: "ghcr.io/oracle/coherence-ce:22.06.12"
           compatibilitySelector: control-plane=coherence
           k8s: kindest/node:v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245
-        - compatibilityVersion: 3.3.3
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.12"
-          compatibilitySelector: control-plane=coherence
-          k8s: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
 
     steps:
     - uses: actions/checkout@v4

Makefile

@@ -1585,6 +1585,18 @@ just-compatibility-test:  ## Run the Operator backwards compatibility tests WITH
 	$(GOTESTSUM) --format standard-verbose --junitfile $(TEST_LOGS_DIR)/operator-e2e-compatibility-test.xml \
 	  -- $(GO_TEST_FLAGS_E2E) ./test/e2e/compatibility/...
 
+helm-install-prev: ## Install previous operator version for the Operator backwards compatibility tests
+	helm repo add coherence https://oracle.github.io/coherence-operator/charts
+	helm repo update
+	helm upgrade --version $(COMPATIBLE_VERSION) \
+		--namespace $(OPERATOR_NAMESPACE) operator coherence/coherence-operator
+
+helm-upgrade-current: ## Upgrade from the previous to the current operator version for the Operator backwards compatibility tests
+	helm upgrade --set image=$(OPERATOR_IMAGE) \
+		--set defaultCoherenceUtilsImage=$(OPERATOR_IMAGE) \
+		--namespace $(OPERATOR_NAMESPACE) \
+		operator $(BUILD_HELM)/coherence-operator
+
 
 # ----------------------------------------------------------------------------------------------------------------------
 # Executes the Go end-to-end Operator Kubernetes versions certification tests.

test/e2e/compatibility/compatibility_test.go

@@ -86,13 +86,33 @@ func TestCompatibility(t *testing.T) {
 
 	// scale up to make sure that the Operator can still manage the Coherence cluster
 	n := fmt.Sprintf("coherence/%s", d.Name)
-	t.Logf("Scaling coherence resource %s in namespace %s to 3 replicas\n", n, ns)
-	cmd := exec.Command("kubectl", "-n", ns, "scale", n, "--replicas=3")
+	replicas := 3
+	testContext.Logf("Scaling coherence resource %s in namespace %s to %d replicas\n", n, ns, replicas)
+	cmd := exec.Command("kubectl", "-n", ns, "scale", n, fmt.Sprintf("--replicas=%d", replicas))
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
 	g.Expect(err).NotTo(HaveOccurred())
-	_ = assertDeploymentEventuallyInDesiredState(t, d, 3)
+	_ = assertDeploymentEventuallyInDesiredState(t, d, int32(replicas))
+
+	testContext.Logf("Updating coherence resource %s in namespace %s\n", d.Name, ns)
+	updated := &cohv1.Coherence{}
+	err = testContext.Client.Get(context.TODO(), d.GetNamespacedName(), updated)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	l := updated.Spec.Labels
+	if l == nil {
+		l = make(map[string]string)
+	}
+	l["testCompatibility"] = "updated"
+	updated.Spec.Labels = l
+
+	err = testContext.Client.Update(testContext.Context, updated)
+	g.Expect(err).NotTo(HaveOccurred())
+
+	testContext.Logf("Waiting for Pods in updated coherence resource %s in namespace %s\n", d.Name, ns)
+	_, err = helper.WaitForPodsWithLabel(testContext, ns, "testCompatibility=updated", replicas, time.Second*20, time.Minute*10)
+	g.Expect(err).NotTo(HaveOccurred())
 }
 
 func InstallPreviousVersion(g *GomegaWithT, ns, name, version, selector string) {
@@ -107,7 +127,9 @@ func InstallPreviousVersion(g *GomegaWithT, ns, name, version, selector string)
 	err = os.MkdirAll(prevDir, os.ModePerm)
 	g.Expect(err).NotTo(HaveOccurred())
 
-	cmd := exec.Command("helm", "fetch", "--version", version,
+	var cmd *exec.Cmd
+
+	cmd = exec.Command("helm", "fetch", "--version", version,
 		"--untar", "--untardir", prevDir, "coherence/coherence-operator")
 
 	cmd.Stdout = os.Stdout