K8SPS-335: Add gr-upgrade and async-upgrade tests

e2e-tests/functions

@@ -54,34 +54,82 @@ wait_for_delete() {
 	done
 }
 
+apply_crd() {
+	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml"
+}
+
+apply_rbac() {
+	local rbac_file
+
+	if [ -n "$OPERATOR_NS" ]; then
+		rbac_file="${DEPLOY_DIR}/cw-rbac.yaml"
+	else
+		rbac_file="${DEPLOY_DIR}/rbac.yaml"
+	fi
+
+	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${rbac_file}"
+}
+
 deploy_operator() {
 	destroy_operator
 
 	if [[ $OPERATOR_NS ]]; then
 		create_namespace "${OPERATOR_NS}"
 	fi
 
-	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml"
+	apply_crd
+	apply_rbac
+
+	local operator_file
 
 	if [ -n "$OPERATOR_NS" ]; then
-		kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${DEPLOY_DIR}/cw-rbac.yaml"
-
-		yq eval \
-			"$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \
-			"${DEPLOY_DIR}/cw-operator.yaml" \
-			| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' \
-			| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' \
-			| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
+		operator_file="${DEPLOY_DIR}/cw-operator.yaml"
 	else
-		kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f "${DEPLOY_DIR}/rbac.yaml"
-
-		yq eval \
-			"$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \
-			"${DEPLOY_DIR}/operator.yaml" \
-			| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' \
-			| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' \
-			| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
+		operator_file="${DEPLOY_DIR}/-operator.yaml"
 	fi
+
+	yq eval \
+		"$(printf 'select(documentIndex==1).spec.template.spec.containers[0].image="%s"' "${IMAGE}")" \
+		"${operator_file}" \
+		| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' \
+		| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' \
+		| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
+}
+
+upgrade_operator_image() {
+	local image=$1
+
+	kubectl -n ${OPERATOR_NS:-$NAMESPACE} set image deployment/percona-server-mysql-operator manager=${image}
+}
+
+deploy_operator_gh() {
+	local git_tag=$1
+
+	echo "applying ${git_tag}/deploy/crd.yaml"
+	kubectl apply \
+		--server-side \
+		--force-conflicts \
+		-f "https://raw.githubusercontent.com/percona/percona-server-mysql-operator/${git_tag}/deploy/crd.yaml"
+
+	local rbac="rbac"
+	local operator="operator"
+	if [ -n "${OPERATOR_NS}" ]; then
+		rbac="cw-rbac"
+		operator="cw-operator"
+
+		create_namespace "${OPERATOR_NS}"
+	fi
+
+	echo "applying ${git_tag}/deploy/${rbac}.yaml"
+	kubectl apply \
+		-n "${OPERATOR_NS:-$NAMESPACE}" \
+		-f "https://raw.githubusercontent.com/percona/percona-server-mysql-operator/${git_tag}/deploy/${rbac}.yaml"
+
+	echo "applying ${git_tag}/deploy/${operator}.yaml"
+	curl -s "https://raw.githubusercontent.com/percona/percona-server-mysql-operator/${git_tag}/deploy/${operator}.yaml" \
+		| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="DISABLE_TELEMETRY").value) = "true"' \
+		| yq eval '(select(documentIndex==1).spec.template.spec.containers[] | select(.name=="manager").env[] | select(.name=="LOG_LEVEL").value) = "DEBUG"' \
+		| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
 }
 
 destroy_operator() {
@@ -547,24 +595,38 @@ get_operator_pod() {
 		-o 'jsonpath={.items[].metadata.name}'
 }
 
+get_operator_image() {
+	kubectl get pods -n "${OPERATOR_NS:-$NAMESPACE}" \
+		--selector=app.kubernetes.io/name=percona-server-mysql-operator \
+		-o 'jsonpath={.items[].spec.containers[].image}'
+}
+
 get_cr() {
 	local name_suffix=$1
-
-	yq eval "$(printf '.metadata.name="%s"' "${test_name}${name_suffix:+-$name_suffix}")" "${DEPLOY_DIR}/cr.yaml" \
+	local image_mysql=${2:-${IMAGE_MYSQL}}
+	local image_backup=${3:-${IMAGE_BACKUP}}
+	local image_orchestrator=${4:-${IMAGE_ORCHESTRATOR}}
+	local image_router=${5:-${IMAGE_ROUTER}}
+	local image_toolkit=${6:-${IMAGE_TOOLKIT}}
+	local image_haproxy=${7:-${IMAGE_HAPROXY}}
+	local image_pmm_client=${8:-${IMAGE_PMM_CLIENT}}
+	local cr_file=${9:-${DEPLOY_DIR}/cr.yaml}
+
+	yq eval "$(printf '.metadata.name="%s"' "${test_name}${name_suffix:+-$name_suffix}")" ${cr_file} \
 		| yq eval "$(printf '.spec.initImage="%s"' "${IMAGE}")" - \
 		| yq eval '.spec.secretsName="test-secrets"' - \
 		| yq eval '.spec.sslSecretName="test-ssl"' - \
 		| yq eval '.spec.upgradeOptions.apply="disabled"' - \
 		| yq eval '.spec.mysql.clusterType="async"' - \
     | yq eval '.spec.mysql.gracePeriod=30' - \
 		| yq eval '.spec.orchestrator.enabled=true' - \
-		| yq eval "$(printf '.spec.mysql.image="%s"' "${IMAGE_MYSQL}")" - \
-		| yq eval "$(printf '.spec.backup.image="%s"' "${IMAGE_BACKUP}")" - \
-		| yq eval "$(printf '.spec.orchestrator.image="%s"' "${IMAGE_ORCHESTRATOR}")" - \
-		| yq eval "$(printf '.spec.proxy.router.image="%s"' "${IMAGE_ROUTER}")" - \
-		| yq eval "$(printf '.spec.toolkit.image="%s"' "${IMAGE_TOOLKIT}")" - \
-		| yq eval "$(printf '.spec.proxy.haproxy.image="%s"' "${IMAGE_HAPROXY}")" - \
-		| yq eval "$(printf '.spec.pmm.image="%s"' "${IMAGE_PMM_CLIENT}")" - \
+		| yq eval "$(printf '.spec.mysql.image="%s"' "${image_mysql}")" - \
+		| yq eval "$(printf '.spec.backup.image="%s"' "${image_backup}")" - \
+		| yq eval "$(printf '.spec.orchestrator.image="%s"' "${image_orchestrator}")" - \
+		| yq eval "$(printf '.spec.proxy.router.image="%s"' "${image_router}")" - \
+		| yq eval "$(printf '.spec.toolkit.image="%s"' "${image_toolkit}")" - \
+		| yq eval "$(printf '.spec.proxy.haproxy.image="%s"' "${image_haproxy}")" - \
+		| yq eval "$(printf '.spec.pmm.image="%s"' "${image_pmm_client}")" - \
 		| if [ -n "${MINIKUBE}" ]; then
 			yq eval '(.. | select(has("antiAffinityTopologyKey")).antiAffinityTopologyKey) |= "none"' - \
 				| yq eval '.spec.proxy.haproxy.resources.requests.cpu="300m"' -
@@ -573,6 +635,16 @@ get_cr() {
 		fi
 }
 
+get_cr_gh() {
+	local name_suffix=$1
+
+	curl -s "https://raw.githubusercontent.com/percona/percona-server-mysql-operator/${git_tag}/deploy/cr.yaml" \
+		| yq eval "$(printf '.metadata.name="%s"' "${test_name}${name_suffix:+-$name_suffix}")" - \
+		| yq eval '.spec.secretsName="test-secrets"' - \
+		| yq eval '.spec.sslSecretName="test-ssl"' - \
+		| yq eval '.spec.upgradeOptions.apply="disabled"' -
+}
+
 get_client_pod() {
 	kubectl -n "${NAMESPACE}" get pods \
 		--selector=name=mysql-client \
@@ -829,6 +901,7 @@ wait_cluster_consistency_gr() {
 
 wait_pod() {
 	local pod=$1
+	local ns=${2:-${NAMESPACE}}
 
 	set +o xtrace
 	retry=0
@@ -1370,3 +1443,49 @@ check_scheduled_backup_labels() {
 		exit 1
 	fi
 }
+
+latest_operator_version_in_vs() {
+	local latest=$(curl -s https://check.percona.com/versions/v1/ps-operator | jq -r '.versions[].operator' | sort -V | tail -n1)
+
+	# if testing on release branch and version service is already updated with new operator images
+	# use the older version of operator as initial point for test
+	if [[ ${latest} == "${OPERATOR_VERSION}" ]]; then
+		latest=$(curl -s https://check.percona.com/versions/v1/ps-operator | jq -r '.versions[].operator' | sort -V | tail -n2 | head -n1)
+	fi
+
+	echo ${latest}
+}
+
+latest_operator_image_in_vs() {
+	local version=$(latest_operator_version_in_vs)
+
+	curl -s https://check.percona.com/versions/v1/ps-operator/${version}/latest | jq -r '.versions[].matrix.operator[].imagePath'
+}
+
+get_cr_with_latest_versions_in_vs() {
+	local version=$(latest_operator_version_in_vs)
+	local latest_versions=$(curl -s https://check.percona.com/versions/v1/ps-operator/${version}/latest)
+
+	local image_mysql=$(echo ${latest_versions} | jq -r '.versions[].matrix.mysql[].imagePath')
+	local image_backup=$(echo ${latest_versions} | jq -r '.versions[].matrix.backup[].imagePath')
+	local image_orchestrator=$(echo ${latest_versions} | jq -r '.versions[].matrix.orchestrator[].imagePath')
+	local image_router=$(echo ${latest_versions} | jq -r '.versions[].matrix.orchestrator[].imagePath')
+	local image_toolkit=$(echo ${latest_versions} | jq -r '.versions[].matrix.toolkit[].imagePath')
+	local image_haproxy=$(echo ${latest_versions} | jq -r '.versions[].matrix.haproxy[].imagePath')
+	local image_pmm_client=$(echo ${latest_versions} | jq -r '.versions[].matrix.pmm[].imagePath')
+
+	local git_tag="v${version}"
+
+	curl "https://raw.githubusercontent.com/percona/percona-server-mysql-operator/${git_tag}/deploy/cr.yaml" \
+		-o ${TEMP_DIR}/cr.yaml
+
+	get_cr "" \
+		${image_mysql} \
+		${image_backup} \
+		${image_orchestrator} \
+		${image_router} \
+		${image_toolkit} \
+		${image_haproxy} \
+		${image_pmm_client} \
+		${TEMP_DIR}/cr.yaml
+}

e2e-tests/run-distro.csv

@@ -15,6 +15,8 @@ gr-security-context
 gr-self-healing
 gr-tls-cert-manager
 gr-users
+gr-upgrade
+async-upgrade
 haproxy
 init-deploy
 one-pod

e2e-tests/run-pr.csv

@@ -20,6 +20,8 @@ gr-security-context
 gr-self-healing
 gr-tls-cert-manager
 gr-users
+gr-upgrade
+async-upgrade
 haproxy
 init-deploy
 limits

e2e-tests/run-release.csv

@@ -20,6 +20,8 @@ gr-security-context
 gr-self-healing
 gr-tls-cert-manager
 gr-users
+gr-upgrade
+async-upgrade
 haproxy
 init-deploy
 limits

e2e-tests/tests/async-upgrade/00-assert.yaml

@@ -0,0 +1,26 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: perconaservermysqls.ps.percona.com
+spec:
+  group: ps.percona.com
+  names:
+    kind: PerconaServerMySQL
+    listKind: PerconaServerMySQLList
+    plural: perconaservermysqls
+    shortNames:
+    - ps
+    singular: perconaservermysql
+  scope: Namespaced
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: kubectl assert exist-enhanced deployment percona-server-mysql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

e2e-tests/tests/async-upgrade/00-deploy-operator.yaml

@@ -0,0 +1,15 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      init_temp_dir # do this only in the first TestStep
+
+      git_tag="v$(latest_operator_version_in_vs)"
+      deploy_operator_gh ${git_tag}
+      deploy_non_tls_cluster_secrets
+      deploy_tls_cluster_secrets
+      deploy_client

e2e-tests/tests/async-upgrade/01-assert.yaml

@@ -0,0 +1,58 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 420
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: async-upgrade-mysql
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: async-upgrade-haproxy
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  updatedReplicas: 3
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: async-upgrade-orc
+status:
+  observedGeneration: 1
+  replicas: 3
+  readyReplicas: 3
+  currentReplicas: 3
+  updatedReplicas: 3
+  collisionCount: 0
+---
+apiVersion: ps.percona.com/v1alpha1
+kind: PerconaServerMySQL
+metadata:
+  name: async-upgrade
+  finalizers:
+    - percona.com/delete-mysql-pods-in-order
+status:
+  mysql:
+    ready: 3
+    size: 3
+    state: ready
+  haproxy:
+    ready: 3
+    size: 3
+    state: ready
+  orchestrator:
+    ready: 3
+    size: 3
+    state: ready
+  state: ready

e2e-tests/tests/async-upgrade/01-create-cluster.yaml

@@ -0,0 +1,22 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      version=$(latest_operator_version_in_vs)
+      init_image=$(get_operator_image)
+
+      if [[ -z ${init_image} ]]; then
+        echo "failed to get operator image"
+        exit 1
+      fi
+
+      get_cr_with_latest_versions_in_vs \
+          | yq eval "$(printf '.spec.initImage="%s"' "${init_image}")" - \
+          | yq eval "$(printf '.spec.crVersion="%s"' "${version}")" - \
+          | yq eval '.spec.mysql.clusterType="async"' - \
+          | kubectl -n "${NAMESPACE}" apply -f -

e2e-tests/tests/async-upgrade/02-upgrade-operator.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - timeout: 120
+    script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      apply_crd
+      apply_rbac
+      upgrade_operator_image ${IMAGE}
+      wait_deployment percona-server-mysql-operator ${OPERATOR_NS:-$NAMESPACE}