compat-ver: fix 2 step upgrade tests.

Signed-off-by: Siyuan Zhang <[email protected]>

Author: siyuanfoundation

experiment/compatibility-versions/common.sh

@@ -224,19 +224,20 @@ EOF
   return 0
 }
 
-build_prev_version_bins() {
+build_test_bins() {
+  local release_branch=$1
   GINKGO_SRC_DIR="vendor/github.com/onsi/ginkgo/v2/ginkgo"
 
-  echo "Building e2e.test binary from release branch ${PREV_RELEASE_BRANCH}..."
+  echo "Building e2e.test binary from release branch ${release_branch}..."
   make all WHAT="cmd/kubectl test/e2e/e2e.test ${GINKGO_SRC_DIR}"
 
   # Ensure the built kubectl is used instead of system
   export PATH="${PWD}/_output/bin:$PATH"
-  echo "Finished building e2e.test binary from ${PREV_RELEASE_BRANCH}."
+  echo "Finished building e2e.test binary from ${release_branch}."
 }
 
 # run e2es with ginkgo-e2e.sh
-run_prev_version_tests() {
+run_e2e_tests() {
   # IPv6 clusters need some CoreDNS changes in order to work in k8s CI:
   # 1. k8s CI doesn´t offer IPv6 connectivity, so CoreDNS should be configured
   # to work in an offline environment:

experiment/compatibility-versions/e2e-k8s-compatibility-versions.sh

@@ -119,8 +119,8 @@ main() {
 
   # enter the cloned prev repo branch (in temp) and run tests
   pushd "${PREV_RELEASE_REPO_PATH}"
-  build_prev_version_bins || res=$?
-  run_prev_version_tests || res=$?
+  build_test_bins "${PREV_RELEASE_BRANCH}" || res=$?
+  run_e2e_tests || res=$?
   popd
 
 

experiment/compatibility-versions/e2e-skip-version-testing.sh

@@ -49,8 +49,8 @@ run_skip_version_tests() {
       return 1
     fi
     pushd "${PREV_RELEASE_REPO_PATH}"
-    build_prev_version_bins || ret=$?
-    run_prev_version_tests || ret=$?
+    build_test_bins "${PREV_RELEASE_BRANCH}" || ret=$?
+    run_e2e_tests || ret=$?
     if [[ "$ret" -ne 0 ]]; then
       echo "Failed running skip version tests for emulated version $EMULATED_VERSION"
       return 1
@@ -66,8 +66,8 @@ run_skip_version_tests() {
   # Test removal of emulated version entirely.
   export PREV_RELEASE_BRANCH="release-${EMULATED_VERSION}"
   delete_emulation_version || ret=$?
-  build_prev_version_bins || ret=$?
-  run_prev_version_tests || ret=$?
+  build_test_bins "${PREV_RELEASE_BRANCH}" || ret=$?
+  run_e2e_tests || ret=$?
   return $ret
 }
 

experiment/compatibility-versions/e2e-two-steps-upgrade.sh

@@ -97,7 +97,7 @@ main() {
   res=0
   create_cluster || res=$?
 
-
+  # first step: upgrade binary version while keeping emulated version the same as previous version
   # Perform the upgrade.  Assume kind-upgrade.sh is in the same directory as this script.
   UPGRADE_SCRIPT="${UPGRADE_SCRIPT:-${PWD}/../test-infra/experiment/compatibility-versions/kind-upgrade.sh}"
   echo "Upgrading cluster with ${UPGRADE_SCRIPT}"
@@ -109,24 +109,36 @@ main() {
     exit $res
   fi
 
-  EMULATED_VERSION_UPGRADE_SCRIPT="${EMULATED_VERSION_UPGRADE_SCRIPT:-${PWD}/../test-infra/experiment/compatibility-versions/emulated-version-upgrade.sh}"
-  echo "Upgrading cluster with ${EMULATED_VERSION_UPGRADE_SCRIPT}"
-  "${EMULATED_VERSION_UPGRADE_SCRIPT}" | tee "${ARTIFACTS}/emulated-upgrade-output.txt"
+  # after first step of upgrading binary version without bumping emulated version, verify all tests from the previous version pass
 
   # Clone the previous versions Kubernetes release branch
   # TODO(aaron-prindle) extend the branches to test from n-1 -> n-1..3 as more k8s releases are done that support compatibility versions
-  export RELEASE_BRANCH="release-${CURRENT_VERSION}"
+  export PREV_RELEASE_BRANCH="release-${PREV_VERSION}"
   # Define the path within the temp directory for the cloned repo
-  RELEASE_REPO_PATH="${TMP_DIR}/release-k8s"
-  echo "Cloning branch ${RELEASE_BRANCH} into ${RELEASE_REPO_PATH}"
-  git clone --filter=blob:none --single-branch --branch "${RELEASE_BRANCH}" https://github.com/kubernetes/kubernetes.git "${RELEASE_REPO_PATH}"
-
+  PREV_RELEASE_REPO_PATH="${TMP_DIR}/prev-release-k8s"
+  echo "Cloning branch ${PREV_RELEASE_BRANCH} into ${PREV_RELEASE_REPO_PATH}"
+  git clone --filter=blob:none --single-branch --branch "${PREV_RELEASE_BRANCH}" https://github.com/kubernetes/kubernetes.git "${PREV_RELEASE_REPO_PATH}"
   # enter the cloned prev repo branch (in temp) and run tests
-  pushd "${RELEASE_REPO_PATH}"
-  build_prev_version_bins || res=$?
-  run_prev_version_tests || res=$?
+  pushd "${PREV_RELEASE_REPO_PATH}"
+  build_test_bins "${PREV_RELEASE_BRANCH}" || res=$?
+  run_e2e_tests || res=$?
+  # debug kubectl version
+  kubectl version
+  # remove "${PWD}/_output/bin" from PATH
+  export PATH="${PATH//${PWD}\/_output\/bin:}"
   popd
 
+  # second step: bump emulated version
+  EMULATED_VERSION_UPGRADE_SCRIPT="${EMULATED_VERSION_UPGRADE_SCRIPT:-${PWD}/../test-infra/experiment/compatibility-versions/emulated-version-upgrade.sh}"
+  echo "Upgrading cluster with ${EMULATED_VERSION_UPGRADE_SCRIPT}"
+  "${EMULATED_VERSION_UPGRADE_SCRIPT}" | tee "${ARTIFACTS}/emulated-upgrade-output.txt"
+
+  # verify all tests from the current version pass after upgrade is complete
+  # debug kubectl version
+  kubectl version
+  # run tests at head
+  build_test_bins "${CURRENT_VERSION}" || res=$?
+  run_e2e_tests || res=$?
 
   cleanup || res=$?
   exit $res