ci: azure-ipam patch upgrade testing (#3678)

* ci: azure-ipam patch upgrade testing conditions and variables added

* update to only use parameter.upgradeScenario and remove

* test empty upgrade vars

* add var check to non-hubble job

* removing upgradeAzureIpam parameter

* make deploy cns independent job from deploy cilium

* pass logtype to artifact name

* move conditional logType check to main template

* pass log type to restart cns

Author: camrynl

.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml

@@ -11,37 +11,39 @@ parameters:
   hubbleEnabled: false
   dualstackVersion: ""
   cni: "cilium"
+  upgradeScenario: ""
 
 # Condition confirms that:
 # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success.
 # CONTROL_CNI either contains 'cniv1' or 'all'. It is not case sensitive
 stages:
-  - stage: create_${{ parameters.name }}
-    condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cilium') , contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) )
-    variables:
-      ${{ if contains(parameters.clusterName, 'rdma') }}:
-        location: $(LOCATION_RDMA)
-      ${{ elseif eq(parameters.arch, 'arm64') }}:
-        location: $(LOCATION_ARM64)
-      ${{ else }}:
-        location: $(LOCATION_AMD64)
-      commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
-    dependsOn:
-      - setup
-      - build_images
-    displayName: "Create Cluster - ${{ parameters.clusterName }}"
-    jobs:
-      - job: create_aks_cluster_with_${{ parameters.name }}
-        pool:
-          name: "$(BUILD_POOL_NAME_DEFAULT)"
-        steps:
-          - template: ../load-test-templates/create-cluster-template.yaml
-            parameters:
-              clusterType: ${{ parameters.clusterType }}
-              clusterName: ${{ parameters.clusterName }}-$(commitID)
-              nodeCount: ${{ parameters.nodeCount }}
-              vmSize: ${{ parameters.vmSize }}
-              region: $(location)
+  - ${{if eq(parameters.upgradeScenario, false)}}:
+    - stage: create_${{ parameters.name }}
+      condition: and( succeeded(), and( or( contains(variables.CONTROL_CNI, 'cilium') , contains(variables.CONTROL_CNI, 'all') ), or( contains(variables.CONTROL_OS, 'linux'), contains(variables.CONTROL_OS, 'all') ) ) )
+      variables:
+        ${{ if contains(parameters.clusterName, 'rdma') }}:
+          location: $(LOCATION_RDMA)
+        ${{ elseif eq(parameters.arch, 'arm64') }}:
+          location: $(LOCATION_ARM64)
+        ${{ else }}:
+          location: $(LOCATION_AMD64)
+        commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
+      dependsOn:
+        - setup
+        - build_images
+      displayName: "Create Cluster - ${{ parameters.clusterName }}"
+      jobs:
+        - job: create_aks_cluster_with_${{ parameters.name }}
+          pool:
+            name: "$(BUILD_POOL_NAME_DEFAULT)"
+          steps:
+            - template: ../load-test-templates/create-cluster-template.yaml
+              parameters:
+                clusterType: ${{ parameters.clusterType }}
+                clusterName: ${{ parameters.clusterName }}-$(commitID)
+                nodeCount: ${{ parameters.nodeCount }}
+                vmSize: ${{ parameters.vmSize }}
+                region: $(location)
 
 # Conditions for below E2E test scenarios confirm that:
 # Pipeline has not been canceled and that the previous job has reports anything other than failure(Succeeded, SuccededWithIssues, Skipped). Previous job is declared by dependsOn:
@@ -60,7 +62,10 @@ stages:
     pool:
       name: "$(BUILD_POOL_NAME_DEFAULT)"
     dependsOn:
-      - create_${{ parameters.name }}
+      - ${{ if eq(parameters.upgradeScenario, false) }}:
+        - create_${{ parameters.name }}
+      - ${{ else }}:
+        - ${{ parameters.dependsOn }} 
       - publish
       - setup
     displayName: "Cilium Test - ${{ parameters.name }}"
@@ -117,27 +122,7 @@ stages:
 
                   envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f -
                   kubectl get po -owide -A
-
-                  if [ -z "$TEST_AZURE_IPAM_VERSION" ]
-                  then
-                    echo "TEST_AZURE_IPAM_VERSION is not set, using default value"
-                    IPAM=$(make azure-ipam-version)
-                  else
-                    IPAM=$(TEST_AZURE_IPAM_VERSION)
-                    echo "TEST_AZURE_IPAM_VERSION is set to $IPAM"
-                  fi
-                  if [ -z "$TEST_CNS_VERSION" ]
-                  then
-                    echo "TEST_CNS_VERSION is not set, using default value"
-                    CNS=$(make cns-version)
-                  else
-                    CNS=$(TEST_CNS_VERSION)
-                    echo "TEST_CNS_VERSION is set to $CNS"
-                  fi
-                  echo "Deploy Azure-CNS"
-                  sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) 
-                  kubectl get po -owide -A
-                  kubectl get crd -A
+                  
       - ${{if eq(parameters.hubbleEnabled, true)}}:
         - job: deploy_cilium_components
           displayName: Deploy Cilium with Hubble
@@ -169,26 +154,71 @@ stages:
                   envsubst '${CILIUM_IMAGE_REGISTRY},${CILIUM_VERSION_TAG}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f -
                   kubectl get po -owide -A
 
-                  if [ -z "$TEST_AZURE_IPAM_VERSION" ]
-                  then
-                    echo "TEST_AZURE_IPAM_VERSION is not set, using default value"
-                    IPAM=$(make azure-ipam-version)
-                  else
-                    IPAM=$(TEST_AZURE_IPAM_VERSION)
-                    echo "TEST_AZURE_IPAM_VERSION is set to $IPAM"
-                  fi
-                  if [ -z "$TEST_CNS_VERSION" ]
-                  then
-                    echo "TEST_CNS_VERSION is not set, using default value"
-                    CNS=$(make cns-version)
-                  else
-                    CNS=$(TEST_CNS_VERSION)
-                    echo "TEST_CNS_VERSION is set to $CNS"
-                  fi
-                  echo "Deploy Azure-CNS"
-                  sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) 
-                  kubectl get po -owide -A
-                  kubectl get crd -A
+      - job: deploy_cns_and_ipam
+        displayName: "Deploy CNS and IPAM"
+        dependsOn: deploy_cilium_components
+        steps:
+          - task: AzureCLI@2
+            displayName: "Install CNS and IPAM"
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              scriptType: "bash"
+              addSpnToEnvironment: true
+              inlineScript: |
+                set -ex
+                az extension add --name aks-preview
+                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
+                ls -lah
+                pwd
+                kubectl cluster-info
+                kubectl get po -owide -A
+                if [ ${{parameters.upgradeScenario}} = "true" ]
+                then
+                    echo "Upgrade scenario is true, using upgrade azure ipam and cns version from pipeline variables"
+                    if [ -z "$UPGRADE_AZURE_IPAM_VERSION" ]
+                    then
+                      echo "UPGRADE_AZURE_IPAM_VERSION is not set, using default value"
+                      IPAM=$(make azure-ipam-version)
+                    else
+                      IPAM=$(UPGRADE_AZURE_IPAM_VERSION)
+                      echo "UPGRADE_AZURE_IPAM_VERSION is set to $IPAM"
+                    fi
+                    
+                    if [ -z "$UPGRADE_CNS_VERSION" ]
+                    then
+                      echo "UPGRADE_CNS_VERSION is not set, using default value"
+                      CNS=$(make cns-version)
+                    else
+                      CNS=$(UPGRADE_CNS_VERSION)
+                      echo "UPGRADE_CNS_VERSION is set to $CNS"
+                    fi
+                    IPAM_IMAGE_REPO=$(UPGRADE_IPAM_IMAGE_REPO)
+                    CNS_IMAGE_REPO=$(UPGRADE_CNS_IMAGE_REPO)
+                    echo Deploying with Azure Ipam version $IPAM from $IPAM_IMAGE_REPO and CNS version $CNS from $CNS_IMAGE_REPO
+                else
+                    if [ -z "$TEST_AZURE_IPAM_VERSION" ]
+                    then
+                      echo "TEST_AZURE_IPAM_VERSION is not set, using default value"
+                      IPAM=$(make azure-ipam-version)
+                    else
+                      IPAM=$(TEST_AZURE_IPAM_VERSION)
+                      echo "TEST_AZURE_IPAM_VERSION is set to $IPAM"
+                    fi
+                    if [ -z "$TEST_CNS_VERSION" ]
+                    then
+                      echo "TEST_CNS_VERSION is not set, using default value"
+                      CNS=$(make cns-version)
+                    else
+                      CNS=$(TEST_CNS_VERSION)
+                      echo "TEST_CNS_VERSION is set to $CNS"
+                    fi
+                fi
+                  
+                echo "Deploy Azure-CNS"
+                sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) IPAM_IMAGE_REPO=$(IPAM_IMAGE_REPO) 
+                kubectl get po -owide -A
+                kubectl get crd -A
 
       - job: deploy_pods
         condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) )
@@ -203,6 +233,8 @@ stages:
               iterations: ${ITERATIONS_CILIUM}
               nodeCount: ${{ parameters.nodeCount }}
               cni: cilium
+              ${{ if eq(parameters.upgradeScenario, 'true') }}:
+                logType: upgradeScaleTest
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -218,6 +250,8 @@ stages:
               os: ${{ parameters.os }}
               cni: cilium
               region: $(location)
+              ${{ if eq(parameters.upgradeScenario, 'true') }}:
+                logType: upgradeRestartNode
           - template: ../load-test-templates/validate-state-template.yaml
             parameters:
               clusterName: ${{ parameters.clusterName }}-$(commitID)
@@ -235,6 +269,8 @@ stages:
               scaleup: ${SCALEUP_CILIUM}
               nodeCount: ${{ parameters.nodeCount }}
               cni: ${{ parameters.cni }}
+              ${{ if eq(parameters.upgradeScenario, 'true') }}:
+                logType: upgradeRestartCNS
       - job: cni_tests
         displayName: "Cilium Test"
         dependsOn: restart_cns

.pipelines/cni/load-test-templates/pod-deployment-template.yaml

@@ -6,6 +6,7 @@ parameters:
   nodeCount: 10
   cni: ""
   jobName: "deploy_pods"
+  logType: "scaleTest"
 
 steps:
   - task: AzureCLI@2
@@ -26,7 +27,7 @@ steps:
   - template: ../../templates/log-template.yaml
     parameters:
       clusterName: ${{ parameters.clusterName }}
-      logType: scaleTest
       os: ${{ parameters.os }}
       cni: ${{ parameters.cni }}
       jobName: ${{ parameters.jobName }}
+      logType: ${{ parameters.logType }}

.pipelines/cni/load-test-templates/restart-cns-template.yaml

@@ -5,6 +5,7 @@ parameters:
   nodeCount: 10
   os: ""
   jobName: "restart_cns"
+  logType: "restartCNS"
 
 steps:
   - task: AzureCLI@2
@@ -38,7 +39,7 @@ steps:
   - template: ../../templates/log-template.yaml
     parameters:
       clusterName: ${{ parameters.clusterName }}
-      logType: restartCNS
+      logType: ${{ parameters.logType }}
       os: ${{ parameters.os }}
       cni: ${{ parameters.cni }}
       jobName: ${{ parameters.jobName }}

.pipelines/cni/load-test-templates/restart-node-template.yaml

@@ -4,6 +4,7 @@ parameters:
   cni: ""
   jobName: "restart_nodes"
   region: ""
+  logType: "restartNode"
 
 steps:
   - task: AzureCLI@2
@@ -56,7 +57,7 @@ steps:
   - template: ../../templates/log-template.yaml
     parameters:
       clusterName: ${{ parameters.clusterName }}
-      logType: restartNode
+      logType: ${{ parameters.logType }}
       os: ${{ parameters.os }}
       cni: ${{ parameters.cni }}
       jobName: ${{ parameters.jobName }}