ci: add node and CNS restart test case for v4overlay cluster (#2152)

* add node restart test case for v4overlay cluster

Author: paulyufan2

.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml

@@ -84,6 +84,53 @@ steps:
     displayName: "Get logs"
     condition: always()
 
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |    
+        set -e
+        clusterName=${{ parameters.clusterName }}
+        echo "Restarting nodes"
+        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
+          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
+        done
+    displayName: "Restart Nodes"
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        cd test/integration/load
+        clusterName=${{ parameters.clusterName }}
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
+        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
+        kubectl get pods -owide -A
+        echo "Validating Node Restart"
+        sudo go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load -restart-case=true
+    displayName: "Validate Node Restart"
+    retryCountOnTaskFailure: 3
+
+  - script: |
+      echo "validate pod IP assignment before CNS restart"
+      kubectl get pod -owide -A
+      echo "validate pod state before CNS restarts"
+      cd test/integration/load
+      sudo go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load
+      kubectl rollout restart ds azure-cns -n kube-system
+      kubectl rollout status ds azure-cns -n kube-system
+      kubectl get pod -owide -A
+      echo "validate pods after CNS restart"
+      sudo go test -timeout 30m -tags load -cni cniv2 -run ^TestValidateState$ -tags=load
+    name: "restartCNS_ValidatePodState"
+    displayName: "Restart CNS and validate pod state"
+    retryCountOnTaskFailure: 3
+
   - task: PublishBuildArtifacts@1
     inputs:
       artifactName: test-output

.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml

@@ -64,5 +64,6 @@ steps:
       sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true
       echo "Dualstack Overlay Linux datapath IPv4 test"
       sudo go test -count=1 datapath_linux_test.go -timeout 1m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration
+    retryCountOnTaskFailure: 3
     name: "DualStack_Overlay_Linux_Tests"
     displayName: "DualStack Overlay Linux Tests"