Skip to content

Commit 742ca41

Browse files
jpayne3506jpayne3506
authored
ci: Add node restart to cniv2 E2E (#2901)
* ci: add node restart to cniv2 E2E * chore: change directory after scale * chore: fixup nodepool * chore: pipeline cleanup * ci: cleanup test resources * chore: address comments
1 parent f3ead91 commit 742ca41

14 files changed

+347
-93
lines changed

‎.pipelines/singletenancy/aks-swift/e2e-job-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ stages:
5959
parameters:
6060
name: ${{ parameters.name }}
6161
clusterName: ${{ parameters.clusterName }}-$(commitID)
62+
scaleup: 100
6263

6364
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
6465
parameters:

‎.pipelines/singletenancy/aks-swift/e2e-step-template.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
parameters:
22
name: ""
33
clusterName: ""
4+
scaleup: ""
45

56
steps:
67
- bash: |
@@ -47,6 +48,42 @@ steps:
4748
name: "aksswifte2e"
4849
displayName: "Run AKS Swift E2E"
4950
51+
- task: AzureCLI@1
52+
inputs:
53+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
54+
scriptLocation: "inlineScript"
55+
scriptType: "bash"
56+
addSpnToEnvironment: true
57+
inlineScript: |
58+
set -e
59+
kubectl get po -owide -A
60+
clusterName=${{ parameters.clusterName }}
61+
echo "Restarting nodes"
62+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
63+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
64+
done
65+
displayName: "Restart Nodes"
66+
67+
- task: AzureCLI@1
68+
inputs:
69+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
70+
scriptLocation: "inlineScript"
71+
scriptType: "bash"
72+
addSpnToEnvironment: true
73+
inlineScript: |
74+
cd test/integration/load
75+
76+
# Scale Cluster Up/Down to confirm functioning CNS
77+
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
78+
kubectl get pods -owide -A
79+
80+
cd ../../..
81+
echo "Validating Node Restart"
82+
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
83+
kubectl delete ns load-test
84+
displayName: "Validate Node Restart"
85+
retryCountOnTaskFailure: 3
86+
5087
- script: |
5188
echo "Run wireserver and metadata connectivity Tests"
5289
bash test/network/wireserver_metadata_test.sh

‎.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ stages:
4545
jobs:
4646
- job: ${{ parameters.name }}_linux
4747
displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }})
48+
timeoutInMinutes: 120
4849
pool:
4950
name: $(BUILD_POOL_NAME_DEFAULT)
5051
demands:
@@ -56,10 +57,39 @@ stages:
5657
name: ${{ parameters.name }}
5758
clusterName: ${{ parameters.clusterName }}-$(commitID)
5859
os: linux
60+
scaleup: 100
61+
62+
- job: windows_nodepool
63+
displayName: Add Windows Nodepool
64+
dependsOn: ${{ parameters.name }}_linux
65+
pool:
66+
name: $(BUILD_POOL_NAME_DEFAULT)
67+
demands:
68+
- agent.os -equals Linux
69+
- Role -equals $(CUSTOM_E2E_ROLE)
70+
steps:
71+
- task: AzureCLI@2
72+
inputs:
73+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
74+
scriptLocation: "inlineScript"
75+
scriptType: "bash"
76+
addSpnToEnvironment: true
77+
inlineScript: |
78+
set -e
79+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
80+
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }}
81+
echo "Windows node are successfully added to v4 Overlay Cluster"
82+
kubectl cluster-info
83+
kubectl get node -owide
84+
kubectl get po -owide -A
85+
name: "Add_Windows_Node"
86+
displayName: "Add windows node on v4 overlay cluster"
87+
5988

6089
- job: ${{ parameters.name }}_windows
6190
displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }})
62-
dependsOn: ${{ parameters.name }}_linux
91+
timeoutInMinutes: 120
92+
dependsOn: windows_nodepool
6393
pool:
6494
name: $(BUILD_POOL_NAME_DEFAULT)
6595
demands:
@@ -71,7 +101,7 @@ stages:
71101
name: ${{ parameters.name }}
72102
clusterName: ${{ parameters.clusterName }}-$(commitID)
73103
os: windows
74-
vmSizeWin: ${{ parameters.vmSize }} # Matching linux vmSize
104+
scaleup: 50
75105

76106
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
77107
parameters:

‎.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml‎

Lines changed: 44 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ parameters:
22
name: ""
33
clusterName: ""
44
os: ""
5+
scaleup: ""
56

67
steps:
78
- bash: |
@@ -64,30 +65,18 @@ steps:
6465
addSpnToEnvironment: true
6566
inlineScript: |
6667
cd test/integration/load
67-
clusterName=${{ parameters.clusterName }}
68-
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
69-
make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
68+
69+
# Scale Cluster Up/Down to confirm functioning CNS
70+
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
7071
kubectl get pods -owide -A
72+
73+
cd ../../..
7174
echo "Validating Node Restart"
72-
CNI_TYPE=cniv2 RESTART_CASE=true go test -timeout 30m -tags load -run ^TestValidateState$
75+
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2
76+
kubectl delete ns load-test
7377
displayName: "Validate Node Restart"
7478
retryCountOnTaskFailure: 3
7579

76-
- script: |
77-
echo "validate pod IP assignment before CNS restart"
78-
kubectl get pod -owide -A
79-
echo "validate pod state before CNS restarts"
80-
cd test/integration/load
81-
CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
82-
kubectl rollout restart ds azure-cns -n kube-system
83-
kubectl rollout status ds azure-cns -n kube-system
84-
kubectl get pod -owide -A
85-
echo "validate pods after CNS restart"
86-
CNI_TYPE=cniv2 go test -timeout 30m -tags load -run ^TestValidateState$
87-
name: "restartCNS_ValidatePodState"
88-
displayName: "Restart CNS and validate pod state"
89-
retryCountOnTaskFailure: 3
90-
9180
- script: |
9281
echo "Run wireserver and metadata connectivity Tests"
9382
bash test/network/wireserver_metadata_test.sh
@@ -96,22 +85,6 @@ steps:
9685
displayName: "Run Wireserver and Metadata Connectivity Tests"
9786
9887
- ${{ if eq(parameters.os, 'windows') }}:
99-
- task: AzureCLI@2
100-
inputs:
101-
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
102-
scriptLocation: "inlineScript"
103-
scriptType: "bash"
104-
addSpnToEnvironment: true
105-
inlineScript: |
106-
set -e
107-
make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }}
108-
echo "Windows node are successfully added to v4 Overlay Cluster"
109-
kubectl cluster-info
110-
kubectl get node -owide
111-
kubectl get po -owide -A
112-
name: "Add_Windows_Node"
113-
displayName: "Add windows node on v4 overlay cluster"
114-
11588
- script: |
11689
nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'`
11790
for node in $nodeList; do
@@ -132,3 +105,39 @@ steps:
132105
name: "WindowsV4OverlayDatapathTests"
133106
displayName: "Windows v4Overlay Datapath Tests"
134107
retryCountOnTaskFailure: 3
108+
109+
- task: AzureCLI@1
110+
inputs:
111+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
112+
scriptLocation: "inlineScript"
113+
scriptType: "bash"
114+
addSpnToEnvironment: true
115+
inlineScript: |
116+
set -e
117+
kubectl get po -owide -A
118+
clusterName=${{ parameters.clusterName }}
119+
echo "Restarting nodes"
120+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
121+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
122+
done
123+
displayName: "Restart Nodes"
124+
125+
- task: AzureCLI@1
126+
inputs:
127+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
128+
scriptLocation: "inlineScript"
129+
scriptType: "bash"
130+
addSpnToEnvironment: true
131+
inlineScript: |
132+
cd test/integration/load
133+
134+
# Scale Cluster Up/Down to confirm functioning CNS
135+
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$
136+
kubectl get pods -owide -A
137+
138+
cd ../../..
139+
echo "Validating Node Restart"
140+
make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2
141+
kubectl delete ns load-test
142+
displayName: "Validate Node Restart"
143+
retryCountOnTaskFailure: 3

‎.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ stages:
5555
parameters:
5656
name: ${{ parameters.name }}
5757
clusterName: ${{ parameters.clusterName }}-$(commitID)
58+
scaleup: 100
5859

5960
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
6061
parameters:

‎.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
parameters:
22
name: ""
33
clusterName: ""
4+
scaleup: ""
45

56
steps:
67

@@ -68,6 +69,42 @@ steps:
6869
name: "CiliumStatus"
6970
displayName: "Cilium Status"
7071
72+
- task: AzureCLI@1
73+
inputs:
74+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
75+
scriptLocation: "inlineScript"
76+
scriptType: "bash"
77+
addSpnToEnvironment: true
78+
inlineScript: |
79+
set -e
80+
kubectl get po -owide -A
81+
clusterName=${{ parameters.clusterName }}
82+
echo "Restarting nodes"
83+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
84+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
85+
done
86+
displayName: "Restart Nodes"
87+
88+
- task: AzureCLI@1
89+
inputs:
90+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
91+
scriptLocation: "inlineScript"
92+
scriptType: "bash"
93+
addSpnToEnvironment: true
94+
inlineScript: |
95+
cd test/integration/load
96+
97+
# Scale Cluster Up/Down to confirm functioning CNS
98+
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
99+
kubectl get pods -owide -A
100+
101+
cd ../../..
102+
echo "Validating Node Restart"
103+
make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack
104+
kubectl delete ns load-test
105+
displayName: "Validate Node Restart"
106+
retryCountOnTaskFailure: 3
107+
71108
- script: |
72109
echo "Run Cilium Connectivity Tests"
73110
cilium status
@@ -109,7 +146,7 @@ steps:
109146
retryCountOnTaskFailure: 3
110147
name: "WireserverMetadataConnectivityTests"
111148
displayName: "Run Wireserver and Metadata Connectivity Tests"
112-
149+
113150
- script: |
114151
cd hack/scripts
115152
chmod +x async-delete-test.sh

‎.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ stages:
5959
name: ${{ parameters.name }}
6060
clusterName: ${{ parameters.clusterName }}-$(commitID)
6161
testHubble: ${{ parameters.testHubble }}
62+
scaleup: 100
6263

6364
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
6465
parameters:

‎.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml‎

Lines changed: 38 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ parameters:
22
name: ""
33
clusterName: ""
44
testHubble: false
5+
scaleup: ""
56

67
steps:
78
- bash: |
@@ -72,6 +73,42 @@ steps:
7273
name: "CiliumStatus"
7374
displayName: "Cilium Status"
7475
76+
- task: AzureCLI@1
77+
inputs:
78+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
79+
scriptLocation: "inlineScript"
80+
scriptType: "bash"
81+
addSpnToEnvironment: true
82+
inlineScript: |
83+
set -e
84+
kubectl get po -owide -A
85+
clusterName=${{ parameters.clusterName }}
86+
echo "Restarting nodes"
87+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
88+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
89+
done
90+
displayName: "Restart Nodes"
91+
92+
- task: AzureCLI@1
93+
inputs:
94+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
95+
scriptLocation: "inlineScript"
96+
scriptType: "bash"
97+
addSpnToEnvironment: true
98+
inlineScript: |
99+
cd test/integration/load
100+
101+
# Scale Cluster Up/Down to confirm functioning CNS
102+
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
103+
kubectl get pods -owide -A
104+
105+
cd ../../..
106+
echo "Validating Node Restart"
107+
make test-validate-state OS_TYPE=linux RESTART_CASE=true
108+
kubectl delete ns load-test
109+
displayName: "Validate Node Restart"
110+
retryCountOnTaskFailure: 3
111+
75112
- script: |
76113
echo "Run Cilium Connectivity Tests"
77114
cilium status
@@ -147,7 +184,7 @@ steps:
147184
retryCountOnTaskFailure: 3
148185
name: "WireserverMetadataConnectivityTests"
149186
displayName: "Run Wireserver and Metadata Connectivity Tests"
150-
187+
151188
- script: |
152189
cd hack/scripts
153190
chmod +x async-delete-test.sh

‎.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-job-template.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ stages:
5757
parameters:
5858
name: ${{ parameters.name }}
5959
clusterName: ${{ parameters.clusterName }}-$(commitID)
60+
scaleup: 100
6061

6162
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
6263
parameters:

0 commit comments

Comments
 (0)