Skip to content

Commit dcd55b3

Browse files
vipul-21vipul-21
authored
ci: [CNI] Add restart node in stage in the load test of cni pipeline (#1916)
1 parent cc6aaf5 commit dcd55b3

File tree

5 files changed

+97
-18
lines changed

5 files changed

+97
-18
lines changed

‎.pipelines/cni/cilium/cilium-cni-load-test.yaml‎

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ stages:
3636
set -ex
3737
az extension add --name aks-preview
3838
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
39+
kubectl apply -f hack/manifests/pod.yaml
40+
kubectl apply -f hack/manifests/hostprocess.yaml
3941
bash hack/scripts/scale_deployment.sh
4042
- stage: validate_state
4143
dependsOn: pod_deployment
@@ -56,8 +58,37 @@ stages:
5658
name: "ValidateState"
5759
displayName: "Validate State"
5860
retryCountOnTaskFailure: 3
59-
- stage: connectivity_tests
61+
- stage: restart_nodes
6062
dependsOn: validate_state
63+
displayName: "Restart Node"
64+
jobs:
65+
- job: restart_nodes
66+
steps:
67+
- task: AzureCLI@1
68+
inputs:
69+
azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
70+
scriptLocation: "inlineScript"
71+
scriptType: "bash"
72+
addSpnToEnvironment: true
73+
inlineScript: |
74+
echo "Scale up the pods and immediated restart the nodes"
75+
make -C ./hack/swift set-kubeconf AZCLI=az CLUSTER=${RESOURCE_GROUP}
76+
make -C ./hack/swift azcfg AZCLI=az REGION=$(LOCATION)
77+
echo "Scaling the pods down to 100 per node"
78+
bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -s
79+
echo "Restarting the nodes"
80+
vmss_name=$(az vmss list -g MC_${RESOURCE_GROUP}_${RESOURCE_GROUP}_$(LOCATION) --query "[].name" -o tsv)
81+
make -C ./hack/swift restart-vmss AZCLI=az CLUSTER=${RESOURCE_GROUP} REGION=$(LOCATION) VMSS_NAME=$vmss_name
82+
bash ./hack/scripts/scale_deployment.sh -n 0 -u 1000 -c
83+
name: "RestartNodes"
84+
displayName: "Restart Nodes"
85+
- script: |
86+
bash hack/scripts/validate_state.sh
87+
name: "ValidateState"
88+
displayName: "Validate State"
89+
retryCountOnTaskFailure: 3
90+
- stage: connectivity_tests
91+
dependsOn: restart_nodes
6192
displayName: "Connectivity Tests"
6293
jobs:
6394
- job: cni_tests

‎hack/scripts/scale_deployment.sh‎

Lines changed: 53 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,23 @@
11
#!/bin/bash
22
set -ex
3-
kubectl apply -f hack/manifests/pod.yaml
4-
kubectl apply -f hack/manifests/hostprocess.yaml
5-
sleep 1m
6-
total_num_of_run=4
3+
total_num_of_run=5
74
scale_up_of_pods=2400
85
scale_down_pods=1
9-
echo "Total num of run $total_num_of_run"
6+
7+
function help()
8+
{
9+
echo "Scale deployment based on the parameters."
10+
echo "By default script will repeat the process of scale up/down"
11+
echo
12+
echo "Syntax: scale [-h|n|u|s|c|r]"
13+
echo "options:"
14+
echo "h Print this help."
15+
echo "n Number of times the scale down/scale up task should run."
16+
echo "u Number of pods to be scaled up."
17+
echo "s Scale the pods single time. Accepted Values: true, default : false"
18+
echo "c Check deployment status. Accepted Values: true, default : false"
19+
echo
20+
}
1021

1122
function check_deployment() {
1223
available=-1
@@ -22,16 +33,42 @@ function check_deployment() {
2233
echo "deployment complete."
2334
}
2435

25-
for ((i=1; i <= total_num_of_run; i++))
26-
do
27-
echo "Current Run: $i"
28-
echo "Scaling pods to : $scale_up_of_pods"
29-
kubectl scale deployment container --replicas $scale_up_of_pods
30-
check_deployment $scale_up_of_pods
31-
echo "Scaling down pods to : $scale_down_pods"
32-
kubectl scale deployment container --replicas $scale_down_pods
33-
check_deployment $scale_down_pods
36+
function scale_deployment()
37+
{
38+
desired_replicas=$1
39+
kubectl scale deployment container --replicas "$desired_replicas"
40+
echo "Scaled the deployment to $desired_replicas"
41+
}
42+
43+
function repeat_deployment() {
44+
echo "Total num of run $total_num_of_run"
45+
for ((i=1; i <= total_num_of_run; i++))
46+
do
47+
echo "Current Run: $i"
48+
echo "Scaling down pods to : $scale_down_pods"
49+
scale_deployment $scale_down_pods
50+
check_deployment $scale_down_pods
51+
echo "Scaling pods to : $scale_up_of_pods"
52+
scale_deployment "$scale_up_of_pods"
53+
check_deployment "$scale_up_of_pods"
54+
done
55+
}
56+
57+
while getopts ":h:n:u:sc" option; do
58+
case $option in
59+
h) help
60+
exit;;
61+
n) total_num_of_run=$OPTARG;;
62+
u) scale_up_of_pods=$OPTARG;;
63+
s) echo "Scale deployment"
64+
scale_deployment "$scale_up_of_pods";;
65+
c) echo "Check deployment"
66+
check_deployment "$scale_up_of_pods";;
67+
\?) echo "Error: Invalid option"
68+
exit;;
69+
esac
3470
done
3571

36-
kubectl scale deployment container --replicas $scale_up_of_pods
37-
check_deployment $scale_up_of_pods
72+
if [ "$total_num_of_run" -gt 0 ]; then
73+
repeat_deployment
74+
fi

‎hack/scripts/validate_state.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@ do
1515
node_name="${node##*/}"
1616
node_ip=$(kubectl get "$node" -o jsonpath='{$.status.addresses[?(@.type=="InternalIP")].address}')
1717
echo "Node internal ip: $node_ip"
18+
echo "checking whether the node has any pods deployed to it or not"
19+
pod_count=$(kubectl get pods -o wide | grep "$node_name" -c)
20+
if [[ $pod_count -eq 0 ]]; then
21+
continue
22+
fi
1823
privileged_pod=$(kubectl get pods -n kube-system -l app=privileged-daemonset -o wide | grep "$node_name" | awk '{print $1}')
1924
echo "privileged pod : $privileged_pod"
2025
if [ "$privileged_pod" == '' ]; then
@@ -57,7 +62,8 @@ do
5762
kubectl get pods -A -o wide
5863
exit 1
5964
fi
60-
total_pods_ips=$(echo "$total_pods" | jq -r '(.items[] | .status.podIP)')
65+
66+
total_pods_ips=$(echo "$total_pods" | jq -r '(.items[] | select(.status.podIP != "" and .status.podIP != null)) | .status.podIP')
6167
pod_ips=()
6268
num_of_pod_ips=0
6369
for ip in $total_pods_ips

‎hack/swift/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ vars: ## Show the input vars configured for the cluster commands
5656
@echo OS_SKU=$(OS_SKU)
5757
@echo VM_SIZE=$(VM_SIZE)
5858
@echo NODE_COUNT=$(NODE_COUNT)
59+
@echo VMSS_NAME=$(VMSS_NAME)
5960

6061

6162
##@ SWIFT Infra
@@ -165,3 +166,6 @@ down: ## Delete the cluster
165166
$(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes
166167
@$(MAKE) unset-kubeconf
167168
@$(MAKE) rg-down
169+
170+
restart-vmss: ## Restarts the nodes in the cluster
171+
$(AZCLI) vmss restart -g MC_${GROUP}_${CLUSTER}_${REGION} --name $(VMSS_NAME)

‎hack/swift/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,4 +29,5 @@ AKS Clusters
2929
swift-cilium-up Bring up a SWIFT Cilium cluster
3030
swift-up Bring up a SWIFT AzCNI cluster
3131
down Delete the cluster
32+
vmss-restart Restart the nodes of the cluster
3233
```

0 commit comments

Comments
 (0)