Skip to content

Commit f7f13eb

Browse files
jpayne3506jpayne3506
authored
ci: Add CNI | LSG Integration Pipeline (#2551)
* ci: Kernel upgrade template * ci: lsg cni intergration template * chore: address comments * chore: template style * ci: clean up env vars * ci: LSG runtime variables * ci: lsg cni intergration template * ci: change logic to check for only 1 kernel type * test: parameter template * test: pipeline variable cleanup * ci: improve upgrade regex * chore: address integ. comments * ci: finalize changes
1 parent f8e451d commit f7f13eb

File tree

4 files changed

+424
-1
lines changed

4 files changed

+424
-1
lines changed

‎.pipelines/cni/lsg/kernel-upgrade-template.yaml‎

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
parameters:
2+
clusterName: ""
3+
4+
steps:
5+
- task: AzureCLI@1
6+
inputs:
7+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
8+
scriptLocation: "inlineScript"
9+
scriptType: "bash"
10+
addSpnToEnvironment: true
11+
inlineScript: |
12+
set -ex
13+
az extension add --name aks-preview
14+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
15+
retryCountOnTaskFailure: 3
16+
displayName: "Set Kubeconfig"
17+
18+
- script: |
19+
set -e
20+
21+
echo "-- Start privileged daemonset --"
22+
kubectl get pods -Aowide
23+
kubectl apply -f test/integration/manifests/load/privileged-daemonset.yaml
24+
sleep 3s
25+
kubectl rollout status ds -n kube-system privileged-daemonset
26+
retryCountOnTaskFailure: 3
27+
name: "priv_daemonset"
28+
displayName: "Create Daemonset"
29+
30+
- script: |
31+
set -e
32+
33+
kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide
34+
privList=`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'`
35+
for pod in $privList; do
36+
echo "-- Update Ubuntu Packages --"
37+
# Not needed, but ensures that the correct packages exist to perform upgrade
38+
kubectl exec -i -n kube-system $pod -- bash -c "apt update && apt-get install software-properties-common -y"
39+
done
40+
retryCountOnTaskFailure: 3
41+
name: "Ubuntu_Setup"
42+
displayName: "Ubuntu Setup"
43+
44+
- script: |
45+
echo "-- Update kernel through daemonset --"
46+
kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide
47+
privList=`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'`
48+
for pod in $privList; do
49+
echo "-- Add proposed repository --"
50+
kubectl exec -i -n kube-system $pod -- bash -c "add-apt-repository ppa:canonical-kernel-team/${{ parameters.ProposedRepoVersion }} -y"
51+
echo "-- Check apt-cache --"
52+
kubectl exec -i -n kube-system $pod -- bash -c "apt-cache madison $(kernelType)"
53+
54+
echo "-- Check current Ubuntu kernel --"
55+
kubectl exec -i -n kube-system $pod -- bash -c "uname -r"
56+
kubectl get node -owide
57+
done
58+
59+
for pod in $privList; do
60+
flag1=""
61+
flag2=""
62+
echo "-- Confirm Proposed Repository --"
63+
temp=`kubectl exec -i -n kube-system $pod -- bash -c "apt-cache madison $(kernelType)" | grep proposed | awk '{print $3}'`
64+
for t in $temp; do
65+
if [[ $t =~ ${{ parameters.KernelVersion }} ]]; then
66+
echo "$t - looking for ${{ parameters.KernelVersion }}"
67+
flag1="true"
68+
fi
69+
done
70+
if [ -z $flag1 ]; then
71+
echo "Missing required proposed repos"
72+
exit 1
73+
fi
74+
done
75+
retryCountOnTaskFailure: 3
76+
name: "Kernel_Setup"
77+
displayName: "Kernel Setup"
78+
79+
- script: |
80+
echo "-- Update kernel through daemonset --"
81+
kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide
82+
privList=`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'`
83+
for pod in $privList; do
84+
echo "-- Install Proposed Kernel --"
85+
kubectl exec -i -n kube-system $pod -- bash -c "apt install -y $(kernelType)"
86+
done
87+
88+
for pod in $privList; do
89+
echo "-- Check for Installed Kernel --"
90+
installed=`kubectl exec -i -n kube-system $pod -- bash -c "apt list $(kernelType) --installed" | grep installed | awk '{print $4}'`
91+
if [ -z $installed ]; then
92+
echo $pod needs to reinstall kernel
93+
exit 1
94+
fi
95+
done
96+
retryCountOnTaskFailure: 3
97+
name: "Kernel_Install"
98+
displayName: "Kernel Install"
99+
100+
- script: |
101+
echo "-- Restart Nodes to Finalize Upgrade Kernel --"
102+
kubectl get node -owide
103+
kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide
104+
105+
privArray=(`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'`)
106+
nodeArray=(`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $7}'`)
107+
108+
i=0
109+
for _ in ${privArray[@]}; do
110+
echo "-- Restarting Node ${nodeArray[i]} through ${privArray[i]} --"
111+
kubectl exec -i -n kube-system ${privArray[i]} -- bash -c "reboot"
112+
echo "-- Waiting for condition NotReady --"
113+
kubectl wait --for=condition=Ready=false -n kube-system pod/${privArray[i]} --timeout=90s
114+
echo "-- Waiting for condition Ready --"
115+
kubectl wait --for=condition=Ready -n kube-system pod/${privArray[i]} --timeout=90s
116+
((i = i + 1))
117+
echo "Wait 10s for pods to settle"
118+
sleep 10s
119+
done
120+
121+
privList=`kubectl get pods -n kube-system -l os=linux,app=privileged-daemonset -owide --no-headers | awk '{print $1}'`
122+
kubectl rollout status ds -n kube-system privileged-daemonset
123+
kubectl get node -owide
124+
125+
for pod in $privList; do
126+
echo "-- Confirm Ubuntu Kernel --"
127+
kernelOut=`kubectl exec -i -n kube-system $pod -- bash -c "uname -r"`
128+
ver=(${kernelOut//-/ }) # Slices $kernelOut with '-'
129+
echo "$pod | kernel version - $kernelOut, expected - ${{ parameters.KernelVersion }}"
130+
if [[ ! ${{ parameters.KernelVersion }} =~ ${ver[1]} ]]; then
131+
exit 1
132+
fi
133+
done
134+
retryCountOnTaskFailure: 3
135+
name: "Kernel_Upgrade"
136+
displayName: "Kernel Upgrade"
137+

‎.pipelines/cni/lsg/lsg-cni-intergration-template.yaml‎

Lines changed: 179 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,179 @@
1+
parameters:
2+
dependsOn: ""
3+
name: ""
4+
clusterType: ""
5+
clusterName: ""
6+
nodeCount: ""
7+
vmSize: ""
8+
os: linux
9+
arch: ""
10+
osSKU: Ubuntu
11+
12+
# CNIv2
13+
# + Should be able to scale up/down the pods successfully certain number of times.
14+
# + Node reboot scenarios should be covered.
15+
# + The CNS state should be validated with that of CNI state.
16+
# + Pods should have ips assigned and connectivity/datapath test should be present.
17+
# + CNS restart and validates the state
18+
19+
stages:
20+
- stage: create_${{ parameters.name }}
21+
variables:
22+
commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
23+
pool:
24+
name: $(BUILD_POOL_NAME_DEFAULT)
25+
dependsOn:
26+
- setup
27+
displayName: "Create Cluster - ${{ parameters.clusterName }}"
28+
jobs:
29+
- job: create_aks_cluster_with_${{ parameters.name }}
30+
steps:
31+
- task: AzureCLI@1
32+
inputs:
33+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
34+
scriptLocation: "inlineScript"
35+
scriptType: "bash"
36+
addSpnToEnvironment: true
37+
inlineScript: |
38+
make -C ./hack/aks ${{ parameters.clusterType }} \
39+
AZCLI=az REGION=$(LOCATION) SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \
40+
CLUSTER=${{ parameters.clusterName }}-$(commitID) \
41+
VM_SIZE=${{ parameters.vmSize }} \
42+
AUTOUPGRADE=none
43+
name: "CreateAksCluster"
44+
displayName: "Create AKS Cluster"
45+
46+
- stage: ${{ parameters.name }}
47+
variables:
48+
commitID: $[ stagedependencies.setup.env.outputs['SetEnvVars.commitID'] ]
49+
cnsVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cnsVersion'] ]
50+
cniVersion: $[ stagedependencies.setup.env.outputs['SetEnvVars.cniVersion'] ]
51+
kernelType: $[ stagedependencies.setup.env.outputs['SetEnvVars.kernelType'] ]
52+
pool:
53+
name: $(BUILD_POOL_NAME_DEFAULT)
54+
dependsOn:
55+
- create_${{ parameters.name }}
56+
- setup
57+
displayName: "CNIv2 Test - ${{ parameters.name }}"
58+
jobs:
59+
- job: integration
60+
displayName: "Integration Test - ${{ parameters.name }}"
61+
steps:
62+
- task: AzureCLI@1
63+
inputs:
64+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
65+
scriptLocation: "inlineScript"
66+
scriptType: "bash"
67+
addSpnToEnvironment: true
68+
inlineScript: |
69+
echo cns version - $(cnsVersion)
70+
echo cni version - $(cniVersion)
71+
72+
echo "Start Integration Tests on Overlay Cluster"
73+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
74+
kubectl cluster-info
75+
kubectl get po -owide -A
76+
sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(cnsVersion) CNI_VERSION=$(cniVersion) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) CNI_IMAGE_REPO=$(CNI_IMAGE_REPO)
77+
name: "overlaye2e"
78+
displayName: "Overlay Integration"
79+
retryCountOnTaskFailure: 2
80+
- job: kernel_upgrade
81+
displayName: "Kernel Node Upgrade"
82+
dependsOn: integration
83+
steps:
84+
- template: kernel-upgrade-template.yaml
85+
parameters:
86+
clusterName: ${{ parameters.clusterName }}-$(commitID)
87+
KernelVersion : ${{ parameters.KernelVersion }}
88+
ProposedRepoVersion : ${{ parameters.ProposedRepoVersion }}
89+
- job: deploy_pods
90+
displayName: "Scale Test"
91+
dependsOn: kernel_upgrade
92+
steps:
93+
- template: ../load-test-templates/pod-deployment-template.yaml
94+
parameters:
95+
clusterName: ${{ parameters.clusterName }}-$(commitID)
96+
scaleup: ${SCALEUP_LINUX}
97+
os: ${{ parameters.os }}
98+
iterations: ${ITERATIONS_LINUX}
99+
nodeCount: ${{ parameters.nodeCount }}
100+
cni: cniv2
101+
- template: ../load-test-templates/validate-state-template.yaml
102+
parameters:
103+
clusterName: ${{ parameters.clusterName }}-$(commitID)
104+
os: ${{ parameters.os }}
105+
cni: cniv2
106+
- job: restart_nodes
107+
displayName: "Restart Test"
108+
dependsOn: deploy_pods
109+
steps:
110+
- template: ../load-test-templates/restart-node-template.yaml
111+
parameters:
112+
clusterName: ${{ parameters.clusterName }}-$(commitID)
113+
os: ${{ parameters.os }}
114+
cni: cniv2
115+
- template: ../load-test-templates/validate-state-template.yaml
116+
parameters:
117+
clusterName: ${{ parameters.clusterName }}-$(commitID)
118+
os: ${{ parameters.os }}
119+
cni: cniv2
120+
restartCase: "true"
121+
- job: restart_cns
122+
displayName: "Restart and Validate CNS"
123+
dependsOn: restart_nodes
124+
steps:
125+
- template: ../load-test-templates/restart-cns-template.yaml
126+
parameters:
127+
clusterName: ${{ parameters.clusterName }}-$(commitID)
128+
os: ${{ parameters.os }}
129+
cni: cniv2
130+
scaleup: ${SCALEUP_LINUX}
131+
nodeCount: ${{ parameters.nodeCount }}
132+
- job: recover
133+
displayName: "Recover Resources"
134+
dependsOn: restart_cns
135+
steps:
136+
- task: AzureCLI@1
137+
inputs:
138+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
139+
scriptLocation: "inlineScript"
140+
scriptType: "bash"
141+
addSpnToEnvironment: true
142+
inlineScript: |
143+
echo "Delete load-test Namespace"
144+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID)
145+
kubectl get ns --no-headers | grep -v 'kube\|default' | awk '{print $1}'
146+
delete=`kubectl get ns --no-headers | grep -v 'kube\|default' | awk '{print $1}'`
147+
kubectl delete ns $delete
148+
kubectl cluster-info
149+
kubectl get po -owide -A
150+
name: "recover"
151+
displayName: "Delete test Namespaces"
152+
- template: ../k8s-e2e/k8s-e2e-job-template.yaml
153+
parameters:
154+
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
155+
clusterName: ${{ parameters.clusterName }}-$(commitID)
156+
os: ${{ parameters.os }}
157+
dependsOn: recover
158+
datapath: true
159+
dns: true
160+
portforward: true
161+
service: true
162+
hostport: true
163+
- job: failedE2ELogs
164+
displayName: "Failure Logs"
165+
dependsOn:
166+
- integration
167+
- kernel_upgrade
168+
- deploy_pods
169+
- restart_nodes
170+
- restart_cns
171+
- recover
172+
- cni_${{ parameters.os }}
173+
condition: failed()
174+
steps:
175+
- template: ../../templates/log-template.yaml
176+
parameters:
177+
clusterName: ${{ parameters.clusterName }}-$(commitID)
178+
os: ${{ parameters.os }}
179+
cni: cniv2

0 commit comments

Comments
 (0)