Skip to content

Commit 00e108f

Browse files
santhoshmprabhujpayne3506github-advanced-security[bot]
authored
Pipeline changes to test cilium nodesubnet clusters (#3031)
* feat: pipeline changes to test cilium nodesubnet clusters * Update name * refactor: remove windows changes * refactor: Accept John's comment. Co-authored-by: John Payne <[email protected]> Signed-off-by: Santhosh Prabhu <[email protected]> * refactor: move common tests out to separate template * refactor: address John's comments * refactor: move interface update to golang * fix: add retries to update code * refactor: move ip config update script * tie ip config count to scale up * fix: handle empty scale up * fix: return errors from command run * fix: escape single quotes in network profile json Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> Signed-off-by: Santhosh Prabhu <[email protected]> * fix typo * fix: fix makefile target * fix: fix env variables * chore: rollback --------- Signed-off-by: Santhosh Prabhu <[email protected]> Co-authored-by: John Payne <[email protected]> Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com>
1 parent fb6f1b5 commit 00e108f

File tree

8 files changed

+479
-85
lines changed

8 files changed

+479
-85
lines changed

‎.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-job-template.yaml‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
parameters:
2+
dependsOn: ""
3+
name: "cilium_nodesubnet_e2e"
4+
clusterType: "nodesubnet-byocni-nokubeproxy-up"
5+
clusterName: "cilndsubnete2e"
6+
vmSize: ""
7+
os: "linux"
8+
arch: ""
9+
osSKU: Ubuntu
10+
hubbleEnabled: false
11+
dualstackVersion: ""
12+
cni: "cilium"
13+
14+
stages:
15+
- stage: ${{ parameters.clusterName }}
16+
displayName: Create Cluster - ${{ parameters.displayName }}
17+
dependsOn:
18+
- ${{ parameters.dependsOn }}
19+
- setup
20+
pool:
21+
name: $(BUILD_POOL_NAME_DEFAULT)
22+
variables:
23+
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
24+
jobs:
25+
- template: ../../templates/create-cluster.yaml
26+
parameters:
27+
name: ${{ parameters.name }}
28+
displayName: ${{ parameters.displayName }}
29+
clusterType: ${{ parameters.clusterType }}
30+
clusterName: ${{ parameters.clusterName }}-$(commitID)
31+
vmSize: ${{ parameters.vmSize }}
32+
region: $(REGION_AKS_CLUSTER_TEST)
33+
34+
- stage: ${{ parameters.name }}
35+
displayName: E2E - ${{ parameters.displayName }}
36+
variables:
37+
TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
38+
CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ]
39+
commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ]
40+
GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
41+
GOBIN: "$(GOPATH)/bin" # Go binaries path
42+
modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
43+
condition: and(succeeded(), eq(variables.TAG, variables.CURRENT_VERSION))
44+
dependsOn:
45+
- setup
46+
- publish
47+
- ${{ parameters.clusterName }}
48+
pool:
49+
name: $(BUILD_POOL_NAME_DEFAULT)
50+
jobs:
51+
- job: ${{ parameters.name }}
52+
displayName: Nodesubnet with Cilium - (${{ parameters.name }})
53+
pool:
54+
name: $(BUILD_POOL_NAME_DEFAULT)
55+
demands:
56+
- agent.os -equals Linux
57+
- Role -equals $(CUSTOM_E2E_ROLE)
58+
steps:
59+
- template: cilium-nodesubnet-e2e-step-template.yaml
60+
parameters:
61+
name: ${{ parameters.name }}
62+
clusterName: ${{ parameters.clusterName }}-$(commitID)
63+
arch: ${{ parameters.arch }}
64+
os: ${{ parameters.os }}
65+
scaleup: ${{ parameters.scaleup }}
66+
67+
- template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml
68+
parameters:
69+
sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
70+
clusterName: ${{ parameters.clusterName }}-$(commitID)
71+
os: ${{ parameters.os }}
72+
datapath: true
73+
dns: true
74+
portforward: true
75+
service: true
76+
hostport: true
77+
dependsOn: ${{ parameters.name }}
78+
79+
- job: failedE2ELogs
80+
displayName: "Failure Logs"
81+
dependsOn:
82+
- ${{ parameters.name }}
83+
- cni_${{ parameters.os }}
84+
condition: failed()
85+
steps:
86+
- template: ../../templates/log-template.yaml
87+
parameters:
88+
clusterName: ${{ parameters.clusterName }}-$(commitID)
89+
os: ${{ parameters.os }}
90+
cni: cilium

‎.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e-step-template.yaml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
parameters:
2+
name: ""
3+
clusterName: ""
4+
scaleup: ""
5+
6+
steps:
7+
- bash: |
8+
echo $UID
9+
sudo rm -rf $(System.DefaultWorkingDirectory)/*
10+
displayName: "Set up OS environment"
11+
12+
- checkout: self
13+
14+
- bash: |
15+
go version
16+
go env
17+
mkdir -p '$(GOBIN)'
18+
mkdir -p '$(GOPATH)/pkg'
19+
mkdir -p '$(modulePath)'
20+
echo '##vso[task.prependpath]$(GOBIN)'
21+
echo '##vso[task.prependpath]$(GOROOT)/bin'
22+
name: "GoEnv"
23+
displayName: "Set up the Go environment"
24+
25+
- task: AzureCLI@2
26+
displayName: 'Update IP configs'
27+
inputs:
28+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
29+
scriptLocation: "inlineScript"
30+
scriptType: "bash"
31+
addSpnToEnvironment: true
32+
inlineScript: |
33+
set -e
34+
clusterName=${{ parameters.clusterName }}
35+
SCALE_UP=${{ parameters.scaleup }}
36+
if [ -z "$SCALE_UP" ]; then
37+
SCALE_UP=32
38+
fi
39+
SECONDARY_IP_COUNT=$((SCALE_UP * 2)) \
40+
RESOURCE_GROUP="MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST)" \
41+
go run $(Build.SourcesDirectory)/test/integration/cilium-nodesubnet/ipconfigupdate.go
42+
43+
- task: KubectlInstaller@0
44+
inputs:
45+
kubectlVersion: latest
46+
47+
- task: AzureCLI@2
48+
inputs:
49+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
50+
scriptLocation: "inlineScript"
51+
scriptType: "bash"
52+
addSpnToEnvironment: true
53+
inlineScript: |
54+
set -e
55+
make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
56+
ls -lah
57+
pwd
58+
kubectl cluster-info
59+
kubectl get po -owide -A
60+
echo "install Cilium ${CILIUM_VERSION_TAG}"
61+
export DIR=${CILIUM_VERSION_TAG%.*}
62+
echo "installing files from ${DIR}"
63+
echo "deploy Cilium ConfigMap"
64+
kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-config/cilium-config.yaml
65+
# Passes Cilium image to daemonset and deployment
66+
kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-agent/files
67+
kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-operator/files
68+
69+
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f -
70+
envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f -
71+
kubectl get po -owide -A
72+
name: "installCilium"
73+
displayName: "Install Cilium"
74+
75+
- template: ../../templates/cilium-cli.yaml
76+
77+
- script: |
78+
echo "Start Nodesubnet E2E Tests"
79+
kubectl get po -owide -A
80+
sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_CNS_NODESUBNET=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true
81+
retryCountOnTaskFailure: 3
82+
name: "nodeSubnetE2ETests"
83+
displayName: "Run NodeSubnet E2E"
84+
85+
- template: ../../templates/cilium-tests.yaml

‎.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml‎

Lines changed: 1 addition & 85 deletions
Original file line numberDiff line numberDiff line change
@@ -64,88 +64,4 @@ steps:
6464
name: "aziliumTest"
6565
displayName: "Run Azilium E2E"
6666
67-
- script: |
68-
kubectl get po -owide -A
69-
echo "Waiting < 2 minutes for cilium to be ready"
70-
# Ensure Cilium is ready Xm\Xs
71-
cilium status --wait --wait-duration 2m
72-
retryCountOnTaskFailure: 3
73-
name: "CiliumStatus"
74-
displayName: "Cilium Status"
75-
76-
- task: AzureCLI@1
77-
inputs:
78-
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
79-
scriptLocation: "inlineScript"
80-
scriptType: "bash"
81-
addSpnToEnvironment: true
82-
inlineScript: |
83-
set -e
84-
kubectl get po -owide -A
85-
clusterName=${{ parameters.clusterName }}
86-
echo "Restarting nodes"
87-
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
88-
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
89-
done
90-
displayName: "Restart Nodes"
91-
92-
- task: AzureCLI@1
93-
inputs:
94-
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
95-
scriptLocation: "inlineScript"
96-
scriptType: "bash"
97-
addSpnToEnvironment: true
98-
inlineScript: |
99-
cd test/integration/load
100-
101-
# Scale Cluster Up/Down to confirm functioning CNS
102-
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
103-
kubectl get pods -owide -A
104-
105-
cd ../../..
106-
echo "Validating Node Restart"
107-
make test-validate-state OS_TYPE=linux RESTART_CASE=true
108-
kubectl delete ns load-test
109-
displayName: "Validate Node Restart"
110-
retryCountOnTaskFailure: 3
111-
112-
- script: |
113-
echo "Run Cilium Connectivity Tests"
114-
cilium status
115-
cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy
116-
ns=`kubectl get ns | grep cilium-test | awk '{print $1}'`
117-
echo "##vso[task.setvariable variable=ciliumNamespace]$ns"
118-
retryCountOnTaskFailure: 3
119-
name: "ciliumConnectivityTests"
120-
displayName: "Run Cilium Connectivity Tests"
121-
122-
- script: |
123-
echo "validate pod IP assignment and check systemd-networkd restart"
124-
kubectl get pod -owide -A
125-
# Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change.
126-
# Saves 17 minutes
127-
kubectl delete deploy -n $(ciliumNamespace) echo-external-node
128-
make test-validate-state
129-
echo "delete cilium connectivity test resources and re-validate state"
130-
kubectl delete ns $(ciliumNamespace)
131-
kubectl get pod -owide -A
132-
make test-validate-state
133-
name: "validatePods"
134-
displayName: "Validate Pods"
135-
136-
- script: |
137-
echo "Run wireserver and metadata connectivity Tests"
138-
bash test/network/wireserver_metadata_test.sh
139-
retryCountOnTaskFailure: 3
140-
name: "WireserverMetadataConnectivityTests"
141-
displayName: "Run Wireserver and Metadata Connectivity Tests"
142-
143-
- script: |
144-
cd hack/scripts
145-
chmod +x async-delete-test.sh
146-
./async-delete-test.sh
147-
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
148-
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
149-
fi
150-
name: "testAsyncDelete"
151-
displayName: "Verify Async Delete when CNS is down"
67+
- template: ../../templates/cilium-tests.yaml

‎.pipelines/templates/cilium-tests.yaml‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
steps:
2+
- script: |
3+
kubectl get po -owide -A
4+
echo "Waiting < 2 minutes for cilium to be ready"
5+
# Ensure Cilium is ready Xm\Xs
6+
cilium status --wait --wait-duration 2m
7+
retryCountOnTaskFailure: 3
8+
name: "CiliumStatus"
9+
displayName: "Cilium Status"
10+
11+
- task: AzureCLI@2
12+
inputs:
13+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
14+
scriptLocation: "inlineScript"
15+
scriptType: "bash"
16+
addSpnToEnvironment: true
17+
inlineScript: |
18+
set -e
19+
kubectl get po -owide -A
20+
clusterName=${{ parameters.clusterName }}
21+
echo "Restarting nodes"
22+
for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do
23+
make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val}
24+
done
25+
displayName: "Restart Nodes"
26+
27+
- task: AzureCLI@2
28+
inputs:
29+
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
30+
scriptLocation: "inlineScript"
31+
scriptType: "bash"
32+
addSpnToEnvironment: true
33+
inlineScript: |
34+
cd test/integration/load
35+
36+
# Scale Cluster Up/Down to confirm functioning CNS
37+
ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$
38+
kubectl get pods -owide -A
39+
40+
cd ../../..
41+
echo "Validating Node Restart"
42+
make test-validate-state OS_TYPE=linux RESTART_CASE=true
43+
kubectl delete ns load-test
44+
displayName: "Validate Node Restart"
45+
retryCountOnTaskFailure: 3
46+
47+
- script: |
48+
echo "Run Cilium Connectivity Tests"
49+
cilium status
50+
cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy
51+
ns=`kubectl get ns | grep cilium-test | awk '{print $1}'`
52+
echo "##vso[task.setvariable variable=ciliumNamespace]$ns"
53+
retryCountOnTaskFailure: 3
54+
name: "ciliumConnectivityTests"
55+
displayName: "Run Cilium Connectivity Tests"
56+
57+
- script: |
58+
echo "validate pod IP assignment and check systemd-networkd restart"
59+
kubectl get pod -owide -A
60+
# Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change.
61+
# Saves 17 minutes
62+
kubectl delete deploy -n $(ciliumNamespace) echo-external-node
63+
make test-validate-state
64+
echo "delete cilium connectivity test resources and re-validate state"
65+
kubectl delete ns $(ciliumNamespace)
66+
kubectl get pod -owide -A
67+
make test-validate-state
68+
name: "validatePods"
69+
displayName: "Validate Pods"
70+
71+
- script: |
72+
echo "Run wireserver and metadata connectivity Tests"
73+
bash test/network/wireserver_metadata_test.sh
74+
retryCountOnTaskFailure: 3
75+
name: "WireserverMetadataConnectivityTests"
76+
displayName: "Run Wireserver and Metadata Connectivity Tests"
77+
78+
- script: |
79+
cd hack/scripts
80+
chmod +x async-delete-test.sh
81+
./async-delete-test.sh
82+
if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then
83+
kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]'
84+
fi
85+
name: "testAsyncDelete"
86+
displayName: "Verify Async Delete when CNS is down"

‎hack/aks/Makefile‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,24 @@ byocni-up: swift-byocni-up ## Alias to swift-byocni-up
9595
cilium-up: swift-cilium-up ## Alias to swift-cilium-up
9696
up: swift-up ## Alias to swift-up
9797

98+
99+
nodesubnet-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an NodeSubnet BYO CNI cluster without kube-proxy
100+
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
101+
--auto-upgrade-channel $(AUTOUPGRADE) \
102+
--node-os-upgrade-channel $(NODEUPGRADE) \
103+
--kubernetes-version $(K8S_VER) \
104+
--node-count $(NODE_COUNT) \
105+
--node-vm-size $(VM_SIZE) \
106+
--load-balancer-sku basic \
107+
--max-pods 250 \
108+
--network-plugin none \
109+
--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
110+
--os-sku $(OS_SKU) \
111+
--no-ssh-key \
112+
--kube-proxy-config ./kube-proxy.json \
113+
--yes
114+
@$(MAKE) set-kubeconf
115+
98116
overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster
99117
$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
100118
--auto-upgrade-channel $(AUTOUPGRADE) \

0 commit comments

Comments
 (0)