Skip to content

Commit 018a323

Browse files
huntergregoryjpayne3506
huntergregory
and
jpayne3506
authored
[backport] fix/ci: Linux NPM race condition editing "except" CIDR and NPM pipeline fixes (#2882)
* [backport] [fix] ci: update cyclonus workflow (#2866) ci: update cyclonus workflow Signed-off-by: Hunter Gregory <[email protected]> * [backport] ci: Update triggers for NPM pipelines (#2867) ci: add triggers to npm pipelines Signed-off-by: Hunter Gregory <[email protected]> * [backport] ci: update Cyclonus workflow triggers (#2868) ci: update Cyclonus triggers Signed-off-by: Hunter Gregory <[email protected]> * [backport] ci: [NPM] speed up Windows conformance and disable Windows Cyclonus/Scale (#2874) * ci: comment out windows cyclonus and windows scale test Signed-off-by: Hunter Gregory <[email protected]> * ci: slim down windows conformance to 14 tests Signed-off-by: Hunter Gregory <[email protected]> * ci: make sure conformance skips "Linux Only" Signed-off-by: Hunter Gregory <[email protected]> * ci: conformance was not running test cases due to formatting Signed-off-by: Hunter Gregory <[email protected]> --------- Signed-off-by: Hunter Gregory <[email protected]> * [backport] ci: [NPM] NPM Conformance Test improvements (#2875) * ci: Always delete npm conformance clusters * test: pin k8s suite version * ci: comment out updated cyclonus * chore: cleanup yaml Signed-off-by: Hunter Gregory <[email protected]> * [backport] ci: [NPM] add bash directive to scale scripts (#2876) ci: add bash directive to scale scripts Signed-off-by: Hunter Gregory <[email protected]> * [backport] fix: [NPM] [Linux] race condition when editing NetPol with "except" CIDR (#2841) * fix: syntax error when deleting nomatch CIDR ipset Signed-off-by: Hunter Gregory <[email protected]> * test: ut members with nomatch Signed-off-by: Hunter Gregory <[email protected]> --------- Signed-off-by: Hunter Gregory <[email protected]> --------- Signed-off-by: Hunter Gregory <[email protected]> Co-authored-by: John Payne <[email protected]>
1 parent a91dcf5 commit 018a323

File tree

9 files changed

+792
-633
lines changed

9 files changed

+792
-633
lines changed

‎.github/workflows/cyclonus-netpol-extended-nightly-test.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ jobs:
7373
uses: actions/upload-artifact@v4
7474
if: always()
7575
with:
76-
name: logs
76+
name: logs-${{ matrix.profile }}
7777
path: |
7878
./npm-logs_${{ matrix.profile }}.txt
7979
./cyclonus-test_${{ matrix.profile }}.txt

‎.github/workflows/cyclonus-netpol-test.yaml‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ on:
55
push:
66
branches:
77
- main
8+
- release/*
89
pull_request:
910
paths:
1011
- 'npm/**'
@@ -49,7 +50,7 @@ jobs:
4950
- name: Make NPM image
5051
run: |
5152
make npm-image NPM_PLATFORM_TAG=cyclonus PLATFORM=linux/amd64 CONTAINER_BUILDER=docker BUILDX_ACTION='--load'
52-
53+
5354
- name: Install Azure NPM
5455
# set the ConfigMap based on the build matrix
5556
# currently have to restart the daemonset because changing the ConfigMap doesn't restart NPM
@@ -60,7 +61,7 @@ jobs:
6061
echo "Applying profile: ${{ matrix.profile }}"
6162
kubectl apply -f ./npm/profiles/${{ matrix.profile }}
6263
kubectl rollout restart ds azure-npm -n kube-system
63-
64+
6465
- name: Check Cluster Components
6566
run: |
6667
sleep 10
@@ -80,7 +81,7 @@ jobs:
8081
uses: actions/upload-artifact@v4
8182
if: always()
8283
with:
83-
name: logs
84+
name: logs-${{ matrix.profile }}
8485
path: |
8586
./npm-logs_${{ matrix.profile }}.txt
8687
./cyclonus-test_${{ matrix.profile }}.txt

‎.pipelines/npm/npm-conformance-tests-latest-release.yaml‎

Lines changed: 138 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
trigger:
2-
- master
2+
branches:
3+
include:
4+
- master
5+
tags:
6+
include:
7+
- "*"
38

49
variables:
510
- name: VNET_NAME
@@ -113,7 +118,7 @@ jobs:
113118
scriptLocation: "inlineScript"
114119
failOnStderr: true
115120
inlineScript: |
116-
# get kubectl
121+
# get kubectl
117122
curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
118123
chmod +x kubectl
119124
echo Cluster $(AZURE_CLUSTER)
@@ -177,7 +182,7 @@ jobs:
177182
./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-ltsc2022-$(TAG)
178183
179184
else
180-
echo "Creating Linux Cluster";
185+
echo "Creating Linux Cluster";
181186
az aks create --no-ssh-key \
182187
--resource-group $(RESOURCE_GROUP) \
183188
--name $(AZURE_CLUSTER) \
@@ -261,7 +266,7 @@ jobs:
261266
declare -a conformancePIDs
262267
for round in $(seq 1 $NUM_PARALLEL_JOBS_FOR_STRESS_TEST); do
263268
# for each iteration, run the conformance test and echos in the background, and write the output of the conformance test to a file
264-
269+
265270
# run the conformance test in the foreground and write the output to stdout and a file
266271
if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching
267272
then
@@ -314,133 +319,134 @@ jobs:
314319
condition: always()
315320
artifact: NpmLogs_$(AZURE_CLUSTER)
316321

317-
- job: Create_Windows_Cluster_and_Run_Test
318-
timeoutInMinutes: 360
319-
displayName: "Run Windows Cyclonus"
320-
pool:
321-
name: $(BUILD_POOL_NAME_DEFAULT)
322-
demands:
323-
- agent.os -equals Linux
324-
- Role -equals Build
325-
dependsOn: [setup]
326-
variables:
327-
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
328-
TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
329-
FQDN: empty
330-
strategy:
331-
matrix:
332-
v2-windows:
333-
PROFILE: "cyc-ws22"
334-
steps:
335-
- checkout: self
336-
- download: none
337-
338-
- task: AzureCLI@2
339-
displayName: "Create AKS Cluster"
340-
inputs:
341-
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
342-
scriptType: "bash"
343-
scriptLocation: "inlineScript"
344-
failOnStderr: true
345-
inlineScript: |
346-
az extension add --name aks-preview
347-
az extension update --name aks-preview
348-
349-
export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
350-
351-
echo "Creating resource group named $CLUSTER_NAME"
352-
az group create --name $CLUSTER_NAME -l $(LOCATION) -o table
353-
354-
echo "Creating resource group named $CLUSTER_NAME"
355-
az aks create \
356-
--resource-group $CLUSTER_NAME \
357-
--name $CLUSTER_NAME \
358-
--generate-ssh-keys \
359-
--windows-admin-username e2eadmin \
360-
--windows-admin-password alpha@numeric!password2 \
361-
--network-plugin azure \
362-
--vm-set-type VirtualMachineScaleSets \
363-
--node-vm-size Standard_D4s_v3 \
364-
--node-count 1
365-
366-
# don't schedule anything on the linux system pool
367-
echo "Updating $CLUSTER_NAME to not schedule anything on linux pool..."
368-
az aks nodepool update \
369-
--cluster-name $CLUSTER_NAME \
370-
-g $CLUSTER_NAME \
371-
-n nodepool1 \
372-
--node-taints CriticalAddonsOnly=true:NoSchedule
373-
374-
echo "Adding Windows nodepool to $CLUSTER_NAME"
375-
az aks nodepool add \
376-
--resource-group $CLUSTER_NAME \
377-
--cluster-name $CLUSTER_NAME \
378-
--name awin22 \
379-
--os-type Windows \
380-
--os-sku Windows2022 \
381-
--node-vm-size Standard_D4s_v3 \
382-
--node-count 3
383-
384-
echo "Getting credentials to $CLUSTER_NAME"
385-
az aks get-credentials -g $CLUSTER_NAME -n $CLUSTER_NAME --overwrite-existing --file ./kubeconfig
386-
mkdir -p ~/.kube/
387-
cp ./kubeconfig ~/.kube/config
388-
389-
- task: AzureCLI@2
390-
displayName: "Deploy NPM to Test Cluster"
391-
inputs:
392-
azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
393-
scriptType: "bash"
394-
scriptLocation: "inlineScript"
395-
failOnStderr: true
396-
inlineScript: |
397-
export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
398-
399-
curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
400-
chmod +x kubectl
401-
402-
# deploy azure-npm
403-
./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
404-
405-
# swap azure-npm image with one built during run
406-
./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-ltsc2022-$(TAG)
407-
408-
echo "sleeping 3 minutes to allow NPM pods to restart"
409-
sleep 180
410-
411-
./kubectl --kubeconfig=./kubeconfig get po -n kube-system -owide -A
412-
413-
echo "Showing cluster status for $CLUSTER_NAME"
414-
FQDN=`az aks show -n $CLUSTER_NAME -g $CLUSTER_NAME --query fqdn -o tsv`
415-
echo "##vso[task.setvariable variable=FQDN]$FQDN"
416-
417-
- script: |
418-
cat ~/.kube/config
419-
curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv
420-
name: download_cyclonus
421-
displayName: "Download Cyclonus"
422-
failOnStderr: false
423-
condition: always()
424-
425-
- script: |
426-
./test/cyclonus/test-cyclonus-windows.sh
427-
name: cyclonus
428-
displayName: "Run Cyclonus Test"
429-
failOnStderr: false
430-
condition: always()
431-
432-
- bash: |
433-
export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
434-
cp cyclonus-$CLUSTER_NAME $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/cyclonus-$CLUSTER_NAME
435-
echo "Getting cluster state for $CLUSTER_NAME"
436-
mkdir -p $(System.DefaultWorkingDirectory)/$CLUSTER_NAME
437-
kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/npm-logs_$(PROFILE).txt
438-
cp ./kubeconfig $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/.kubeconfig
439-
condition: always()
440-
441-
- publish: $(System.DefaultWorkingDirectory)/$(RESOURCE_GROUP)-$(PROFILE)
442-
condition: always()
443-
artifact: NpmLogs-$(RESOURCE_GROUP)-$(PROFILE)
322+
# 2024/07/23: Windows Cyclonus is consistently timing after 6 hours
323+
# - job: Create_Windows_Cluster_and_Run_Test
324+
# timeoutInMinutes: 360
325+
# displayName: "Run Windows Cyclonus"
326+
# pool:
327+
# name: $(BUILD_POOL_NAME_DEFAULT)
328+
# demands:
329+
# - agent.os -equals Linux
330+
# - Role -equals Build
331+
# dependsOn: [setup]
332+
# variables:
333+
# RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
334+
# TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.TAG'] ]
335+
# FQDN: empty
336+
# strategy:
337+
# matrix:
338+
# v2-windows:
339+
# PROFILE: "cyc-ws22"
340+
# steps:
341+
# - checkout: self
342+
# - download: none
343+
344+
# - task: AzureCLI@2
345+
# displayName: "Create AKS Cluster"
346+
# inputs:
347+
# azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
348+
# scriptType: "bash"
349+
# scriptLocation: "inlineScript"
350+
# failOnStderr: true
351+
# inlineScript: |
352+
# az extension add --name aks-preview
353+
# az extension update --name aks-preview
354+
355+
# export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
356+
357+
# echo "Creating resource group named $CLUSTER_NAME"
358+
# az group create --name $CLUSTER_NAME -l $(LOCATION) -o table
359+
360+
# echo "Creating resource group named $CLUSTER_NAME"
361+
# az aks create \
362+
# --resource-group $CLUSTER_NAME \
363+
# --name $CLUSTER_NAME \
364+
# --generate-ssh-keys \
365+
# --windows-admin-username e2eadmin \
366+
# --windows-admin-password alpha@numeric!password2 \
367+
# --network-plugin azure \
368+
# --vm-set-type VirtualMachineScaleSets \
369+
# --node-vm-size Standard_D4s_v3 \
370+
# --node-count 1
371+
372+
# # don't schedule anything on the linux system pool
373+
# echo "Updating $CLUSTER_NAME to not schedule anything on linux pool..."
374+
# az aks nodepool update \
375+
# --cluster-name $CLUSTER_NAME \
376+
# -g $CLUSTER_NAME \
377+
# -n nodepool1 \
378+
# --node-taints CriticalAddonsOnly=true:NoSchedule
379+
380+
# echo "Adding Windows nodepool to $CLUSTER_NAME"
381+
# az aks nodepool add \
382+
# --resource-group $CLUSTER_NAME \
383+
# --cluster-name $CLUSTER_NAME \
384+
# --name awin22 \
385+
# --os-type Windows \
386+
# --os-sku Windows2022 \
387+
# --node-vm-size Standard_D4s_v3 \
388+
# --node-count 3
389+
390+
# echo "Getting credentials to $CLUSTER_NAME"
391+
# az aks get-credentials -g $CLUSTER_NAME -n $CLUSTER_NAME --overwrite-existing --file ./kubeconfig
392+
# mkdir -p ~/.kube/
393+
# cp ./kubeconfig ~/.kube/config
394+
395+
# - task: AzureCLI@2
396+
# displayName: "Deploy NPM to Test Cluster"
397+
# inputs:
398+
# azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
399+
# scriptType: "bash"
400+
# scriptLocation: "inlineScript"
401+
# failOnStderr: true
402+
# inlineScript: |
403+
# export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
404+
405+
# curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl
406+
# chmod +x kubectl
407+
408+
# # deploy azure-npm
409+
# ./kubectl --kubeconfig=./kubeconfig apply -f https://raw.githubusercontent.com/Azure/azure-container-networking/master/npm/examples/windows/azure-npm.yaml
410+
411+
# # swap azure-npm image with one built during run
412+
# ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-ltsc2022-$(TAG)
413+
414+
# echo "sleeping 3 minutes to allow NPM pods to restart"
415+
# sleep 180
416+
417+
# ./kubectl --kubeconfig=./kubeconfig get po -n kube-system -owide -A
418+
419+
# echo "Showing cluster status for $CLUSTER_NAME"
420+
# FQDN=`az aks show -n $CLUSTER_NAME -g $CLUSTER_NAME --query fqdn -o tsv`
421+
# echo "##vso[task.setvariable variable=FQDN]$FQDN"
422+
423+
# - script: |
424+
# cat ~/.kube/config
425+
# curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv
426+
# name: download_cyclonus
427+
# displayName: "Download Cyclonus"
428+
# failOnStderr: false
429+
# condition: always()
430+
431+
# - script: |
432+
# ./test/cyclonus/test-cyclonus-windows.sh
433+
# name: cyclonus
434+
# displayName: "Run Cyclonus Test"
435+
# failOnStderr: false
436+
# condition: always()
437+
438+
# - bash: |
439+
# export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE)
440+
# cp cyclonus-$CLUSTER_NAME $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/cyclonus-$CLUSTER_NAME
441+
# echo "Getting cluster state for $CLUSTER_NAME"
442+
# mkdir -p $(System.DefaultWorkingDirectory)/$CLUSTER_NAME
443+
# kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/npm-logs_$(PROFILE).txt
444+
# cp ./kubeconfig $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/.kubeconfig
445+
# condition: always()
446+
447+
# - publish: $(System.DefaultWorkingDirectory)/$(RESOURCE_GROUP)-$(PROFILE)
448+
# condition: always()
449+
# artifact: NpmLogs-$(RESOURCE_GROUP)-$(PROFILE)
444450

445451
- job: clean_up
446452
displayName: "Cleanup"
@@ -450,7 +456,7 @@ jobs:
450456
- agent.os -equals Linux
451457
- Role -equals Build
452458
dependsOn:
453-
[Create_Cluster_and_Run_Test, Create_Windows_Cluster_and_Run_Test, setup]
459+
[Create_Cluster_and_Run_Test, setup]
454460
variables:
455461
RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
456462
steps:

0 commit comments

Comments
 (0)