diff --git a/.pipelines/npm/npm-conformance-tests.yaml b/.pipelines/npm/npm-conformance-tests.yaml index ae68233dc0..f46df93efd 100644 --- a/.pipelines/npm/npm-conformance-tests.yaml +++ b/.pipelines/npm/npm-conformance-tests.yaml @@ -74,10 +74,6 @@ stages: arch: amd64 name: npm os: linux - npm_windows_amd64: - arch: amd64 - name: npm - os: windows steps: - template: ../containers/container-template.yaml parameters: @@ -102,10 +98,6 @@ stages: AZURE_CLUSTER: "conformance-v2-background" PROFILE: "v2-background" IS_STRESS_TEST: "false" - v2-ws22: - AZURE_CLUSTER: "conformance-v2-ws22" - PROFILE: "v2-default-ws22" - IS_STRESS_TEST: "false" v2-linux-stress: AZURE_CLUSTER: "conformance-v2-linux-stress" PROFILE: "v2-linux-stress" @@ -172,90 +164,30 @@ stages: export PUBLIC_IP_ID=$(az network public-ip show -g $(RESOURCE_GROUP) -n $(PUBLIC_IP_NAME) --query id -o tsv) echo Public IP ID $PUBLIC_IP_ID - if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching - then - az extension add --name aks-preview - az extension update --name aks-preview - - echo "creating WS22 Cluster"; - az aks create \ - --resource-group $(RESOURCE_GROUP) \ - --name $(AZURE_CLUSTER) \ - --generate-ssh-keys \ - --windows-admin-username e2eadmin \ - --windows-admin-password alpha@numeric!password2 \ - --network-plugin azure \ - --vm-set-type VirtualMachineScaleSets \ - --node-vm-size Standard_D4s_v3 \ - --node-count 1 \ - --load-balancer-outbound-ips $PUBLIC_IP_ID - - if [ $? != 0 ] - then - echo "Failing fast since previous command failed" - exit 1 - fi - - # don't schedule anything on the linux system pool - echo "Updating $(AZURE_CLUSTER) to not schedule anything on linux pool..." - az aks nodepool update \ - --cluster-name $(AZURE_CLUSTER) \ - -g $(RESOURCE_GROUP) \ - -n nodepool1 \ - --node-taints CriticalAddonsOnly=true:NoSchedule - - if [ $? != 0 ] - then - echo "Failing fast since previous command failed" - exit 1 - fi - - echo "Adding Windows nodepool to $(AZURE_CLUSTER) to group $(RESOURCE_GROUP)" - az aks nodepool add \ - --resource-group $(RESOURCE_GROUP) \ - --cluster-name $(AZURE_CLUSTER) \ - --name awin22 \ - --os-type Windows \ - --os-sku Windows2022 \ - --node-vm-size Standard_D4s_v3 \ - --node-count 2 - - if [ $? != 0 ] - then - echo "Failing fast since previous command failed" - exit 1 - fi - - az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig - ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/examples/windows/azure-npm.yaml - ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-$(TAG) + echo "Creating Linux Cluster"; + az aks create --no-ssh-key \ + --resource-group $(RESOURCE_GROUP) \ + --name $(AZURE_CLUSTER) \ + --network-plugin azure \ + --load-balancer-outbound-ips $PUBLIC_IP_ID - else - echo "Creating Linux Cluster"; - az aks create --no-ssh-key \ - --resource-group $(RESOURCE_GROUP) \ - --name $(AZURE_CLUSTER) \ - --network-plugin azure \ - --load-balancer-outbound-ips $PUBLIC_IP_ID - - if [ $? != 0 ] - then - echo "Failing fast since previous command failed" - exit 1 - fi + if [ $? != 0 ] + then + echo "Failing fast since previous command failed" + exit 1 + fi - az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig + az aks get-credentials -n $(AZURE_CLUSTER) -g $(RESOURCE_GROUP) --file ./kubeconfig - # deploy azure-npm - ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/azure-npm.yaml + # deploy azure-npm + ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/azure-npm.yaml - # swap azure-npm image with one built during run - ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:linux-amd64-$(TAG) + # swap azure-npm image with one built during run + ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:linux-amd64-$(TAG) - # swap NPM profile with one specified as parameter - ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/profiles/$(PROFILE).yaml - ./kubectl --kubeconfig=./kubeconfig rollout restart ds azure-npm -n kube-system - fi + # swap NPM profile with one specified as parameter + ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/profiles/$(PROFILE).yaml + ./kubectl --kubeconfig=./kubeconfig rollout restart ds azure-npm -n kube-system ./kubectl --kubeconfig=./kubeconfig describe daemonset azure-npm -n kube-system @@ -324,46 +256,6 @@ stages: # there can't be a command after e2e.test/policy-assistant because the exit code is important } - runConformanceWindows () { - # full suite of ~32 test cases is taking too long... - # nomatch1="should enforce policy based on PodSelector or NamespaceSelector" - # nomatch2="should enforce policy based on NamespaceSelector with MatchExpressions using default ns label" - # nomatch3="should enforce policy based on PodSelector and NamespaceSelector" - # nomatch4="should enforce policy based on Multiple PodSelectors and NamespaceSelectors" - # cidrExcept1="should ensure an IP overlapping both IPBlock.CIDR and IPBlock.Except is allowed" - # cidrExcept2="should enforce except clause while egress access to server in CIDR block" - # namedPorts="named port" - # wrongK8sVersion="Netpol API" - # toSkip="\[LinuxOnly\]|$nomatch1|$nomatch2|$nomatch3|$nomatch4|$cidrExcept1|$cidrExcept2|$namedPorts|$wrongK8sVersion|SCTP" - - # slimmed down to 14 tests like npm-cni-integration-test.yaml - # NetworkPolicy between server and... - f1="client should enforce policy to allow traffic only from a different namespace, based on NamespaceSelector" - f2="client should deny egress from pods based on PodSelector" - f3="client should enforce multiple, stacked policies with overlapping podSelectors" - f4="client should enforce egress policy allowing traffic to a server in a different namespace based on PodSelector and NamespaceSelector" - f5="client should work with Ingress, Egress specified together" - f6="client should enforce ingress policy allowing any port traffic to a server on a specific protocol" - f7="client should not allow access by TCP when a policy specifies only UDP" - f8="client should allow egress access to server in CIDR block" - f9="client should enforce policy based on Ports" - f10="client should support allow-all policy" - f11="client should enforce updated policy" - f12="client should support denying of egress traffic on the client side" - f13="client should stop enforcing policies after they are deleted" - f14="client should support a 'default-deny-ingress' policy" - focus="$f1|$f2|$f3|$f4|$f5|$f6|$f7|$f8|$f9|$f10|$f11|$f12|$f13|$f14" - KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 \ - ./e2e.test \ - --provider=local \ - --ginkgo.focus="$focus" \ - --ginkgo.skip="\[LinuxOnly\]|NetworkPolicyLegacy|SCTP" \ - --node-os-distro=windows \ - --allowed-not-ready-nodes=1 \ - --kubeconfig=./kubeconfig \ - --ginkgo.timeout="2h" - } - exitCode=0 if [ $(IS_STRESS_TEST) == "true" ]; then echo "Running $NUM_PARALLEL_JOBS_FOR_STRESS_TEST conformance tests at once and writing outputs to files" @@ -372,20 +264,11 @@ stages: # for each iteration, run the conformance test and echos in the background, and write the output of the conformance test to a file # run the conformance test in the foreground and write the output to stdout and a file - if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching - then - echo "starting conformance test #$round" && \ - runConformanceWindows | tee $npmLogsFolder/conformance-results-$round && \ - echo "finished conformance test #$round" & - pidOfConformanceTest=$! - conformancePIDs+=($pidOfConformanceTest) - else - echo "starting windows conformance test #$round" && \ - runConformance > $npmLogsFolder/conformance-results-$round && \ - echo "finished conformance test #$round" & - pidOfConformanceTest=$! - conformancePIDs+=($pidOfConformanceTest) - fi + echo "starting conformance test #$round" && \ + runConformance > $npmLogsFolder/conformance-results-$round && \ + echo "finished conformance test #$round" & + pidOfConformanceTest=$! + conformancePIDs+=($pidOfConformanceTest) done # wait until all conformance tests finish and take note of any failed tests @@ -399,15 +282,9 @@ stages: fi done else - # run the conformance test in the foreground and write the output to stdout and a file - if [[ $(AZURE_CLUSTER) == *ws22 ]] # * is used for pattern matching - then - runConformanceWindows | tee $npmLogsFolder/conformance-results - exitCode=$? - else - runConformance | tee $npmLogsFolder/conformance-results - exitCode=$? - fi + # run the conformance test in the foreground and write the output to stdout and a file + runConformance | tee $npmLogsFolder/conformance-results + exitCode=$? fi # get all current npm pods @@ -461,177 +338,3 @@ stages: inlineScript: | echo Deleting $(RESOURCE_GROUP) az group delete -n $(RESOURCE_GROUP) --yes - - # - stage: cyclonus_win - # displayName: Windows Cyclonus - # dependsOn: [setup, build] - # jobs: - # - job: Create_Windows_Cluster_and_Run_Test - # timeoutInMinutes: 360 - # displayName: "Run Windows Cyclonus" - # pool: - # name: $(BUILD_POOL_NAME_DEFAULT) - # demands: - # - agent.os -equals Linux - # - Role -equals Build - # variables: - # RESOURCE_GROUP: $[ stagedependencies.setup.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ] - # TAG: $[ stagedependencies.setup.setup.outputs['EnvironmentalVariables.TAG'] ] - # FQDN: empty - # strategy: - # matrix: - # v2-windows: - # PROFILE: "cyc-ws22" - # steps: - # - checkout: self - # - task: AzureCLI@2 - # displayName: "Create AKS Cluster" - # inputs: - # azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - # scriptType: "bash" - # scriptLocation: "inlineScript" - # failOnStderr: true - # inlineScript: | - # az extension add --name aks-preview - # az extension update --name aks-preview - - # export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE) - - # echo "Creating resource group named $CLUSTER_NAME" - # az group create --name $CLUSTER_NAME -l $(LOCATION) -o table - - # echo "Creating resource group named $CLUSTER_NAME" - # az aks create \ - # --resource-group $CLUSTER_NAME \ - # --name $CLUSTER_NAME \ - # --generate-ssh-keys \ - # --windows-admin-username e2eadmin \ - # --windows-admin-password alpha@numeric!password2 \ - # --network-plugin azure \ - # --vm-set-type VirtualMachineScaleSets \ - # --node-vm-size Standard_D8s_v3 \ - # --node-count 1 \ - # --uptime-sla - - # # don't schedule anything on the linux system pool - # echo "Updating $CLUSTER_NAME to not schedule anything on linux pool..." - # az aks nodepool update \ - # --cluster-name $CLUSTER_NAME \ - # -g $CLUSTER_NAME \ - # -n nodepool1 \ - # --node-taints CriticalAddonsOnly=true:NoSchedule - - # echo "Adding Windows nodepool to $CLUSTER_NAME" - # az aks nodepool add \ - # --resource-group $CLUSTER_NAME \ - # --cluster-name $CLUSTER_NAME \ - # --name awin22 \ - # --os-type Windows \ - # --os-sku Windows2022 \ - # --node-vm-size Standard_D4s_v3 \ - # --node-count 3 - - # echo "Getting credentials to $CLUSTER_NAME" - # az aks get-credentials -g $CLUSTER_NAME -n $CLUSTER_NAME --overwrite-existing --file ./kubeconfig - # mkdir -p ~/.kube/ - # cp ./kubeconfig ~/.kube/config - - # - task: AzureCLI@2 - # displayName: "Deploy NPM to Test Cluster" - # inputs: - # azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - # scriptType: "bash" - # scriptLocation: "inlineScript" - # failOnStderr: true - # inlineScript: | - # export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE) - - # curl -LO https://dl.k8s.io/release/v1.23.0/bin/linux/amd64/kubectl - # chmod +x kubectl - - # # deploy azure-npm - # ./kubectl --kubeconfig=./kubeconfig apply -f $(Pipeline.Workspace)/s/npm/examples/windows/azure-npm.yaml - - # # swap azure-npm image with one built during run - # ./kubectl --kubeconfig=./kubeconfig set image daemonset/azure-npm-win -n kube-system azure-npm=$IMAGE_REGISTRY/azure-npm:windows-amd64-$(TAG) - - # echo "sleeping and waiting for NPM pods to be ready..." - # sleep 1m - # ./kubectl --kubeconfig=./kubeconfig wait --for=condition=Ready pod -n kube-system -l k8s-app=azure-npm --timeout=5m - # echo "sleeping 3 more minutes to let windows NPM finish bootup phase" - - # ./kubectl --kubeconfig=./kubeconfig get po -n kube-system -owide -A - - # echo "Showing cluster status for $CLUSTER_NAME" - # FQDN=`az aks show -n $CLUSTER_NAME -g $CLUSTER_NAME --query fqdn -o tsv` - # echo "##vso[task.setvariable variable=FQDN]$FQDN" - - # - script: | - # cat ~/.kube/config - # curl -fsSL github.com/mattfenwick/cyclonus/releases/latest/download/cyclonus_linux_amd64.tar.gz | tar -zxv - # name: download_cyclonus - # displayName: "Download Cyclonus" - # failOnStderr: false - # condition: always() - - # - script: | - # ./test/cyclonus/test-cyclonus-windows.sh - # name: cyclonus - # displayName: "Run Cyclonus Test" - # failOnStderr: false - # condition: succeeded() - - # - bash: | - # export CLUSTER_NAME=$(RESOURCE_GROUP)-$(PROFILE) - # cp cyclonus-$CLUSTER_NAME $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/cyclonus-$CLUSTER_NAME - # echo "Getting cluster state for $CLUSTER_NAME" - # mkdir -p $(System.DefaultWorkingDirectory)/$CLUSTER_NAME - # kubectl get pods -n kube-system | grep npm - # kubectl logs -n kube-system -l k8s-app=azure-npm --tail -1 --prefix > $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/npm-logs_$(PROFILE).txt - # # capture any previous logs in case there was a crash - # npmPodList=`kubectl get pods -n kube-system | grep npm | awk '{print $1}'` - # for npmPod in $npmPodList; do - # previousLogFile=$(System.DefaultWorkingDirectory)/$CLUSTER_NAME/previous-npm-logs_$(PROFILE).txt - # kubectl logs -n kube-system $npmPod -p > $previousLogFile - # if [[ $? -ne 0 ]]; then - # # remove the empty file if kubectl logs failed (e.g. there was no previous terminated container) - # rm $previousLogFile - # fi - # done - # cp ./kubeconfig $(System.DefaultWorkingDirectory)/$CLUSTER_NAME/.kubeconfig - # condition: always() - - # - publish: $(System.DefaultWorkingDirectory)/$(RESOURCE_GROUP)-$(PROFILE) - # condition: always() - # artifact: NpmLogs-$(RESOURCE_GROUP)-$(PROFILE) - - # - stage: clean_up_cyclonus_win - # displayName: Cleanup Cyclonus - # dependsOn: [setup, cyclonus_win] - # condition: always() - # jobs: - # - job: clean_up - # displayName: "Cleanup" - # pool: - # name: $(BUILD_POOL_NAME_DEFAULT) - # demands: - # - agent.os -equals Linux - # - Role -equals Build - # variables: - # RESOURCE_GROUP: $[ stagedependencies.setup.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ] - # strategy: - # matrix: - # v2-windows: - # PROFILE: "cyc-ws22" - # steps: - # - checkout: none - # - task: AzureCLI@2 - # displayName: "Delete Test Cluster Resource Group" - # inputs: - # azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - # scriptType: "bash" - # scriptLocation: "inlineScript" - # inlineScript: | - # echo Deleting $(RESOURCE_GROUP)-$(PROFILE) - # az group delete -n $(RESOURCE_GROUP)-$(PROFILE) --yes - diff --git a/npm/azure-npm.yaml b/npm/azure-npm.yaml index a19a1b974e..aa701ab428 100644 --- a/npm/azure-npm.yaml +++ b/npm/azure-npm.yaml @@ -79,7 +79,7 @@ spec: operator: Exists containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.45.3 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 resources: limits: cpu: 250m diff --git a/npm/deploy/kustomize/overlays/controller/deployment.yaml b/npm/deploy/kustomize/overlays/controller/deployment.yaml index d09ac36cc8..4fae5033bb 100644 --- a/npm/deploy/kustomize/overlays/controller/deployment.yaml +++ b/npm/deploy/kustomize/overlays/controller/deployment.yaml @@ -36,7 +36,7 @@ spec: containerPort: 10091 - name: http containerPort: 10092 - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.1 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 command: ["azure-npm"] args: ["controlplane"] resources: diff --git a/npm/deploy/kustomize/overlays/daemon/deployment.yaml b/npm/deploy/kustomize/overlays/daemon/deployment.yaml index 33387f7c8c..978d493ab5 100644 --- a/npm/deploy/kustomize/overlays/daemon/deployment.yaml +++ b/npm/deploy/kustomize/overlays/daemon/deployment.yaml @@ -34,7 +34,7 @@ spec: ports: - name: metrics containerPort: 10091 - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.1 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 command: ["azure-npm"] args: ["daemon"] resources: diff --git a/npm/deploy/manifests/controller/azure-npm.yaml b/npm/deploy/manifests/controller/azure-npm.yaml index bd6df97018..d36c67e2a7 100644 --- a/npm/deploy/manifests/controller/azure-npm.yaml +++ b/npm/deploy/manifests/controller/azure-npm.yaml @@ -149,7 +149,7 @@ spec: fieldPath: spec.nodeName - name: NPM_CONFIG value: /etc/azure-npm/azure-npm.json - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.1 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 name: azure-npm resources: limits: diff --git a/npm/deploy/manifests/daemon/azure-npm.yaml b/npm/deploy/manifests/daemon/azure-npm.yaml index 2abf2c4c1c..0e69605581 100644 --- a/npm/deploy/manifests/daemon/azure-npm.yaml +++ b/npm/deploy/manifests/daemon/azure-npm.yaml @@ -135,7 +135,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.1 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 name: azure-npm ports: - name: metrics diff --git a/npm/deploy/npm/azure-npm.yaml b/npm/deploy/npm/azure-npm.yaml index bf4be6d675..2d71c426af 100644 --- a/npm/deploy/npm/azure-npm.yaml +++ b/npm/deploy/npm/azure-npm.yaml @@ -79,7 +79,7 @@ spec: operator: Exists containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.1 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 resources: limits: cpu: 250m diff --git a/npm/examples/azure-npm-lite.yaml b/npm/examples/azure-npm-lite.yaml index cbd8666536..aa94112d03 100644 --- a/npm/examples/azure-npm-lite.yaml +++ b/npm/examples/azure-npm-lite.yaml @@ -79,7 +79,7 @@ spec: operator: Exists containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.4.45.3 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 resources: limits: cpu: 250m diff --git a/npm/examples/windows/azure-npm-capz.yaml b/npm/examples/windows/azure-npm-capz.yaml index 6d85a290a9..1e859f1bee 100644 --- a/npm/examples/windows/azure-npm-capz.yaml +++ b/npm/examples/windows/azure-npm-capz.yaml @@ -83,7 +83,7 @@ spec: hostNetwork: true containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.5.5 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 command: ["powershell.exe"] args: [ diff --git a/npm/examples/windows/azure-npm-lite-win.yaml b/npm/examples/windows/azure-npm-lite-win.yaml index 12552cb45a..092ae3d1f8 100644 --- a/npm/examples/windows/azure-npm-lite-win.yaml +++ b/npm/examples/windows/azure-npm-lite-win.yaml @@ -83,7 +83,7 @@ spec: hostNetwork: true containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.5.5 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 command: ["powershell.exe"] args: [ diff --git a/npm/examples/windows/azure-npm.yaml b/npm/examples/windows/azure-npm.yaml index db7b4befd3..30675a506f 100644 --- a/npm/examples/windows/azure-npm.yaml +++ b/npm/examples/windows/azure-npm.yaml @@ -83,7 +83,7 @@ spec: hostNetwork: true containers: - name: azure-npm - image: mcr.microsoft.com/containernetworking/azure-npm:v1.5.5 + image: mcr.microsoft.com/containernetworking/azure-npm:v1.6.29 command: ["powershell.exe"] args: [