diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 7e61710ca2..9603020072 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -283,14 +283,27 @@ stages: # Azure Overlay E2E tests - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml parameters: - name: "azure_overlay_e2e" - displayName: Azure Overlay + name: "linux_azure_overlay_e2e" + displayName: Azure Overlay Linux os: linux clusterType: overlay-byocni-up - clusterName: "azovere2e" + clusterName: "linuxazovere2e" vmSize: Standard_B2ms k8sVersion: "" dependsOn: "containerize" + scaleup: 100 + + - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml + parameters: + name: "win_azure_overlay_e2e" + displayName: Azure Overlay Windows + os: windows + clusterType: overlay-byocni-up + clusterName: "winazovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" + scaleup: 50 # Azure Overlay E2E Stateless CNI tests - template: singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml @@ -357,20 +370,33 @@ stages: # CNI dual stack overlay E2E tests - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml parameters: - name: "dualstackoverlay_e2e" - displayName: AKS DualStack Overlay + name: "linux_dualstackoverlay_e2e" + displayName: AKS DualStack Overlay Linux os: linux clusterType: dualstack-overlay-byocni-up - clusterName: "dsovere2e" + clusterName: "linuxdsovere2e" + vmSize: Standard_B2ms + dependsOn: "containerize" + scaleup: 100 + + - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml + parameters: + name: "win_dualstackoverlay_e2e" + displayName: AKS DualStack Overlay Windows + os: windows + clusterType: dualstack-overlay-byocni-up + clusterName: "windsovere2e" vmSize: Standard_B2ms dependsOn: "containerize" + scaleup: 50 - stage: delete displayName: Delete Clusters condition: always() dependsOn: - setup - - azure_overlay_e2e + - linux_azure_overlay_e2e + - win_azure_overlay_e2e - azure_overlay_stateless_e2e - aks_swift_e2e - cilium_e2e @@ -380,7 +406,8 @@ stages: - aks_ubuntu_22_linux_e2e - aks_swift_vnetscale_e2e - aks_windows_22_e2e - - dualstackoverlay_e2e + - linux_dualstackoverlay_e2e + - win_dualstackoverlay_e2e - cilium_dualstackoverlay_e2e variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] @@ -407,9 +434,13 @@ stages: name: cilium_h_overlay_e2e clusterName: "cilwhleovere2e" region: $(REGION_AKS_CLUSTER_TEST) - azure_overlay_e2e: - name: azure_overlay_e2e - clusterName: "azovere2e" + linux_azure_overlay_e2e: + name: linux_azure_overlay_e2e + clusterName: "linuxazovere2e" + region: $(REGION_AKS_CLUSTER_TEST) + windows_azure_overlay_e2e: + name: win_azure_overlay_e2e + clusterName: "winazovere2e" region: $(REGION_AKS_CLUSTER_TEST) azure_overlay_stateless_e2e: name: azure_overlay_stateless_e2e @@ -431,9 +462,13 @@ stages: name: aks_windows_22_e2e clusterName: "win22e2e" region: $(REGION_AKS_CLUSTER_TEST) - dualstackoverlay_e2e: - name: dualstackoverlay_e2e - clusterName: "dsovere2e" + linux_dualstackoverlay_e2e: + name: linux_dualstackoverlay_e2e + clusterName: "linuxdsovere2e" + region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) + windows_dualstackoverlay_e2e: + name: windows_dualstackoverlay_e2e + clusterName: "windsovere2e" region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) cilium_dualstackoverlay_e2e: name: cilium_dualstackoverlay_e2e diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml index 0c9455c3a8..62b1d0a6fa 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -6,6 +6,8 @@ parameters: vmSize: "" k8sVersion: "" dependsOn: "" + scaleup: "" + os: "" stages: - stage: ${{ parameters.clusterName }} @@ -25,8 +27,10 @@ stages: clusterType: ${{ parameters.clusterType }} clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} + vmSizeWin: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} dependsOn: ${{ parameters.dependsOn }} + os: ${{ parameters.os }} region: $(REGION_AKS_CLUSTER_TEST) - stage: ${{ parameters.name }} @@ -43,8 +47,8 @@ stages: pool: name: $(BUILD_POOL_NAME_DEFAULT) jobs: - - job: ${{ parameters.name }}_linux - displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }}) + - job: ${{ parameters.name }}_${{ parameters.os }} + displayName: Azure CNI Overlay Test Suite | ${{ parameters.os }} - (${{ parameters.name }}) timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) @@ -56,78 +60,31 @@ stages: parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) - os: linux - scaleup: 100 - - template: ../../templates/add-windows-nodepool-job.yaml - parameters: - depend: ${{ parameters.name }}_linux - clusterName: ${{ parameters.clusterName }}-$(commitID) - vmSize: ${{ parameters.vmSize }} - - job: ${{ parameters.name }}_windows - displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }}) - timeoutInMinutes: 120 - dependsOn: windows_nodepool - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - steps: - - template: azure-cni-overlay-e2e-step-template.yaml - parameters: - name: ${{ parameters.name }} - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: windows - scaleup: 50 + os: ${{ parameters.os }} + scaleup: ${{ parameters.scaleup }} # 50 in windows or 100 in linux - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} - dependsOn: ${{ parameters.name }}_windows + dependsOn: ${{ parameters.name }}_${{ parameters.os }} datapath: true dns: true portforward: true hostport: true service: true + hybridWin: ${{ eq(parameters.os, 'windows') }} - - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml - parameters: - sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: windows - dependsOn: cni_${{ parameters.os }} - datapath: true - dns: true - portforward: true - hostport: true - service: true - hybridWin: true - - - job: failedE2ELogs_linux - displayName: "Linux Failure Logs" + - job: failedE2ELogs_${{ parameters.os }} + displayName: "${{ parameters.os }} Failure Logs" dependsOn: - - ${{ parameters.name }}_linux - - cni_linux - condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed') + - ${{ parameters.name }}_${{ parameters.os }} + - CNI_${{ parameters.os }} + condition: in(dependencies.${{ parameters.name }}_${{ parameters.os }}.result, 'Failed') steps: - template: ../../templates/log-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) - os: linux + os: ${{ parameters.os }} cni: cniv2 - - - job: failedE2ELogs_windows - displayName: "Windows Failure Logs" - dependsOn: - - ${{ parameters.name }}_windows - - cni_windows - condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed') - steps: - - template: ../../templates/log-template.yaml - parameters: - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: windows - cni: cniv2 - diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml index 97a7fcee66..5302998d16 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml @@ -6,6 +6,8 @@ parameters: vmSize: "" k8sVersion: "" dependsOn: "" + os: "" + scaleup: "" stages: - stage: ${{ parameters.clusterName }} @@ -25,12 +27,13 @@ stages: clusterType: ${{ parameters.clusterType }} clusterName: ${{ parameters.clusterName }}-$(commitID) vmSize: ${{ parameters.vmSize }} + vmSizeWin: ${{ parameters.vmSize }} k8sVersion: ${{ parameters.k8sVersion }} dependsOn: ${{ parameters.dependsOn }} + os: ${{ parameters.os }} region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement - stage: ${{ parameters.name }} - condition: and( succeeded(), not(eq(dependencies.dualstackoverlaye2e.result,'SucceededWithIssues')) ) # Cant use parameters in dependencies displayName: E2E - ${{ parameters.displayName }} dependsOn: - setup @@ -44,8 +47,8 @@ stages: pool: name: $(BUILD_POOL_NAME_DEFAULT) jobs: - - job: ${{ parameters.name }}_linux - displayName: DualStack Overlay Test Suite | Linux - (${{ parameters.name }}) + - job: ${{ parameters.name }}_${{ parameters.os }} + displayName: DualStack Overlay Test Suite | ${{ parameters.os }} - (${{ parameters.name }}) timeoutInMinutes: 120 pool: name: $(BUILD_POOL_NAME_DEFAULT) @@ -57,77 +60,31 @@ stages: parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) - os: linux - scaleup: 100 - - template: ../../templates/add-windows-nodepool-job.yaml - parameters: - depend: ${{ parameters.name }}_linux - clusterName: ${{ parameters.clusterName }}-$(commitID) - vmSize: "Standard_D2_v3" - - job: ${{ parameters.name }}_windows - displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }}) - timeoutInMinutes: 120 - dependsOn: windows_nodepool - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - steps: - - template: dualstackoverlay-e2e-step-template.yaml - parameters: - name: ${{ parameters.name }} - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: windows - scaleup: 50 - - - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml - parameters: - sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: linux - dependsOn: ${{ parameters.name }}_windows - dualstack: true - dns: true - portforward: true - hostport: true - service: true + os: ${{ parameters.os }} + scaleup: ${{ parameters.scaleup }} # 50 in windows or 100 in linux - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) - os: windows - dependsOn: cni_linux - # dualstack: true # Currently broken for scenario and blocking releases, HNS is investigating. Covered by go test in E2E step template + os: ${{ parameters.os }} + dependsOn: ${{ parameters.name }}_${{ parameters.os }} + dualstack: ${{ eq(parameters.os, 'linux') }} # RUN IN LINUX not WINDOWS Currently broken for scenario and blocking releases, HNS is investigating. Covered by go test in E2E step template dns: true portforward: true - # service: true # Currently broken for scenario and blocking releases, HNS is investigating. + service: ${{ eq(parameters.os, 'linux') }} # RUN IN LINUX NOT WINDOWS Currently broken for scenario and blocking releases, HNS is investigating. hostport: true - hybridWin: true - - - job: failedE2ELogs_linux - displayName: "Linux Failure Logs" - dependsOn: - - ${{ parameters.name }}_linux - - cni_linux - condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed') - steps: - - template: ../../templates/log-template.yaml - parameters: - clusterName: ${{ parameters.clusterName }}-$(commitID) - os: linux - cni: cniv2 + hybridWin: ${{ eq(parameters.os, 'windows') }} - - job: failedE2ELogs_windows - displayName: "Windows Failure Logs" + - job: failedE2ELogs_${{ parameters.os }} + displayName: "${{ parameters.os }} Failure Logs" dependsOn: - - ${{ parameters.name }}_windows - - cni_windows - condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed') + - ${{ parameters.name }}_${{ parameters.os }} + - CNI_${{ parameters.os }} + condition: in(dependencies.${{ parameters.name }}_${{ parameters.os }}.result, 'Failed') steps: - template: ../../templates/log-template.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) - os: windows + os: ${{ parameters.os }} cni: cniv2 diff --git a/Makefile b/Makefile index b564ba6c6d..4911f52476 100644 --- a/Makefile +++ b/Makefile @@ -772,7 +772,7 @@ test-load: ## run all load tests AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \ CNI_VERSION=$(CNI_VERSION) CNS_VERSION=$(CNS_VERSION) \ - go test -timeout 30m -race -tags=load ./test/integration/load... -v + go test -timeout 40m -race -tags=load ./test/integration/load... -v test-validate-state: cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load --skip 'TestE2E*' -run ^TestValidateState diff --git a/hack/aks/Makefile b/hack/aks/Makefile index 2df03770b2..055bbc9e00 100644 --- a/hack/aks/Makefile +++ b/hack/aks/Makefile @@ -114,7 +114,7 @@ nodesubnet-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an NodeSubne --yes @$(MAKE) set-kubeconf -overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster +overlay-byocni-up: rg-up overlay-net-up ## Brings up a Linux Overlay BYO CNI cluster $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ --auto-upgrade-channel $(AUTOUPGRADE) \ --node-os-upgrade-channel $(NODEUPGRADE) \ @@ -129,10 +129,11 @@ overlay-byocni-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster --no-ssh-key \ --os-sku $(OS_SKU) \ --yes + @$(MAKE) set-kubeconf ifeq ($(OS),windows) - @$(MAKE) windows-nodepool-up + AZCLI="az" GROUP="$(GROUP)" CLUSTER="$(CLUSTER)" sh ../scripts/wait-cluster-update.sh + $(MAKE) windows-nodepool-up endif - @$(MAKE) set-kubeconf overlay-byocni-nokubeproxy-up: rg-up overlay-net-up ## Brings up an Overlay BYO CNI cluster without kube-proxy $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ @@ -404,6 +405,10 @@ dualstack-overlay-byocni-up: rg-up overlay-net-up ## Brings up an dualstack Over --no-ssh-key \ --yes @$(MAKE) set-kubeconf +ifeq ($(OS),windows) + AZCLI="az" GROUP="$(GROUP)" CLUSTER="$(CLUSTER)" sh ../scripts/wait-cluster-update.sh + $(MAKE) windows-nodepool-up +endif cilium-dualstack-up: rg-up overlay-net-up ## Brings up a Cilium Dualstack Overlay cluster with Linux node only $(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \ diff --git a/hack/scripts/wait-cluster-update.sh b/hack/scripts/wait-cluster-update.sh new file mode 100644 index 0000000000..51b9cfd28d --- /dev/null +++ b/hack/scripts/wait-cluster-update.sh @@ -0,0 +1,16 @@ +# wait for cluster to update +while true; do + cluster_state=$($AZCLI aks show \ + --name "$CLUSTER" \ + --resource-group "$GROUP" \ + --query provisioningState) + + if echo "$cluster_state" | grep -q "Updating"; then + echo "Cluster is updating. Sleeping for 30 seconds..." + sleep 30 + else + break + fi +done +# cluster state is always set and visible outside the loop +echo "Cluster state is: $cluster_state"