From bc4fed1a63416f538d196e6db5ce84f9f5d6afd7 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Wed, 9 Oct 2024 17:15:10 -0700 Subject: [PATCH 1/5] add windows node template first pass --- ...ni-overlay-stateless-e2e-job-template.yaml | 30 ++---------- .../azure-cni-overlay-e2e-job-template.yaml | 33 ++----------- .../dualstackoverlay-e2e-job-template.yaml | 32 ++----------- .../templates/add-windows-nodepool-job.yaml | 46 +++++++++++++++++++ 4 files changed, 61 insertions(+), 80 deletions(-) create mode 100644 .pipelines/templates/add-windows-nodepool-job.yaml diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml index c7b536e88e..c21c1298d3 100644 --- a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml @@ -28,31 +28,11 @@ stages: k8sVersion: ${{ parameters.k8sVersion }} dependsOn: ${{ parameters.dependsOn }} region: $(REGION_AKS_CLUSTER_TEST) - - job: windows_nodepool - displayName: Add Windows Nodepool - dependsOn: ${{ parameters.name }} - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - steps: - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} - echo "Windows node are successfully added to v4 Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node on v4 overlay cluster" + - template: ../../templates/add-windows-nodepool-job.yaml + parameters: + depend: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml index 2bc03a4726..0c9455c3a8 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml @@ -58,34 +58,11 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux scaleup: 100 - - - job: windows_nodepool - displayName: Add Windows Nodepool - dependsOn: ${{ parameters.name }}_linux - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - steps: - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} - echo "Windows node are successfully added to v4 Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node on v4 overlay cluster" - - + - template: ../../templates/add-windows-nodepool-job.yaml + parameters: + depend: ${{ parameters.name }}_linux + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} - job: ${{ parameters.name }}_windows displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }}) timeoutInMinutes: 120 diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml index 6780a82709..97a7fcee66 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-job-template.yaml @@ -59,33 +59,11 @@ stages: clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux scaleup: 100 - - - job: windows_nodepool - displayName: Add Windows Nodepool - dependsOn: ${{ parameters.name }}_linux - pool: - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - steps: - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} - echo "Windows nodes have been successfully added to DualStack Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node" - + - template: ../../templates/add-windows-nodepool-job.yaml + parameters: + depend: ${{ parameters.name }}_linux + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: "Standard_D2_v3" - job: ${{ parameters.name }}_windows displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }}) timeoutInMinutes: 120 diff --git a/.pipelines/templates/add-windows-nodepool-job.yaml b/.pipelines/templates/add-windows-nodepool-job.yaml new file mode 100644 index 0000000000..4ad1d81743 --- /dev/null +++ b/.pipelines/templates/add-windows-nodepool-job.yaml @@ -0,0 +1,46 @@ +parameters: + depend: "" + clusterName: "" # unique identifier + vmSize: "" + retryCount: 5 + +jobs: +- job: windows_nodepool + displayName: Add Windows Nodepool + dependsOn: ${{ parameters.depend }} + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - task: AzureCLI@2 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + + windows_nodepool=$(az aks nodepool list \ + --resource-group ${{ parameters.clusterName }} \ + --cluster-name ${{ parameters.clusterName }} \ + --query "[?osType=='Windows']" \ + --output tsv) + + if [ -z "$windows_nodepool" ]; then + echo "No Windows node pool found in the AKS cluster." + + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSize }} + echo "Windows node was successfully added to v4 Overlay Cluster" + kubectl cluster-info + kubectl get node -owide + kubectl get po -owide -A + else + echo "Windows node pool already exists in the AKS cluster." + fi + name: "Add_Windows_Node" + displayName: "Add windows node on v4 overlay cluster" + retryCountOnTaskFailure: ${{ parameters.retryCount }} From 9f2b4f159b15cd5e0b6955110ad0cc4dd3ce4c03 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 10 Oct 2024 10:13:44 -0700 Subject: [PATCH 2/5] wait for cluster to exit updating state before adding windows nodepool --- ...cni-overlay-stateless-e2e-job-template.yaml | 1 - .../templates/add-windows-nodepool-job.yaml | 18 +++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml index c21c1298d3..c084cffd9e 100644 --- a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml @@ -11,7 +11,6 @@ stages: - stage: ${{ parameters.clusterName }} displayName: Create Cluster - ${{ parameters.displayName }} dependsOn: - - ${{ parameters.dependsOn }} - setup pool: name: $(BUILD_POOL_NAME_DEFAULT) diff --git a/.pipelines/templates/add-windows-nodepool-job.yaml b/.pipelines/templates/add-windows-nodepool-job.yaml index 4ad1d81743..bae2a349fd 100644 --- a/.pipelines/templates/add-windows-nodepool-job.yaml +++ b/.pipelines/templates/add-windows-nodepool-job.yaml @@ -13,6 +13,7 @@ jobs: demands: - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) + timeoutInMinutes: 30 steps: - task: AzureCLI@2 inputs: @@ -31,7 +32,22 @@ jobs: if [ -z "$windows_nodepool" ]; then echo "No Windows node pool found in the AKS cluster." - + + # wait for cluster to update + while true; do + cluster_state=$(az aks show \ + --name "${{ parameters.clusterName }}" \ + --resource-group "${{ parameters.clusterName }}" \ + --query provisioningState) + + if echo "$cluster_state" | grep -q "Updating"; then + echo "Cluster is updating. Sleeping for 30 seconds..." + sleep 30 + else + break + fi + done + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSize }} echo "Windows node was successfully added to v4 Overlay Cluster" From 3cd43c0fdb70e8610663ceaa528819c1de58e2f3 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 10 Oct 2024 11:08:03 -0700 Subject: [PATCH 3/5] remove change made for testing --- .../azure-cni-overlay-stateless-e2e-job-template.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml index c084cffd9e..c21c1298d3 100644 --- a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e-job-template.yaml @@ -11,6 +11,7 @@ stages: - stage: ${{ parameters.clusterName }} displayName: Create Cluster - ${{ parameters.displayName }} dependsOn: + - ${{ parameters.dependsOn }} - setup pool: name: $(BUILD_POOL_NAME_DEFAULT) From c594bef23fd475c4735ef85b2445d2e9a8f5f681 Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 10 Oct 2024 11:34:50 -0700 Subject: [PATCH 4/5] fix naming --- .pipelines/templates/add-windows-nodepool-job.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/add-windows-nodepool-job.yaml b/.pipelines/templates/add-windows-nodepool-job.yaml index bae2a349fd..01b2bfeb2e 100644 --- a/.pipelines/templates/add-windows-nodepool-job.yaml +++ b/.pipelines/templates/add-windows-nodepool-job.yaml @@ -50,7 +50,7 @@ jobs: make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSize }} - echo "Windows node was successfully added to v4 Overlay Cluster" + echo "Windows node was successfully added" kubectl cluster-info kubectl get node -owide kubectl get po -owide -A @@ -58,5 +58,5 @@ jobs: echo "Windows node pool already exists in the AKS cluster." fi name: "Add_Windows_Node" - displayName: "Add windows node on v4 overlay cluster" + displayName: "Add windows node to cluster" retryCountOnTaskFailure: ${{ parameters.retryCount }} From 449520e5aba7804f965ae332a6b503a4194735ad Mon Sep 17 00:00:00 2001 From: QxBytes Date: Thu, 10 Oct 2024 13:27:20 -0700 Subject: [PATCH 5/5] address feedback --- .pipelines/templates/add-windows-nodepool-job.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.pipelines/templates/add-windows-nodepool-job.yaml b/.pipelines/templates/add-windows-nodepool-job.yaml index 01b2bfeb2e..fe700371b4 100644 --- a/.pipelines/templates/add-windows-nodepool-job.yaml +++ b/.pipelines/templates/add-windows-nodepool-job.yaml @@ -2,7 +2,6 @@ parameters: depend: "" clusterName: "" # unique identifier vmSize: "" - retryCount: 5 jobs: - job: windows_nodepool @@ -47,6 +46,8 @@ jobs: break fi done + # cluster state is always set and visible outside the loop + echo "Cluster state is: $cluster_state" make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSize }} @@ -59,4 +60,4 @@ jobs: fi name: "Add_Windows_Node" displayName: "Add windows node to cluster" - retryCountOnTaskFailure: ${{ parameters.retryCount }} + retryCountOnTaskFailure: 5