Add Task to scale down replicas for addons in order to fix the aiml-load Pipeline (#576)

Description / Motivation:
Addons like coredns and ebs-csi-controllers have PDB set due to which Karpenter is not able to scale down the nodepools. Added Task to scale these addons replicas to 0 before scaling down nodepools in aiml-load pipeline.

In order to stop karpenter logs Task after nodepools are scaled down, I am adding another step in the Task which sets the Karpenter replicas to zero. This is done to force stop karpenter logs which keeps running even when job is done and prevents the Teardown step to run.

Desktop Testing: Tested by triggering Tekton test run.

Co-authored-by: Chithresh Azad <[email protected]>

Author: chithreshazad

tests/tekton-resources/pipelines/eks/awscli-eks-aiml-load.yaml

@@ -48,8 +48,6 @@ spec:
   - default: https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_role.json
     name: node-role-cfn-url
     type: string
-  - name: manifest-id
-    type: string
   - default: ""
     name: eksadm-s3-path
     type: string
@@ -194,8 +192,6 @@ spec:
       value: $(params.endpoint)
     - name: vpc-stack-name
       value: $(params.cluster-name)
-    - name: manifest-id
-      value: $(params.manifest-id)
     - name: eksadm-s3-path
       value: $(params.eksadm-s3-path)
     - name: kubernetes-version
@@ -602,6 +598,24 @@ spec:
     taskRef:
       kind: Task
       name: load-aiml-multiple-fine-tuning
+  - name: scale-down-addons
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: coredns-replicas
+      value: 0
+    - name: ebs-csi-replicas
+      value: 0
+    - name: endpoint
+      value: $(params.endpoint)
+    runAfter:
+    - load-aiml-multiple-fine-tuning
+    taskRef:
+      kind: Task
+      name: scale-addons
+    workspaces:
+    - name: config
+      workspace: config
   - name: scale-down-training
     params:
     - name: cluster-name
@@ -613,7 +627,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -643,7 +657,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -673,7 +687,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -703,7 +717,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -733,7 +747,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -752,6 +766,26 @@ spec:
     taskRef:
       kind: Task
       name: nodepool-replicas-wait
+  - name: stop-karpenter-logs
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: endpoint
+      value: $(params.endpoint)
+    - name: karpenter-replicas
+      value: "0"
+    runAfter:
+    - wait-for-scale-down-training
+    - wait-for-scale-down-inference
+    - wait-for-scale-down-operator
+    - wait-for-scale-down-monitoring
+    - wait-for-scale-down-titan-pool
+    taskRef:
+      kind: Task
+      name: scale-addons
+    workspaces:
+    - name: config
+      workspace: config
   finally:
   - name: teardown
     retries: 10 # To deal with throttling during deletion

tests/tekton-resources/tasks/setup/eks/kubectl-addons-scale.yaml

@@ -0,0 +1,74 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: scale-addons
+  namespace: scalability
+spec:
+  description: |
+    Optionally scales CoreDNS, EBS CSI Controller, and Karpenter deployments to the specified number of replicas.
+    This task configures kubectl access to the EKS cluster and scales only the deployments for which replica counts are provided.
+    Waits for the scaling operations to complete.
+  params:
+  - name: coredns-replicas
+    description: Number of replicas to scale CoreDNS to (optional, skipped if empty)
+    default: ""
+  - name: ebs-csi-replicas
+    description: Number of replicas to scale EBS CSI Controller to (optional, skipped if empty)
+    default: ""
+  - name: karpenter-replicas
+    description: Number of replicas to scale Karpenter to (optional, skipped if empty)
+    default: ""
+  - name: cluster-name
+    description: The name of the EKS cluster
+  - name: endpoint
+    description: EKS cluster endpoint URL (optional)
+    default: ""
+  - name: aws-region
+    description: AWS region where the cluster is located
+    default: "us-west-2"
+  - name: timeout
+    description: Timeout for rollout status in seconds
+    default: "1800"
+  workspaces:
+  - name: config
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
+  steps:
+  - name: update-kubeconfig
+    image: alpine/k8s:1.35.0
+    script: |
+      ENDPOINT_FLAG=""
+      if [ -n "$(params.endpoint)" ]; then
+        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
+      fi
+      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.aws-region)
+  - name: scale-coredns
+    image: alpine/k8s:1.35.0
+    script: |
+      if [ -n "$(params.coredns-replicas)" ]; then
+        kubectl scale deployment coredns -n kube-system --replicas=$(params.coredns-replicas)
+        kubectl rollout status deployment/coredns -n kube-system --timeout=$(params.timeout)s
+      else
+        echo "Skipping CoreDNS scaling (coredns-replicas not provided)"
+      fi
+  - name: scale-ebs-csi-controller
+    image: alpine/k8s:1.35.0
+    script: |
+      if [ -n "$(params.ebs-csi-replicas)" ]; then
+        kubectl scale deployment ebs-csi-controller -n kube-system --replicas=$(params.ebs-csi-replicas)
+        kubectl rollout status deployment/ebs-csi-controller -n kube-system --timeout=$(params.timeout)s
+      else
+        echo "Skipping EBS CSI Controller scaling (ebs-csi-replicas not provided)"
+      fi
+  - name: scale-karpenter
+    image: alpine/k8s:1.35.0
+    script: |
+      if [ -n "$(params.karpenter-replicas)" ]; then
+        kubectl scale deployment karpenter -n karpenter --replicas=$(params.karpenter-replicas)
+        echo "Karpenter scaled to $(params.karpenter-replicas) replicas"
+      else
+        echo "Skipping Karpenter scaling (karpenter-replicas not provided)"
+      fi