Add Task to scale down replicas for addons and karpenter in order to fix the aiml-load Pipeline

Description / Motivation:
Addons like coredns and ebs-csi-controllers have PDB set due to which Karpenter is not able to scale down the nodepools. Added Task to scale these addons replicas to 0 before scaling down nodepools in aiml-load pipeline.

In order to stop karpenter logs Task after nodepools are scaled down, I am adding another Task which sets the Karpenter replicas to zero. This is done to force stop karpenter logs which keeps running even when job is done and prevents the Teardown step to run.

Desktop Testing: Tested by triggering Tekton test run.

Author: chithresha

tests/tekton-resources/pipelines/eks/awscli-eks-aiml-load.yaml

@@ -281,6 +281,7 @@ spec:
       kind: Task
       name: helm-karpenter-install
   - name: get-karp-logs
+    timeout: "4h"
     params:
     - name: cluster-name
       value: $(params.cluster-name)
@@ -602,6 +603,24 @@ spec:
     taskRef:
       kind: Task
       name: load-aiml-multiple-fine-tuning
+  - name: scale-down-addons
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: coredns-replicas
+      value: 0
+    - name: ebs-csi-replicas
+      value: 0
+    - name: endpoint
+      value: $(params.endpoint)
+    runAfter:
+    - load-aiml-multiple-fine-tuning
+    taskRef:
+      kind: Task
+      name: scale-addons
+    workspaces:
+    - name: config
+      workspace: config
   - name: scale-down-training
     params:
     - name: cluster-name
@@ -613,7 +632,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -643,7 +662,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -673,7 +692,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -703,7 +722,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -733,7 +752,7 @@ spec:
     - name: replicas
       value: 0
     runAfter:
-    - load-aiml-multiple-fine-tuning
+    - scale-down-addons
     taskRef:
       kind: Task
       name: scale-nodepool
@@ -752,6 +771,23 @@ spec:
     taskRef:
       kind: Task
       name: nodepool-replicas-wait
+  - name: stop-karpenter-logs
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: endpoint
+      value: $(params.endpoint)
+    - name: replicas
+      value: 0
+    runAfter:
+    - wait-for-scale-down-training
+    - wait-for-scale-down-inference
+    - wait-for-scale-down-operator
+    - wait-for-scale-down-monitoring
+    - wait-for-scale-down-titan-pool
+    taskRef:
+      kind: Task
+      name: scale-karpenter
   finally:
   - name: teardown
     retries: 10 # To deal with throttling during deletion

tests/tekton-resources/tasks/setup/eks/kubectl-addons-scale.yaml

@@ -0,0 +1,52 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: scale-addons
+  namespace: scalability
+spec:
+  description: |
+    Scales the CoreDNS and EBS CSI Controller deployments to the specified number of replicas.
+    This task configures kubectl access to the EKS cluster, scales the deployments,
+    and waits for the scaling operations to complete.
+  params:
+  - name: coredns-replicas
+    description: Number of replicas to scale CoreDNS to (target replica count)
+  - name: ebs-csi-replicas
+    description: Number of replicas to scale EBS CSI Controller to (target replica count)
+  - name: cluster-name
+    description: The name of the EKS cluster
+  - name: endpoint
+    description: EKS cluster endpoint URL (optional)
+    default: ""
+  - name: aws-region
+    description: AWS region where the cluster is located
+    default: "us-west-2"
+  - name: timeout
+    description: Timeout for rollout status in seconds
+    default: "1800"
+  workspaces:
+  - name: config
+    mountPath: /config/
+  stepTemplate:
+    env:
+    - name: KUBECONFIG
+      value: /config/kubeconfig
+  steps:
+  - name: update-kubeconfig
+    image: alpine/k8s:1.35.0
+    script: |
+      ENDPOINT_FLAG=""
+      if [ -n "$(params.endpoint)" ]; then
+        ENDPOINT_FLAG="--endpoint $(params.endpoint)"
+      fi
+      aws eks $ENDPOINT_FLAG update-kubeconfig --name $(params.cluster-name) --region $(params.aws-region)
+  - name: scale-coredns
+    image: alpine/k8s:1.35.0
+    script: |
+      kubectl scale deployment coredns -n kube-system --replicas=$(params.coredns-replicas)
+      kubectl rollout status deployment/coredns -n kube-system --timeout=$(params.timeout)s
+  - name: scale-ebs-csi-controller
+    image: alpine/k8s:1.35.0
+    script: |
+      kubectl scale deployment ebs-csi-controller -n kube-system --replicas=$(params.ebs-csi-replicas)
+      kubectl rollout status deployment/ebs-csi-controller -n kube-system --timeout=$(params.timeout)s

tests/tekton-resources/tasks/teardown/karpenter/kubectl-karpenter-scale.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: scale-karpenter
+  namespace: scalability
+spec:
+  description: "Scale Karpenter deployment"
+  params:
+  - name: cluster-name
+    description: The name of the cluster
+  - name: endpoint
+    description: eks endpoint to use
+  - name: aws-region
+    description: AWS region where the cluster is located
+    default: us-west-2
+  - name: namespace
+    description: Namespace where karpenter is installed
+    default: karpenter
+  - name: replicas
+    description: Number of replicas to scale to
+  steps:
+  - name: scale-karpenter
+    image: alpine/k8s:1.35.0
+    script: |
+      aws eks update-kubeconfig --name $(params.cluster-name) --endpoint $(params.endpoint) --region $(params.aws-region)
+      kubectl scale deployment karpenter -n $(params.namespace) --replicas=$(params.replicas)
+      echo "Karpenter scaled to $(params.replicas) replicas"