PMM-14346: PMM HA EKS testing pipeline with ALB, Route53, and Access Entries

pmm/v3/pmm3-ha-eks-cleanup.groovy

@@ -1,33 +1,60 @@
+/**
+ * PMM HA EKS Cleanup Pipeline
+ *
+ * Manages cleanup of PMM HA test clusters. Supports manual and scheduled runs.
+ * Deletes Route53 records, ALB ingress, and EKS clusters.
+ *
+ * Actions:
+ *   - LIST_ONLY: List all test clusters with age
+ *   - DELETE_CLUSTER: Delete a specific cluster
+ *   - DELETE_ALL: Delete all test clusters (respects SKIP_NEWEST)
+ *   - DELETE_OLD (cron): Delete clusters older than 24 hours
+ *   - CLEANUP_ORPHANS: Delete orphaned VPCs and failed CF stacks
+ *
+ * Related:
+ *   - Create: pmm3-ha-eks.groovy
+ *   - Shared library: vars/pmmHaEks.groovy
+ */
+library changelog: false, identifier: 'lib@fix/pmm-ha-eks-access-entries', retriever: modernSCM([
+    $class: 'GitSCMSource',
+    remote: 'https://github.com/Percona-Lab/jenkins-pipelines'
+])
+
 pipeline {
     agent {
-        label 'agent-amd64-ol9'
+        label 'cli'
     }
 
     triggers {
-        cron('H 0,12 * * *') // Runs twice daily at 00:00 & 12:00
+        cron('H 0,12 * * *')
     }
 
     parameters {
         choice(
             name: 'ACTION',
-            choices: ['LIST_ONLY', 'DELETE_CLUSTER', 'DELETE_ALL'],
+            choices: ['LIST_ONLY', 'DELETE_CLUSTER', 'DELETE_ALL', 'CLEANUP_ORPHANS'],
             description: '''
                 LIST_ONLY - list all test clusters<br/>
                 DELETE_CLUSTER - delete a specific cluster (requires CLUSTER_NAME)<br/>
-                DELETE_ALL - delete all test clusters<br/><br/>
+                DELETE_ALL - delete all test clusters<br/>
+                CLEANUP_ORPHANS - delete orphaned VPCs and failed CF stacks<br/><br/>
                 Note: Daily cron automatically deletes clusters older than 1 day.
             '''
         )
         string(name: 'CLUSTER_NAME', defaultValue: '', description: 'Required only for DELETE_CLUSTER')
+        booleanParam(name: 'SKIP_NEWEST', defaultValue: true, description: 'Skip the most recent cluster (protects in-progress builds)')
     }
 
     options {
         buildDiscarder(logRotator(numToKeepStr: '30'))
+        disableConcurrentBuilds()
+        timeout(time: 60, unit: 'MINUTES')
     }
 
     environment {
-        REGION = "us-east-2"
-        CLUSTER_PREFIX = "pmm-ha-test-"
+        REGION = 'us-east-2'
+        CLUSTER_PREFIX = 'pmm-ha-test-'
+        R53_ZONE_NAME = 'cd.percona.com'
     }
 
     stages {
@@ -36,14 +63,14 @@ pipeline {
                 script {
                     if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause')) {
                         env.ACTION = 'DELETE_OLD'
-                        echo "Triggered by cron - will delete clusters older than 1 day."
+                        echo 'Triggered by cron - will delete clusters older than 1 day.'
                     } else {
                         env.ACTION = params.ACTION
                         echo "Manual run with ACTION=${params.ACTION}"
                     }
 
                     if (env.ACTION == 'DELETE_CLUSTER' && !params.CLUSTER_NAME) {
-                        error("CLUSTER_NAME is required for DELETE_CLUSTER.")
+                        error('CLUSTER_NAME is required for DELETE_CLUSTER.')
                     }
                     if (params.CLUSTER_NAME && !params.CLUSTER_NAME.startsWith(env.CLUSTER_PREFIX)) {
                         error("Cluster name must start with ${env.CLUSTER_PREFIX}")
@@ -56,29 +83,30 @@ pipeline {
             when { expression { env.ACTION == 'LIST_ONLY' } }
             steps {
                 withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
-                    sh '''
-                        set +x
-
-                        CLUSTERS=$(aws eks list-clusters --region "$REGION" \
-                            --query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" \
-                            --output text)
-
-                        if [ -z "$CLUSTERS" ]; then
-                            echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
-                            exit 0
-                        fi
-
-                        for c in $CLUSTERS; do
-                            CREATED=$(aws eks describe-cluster \
-                                --name "$c" --region "$REGION" \
-                                --query "cluster.createdAt" --output text)
-
-                            CREATED_EPOCH=$(date -d "$CREATED" +%s)
-                            AGE_HOURS=$(( ( $(date +%s) - CREATED_EPOCH ) / 3600 ))
-
-                            echo "• $c | Created: $CREATED | Age: ${AGE_HOURS}h"
-                        done
-                    '''
+                    script {
+                        def clusters = pmmHaEks.listClusters(env.REGION)
+
+                        if (!clusters) {
+                            echo "No clusters found with prefix '${env.CLUSTER_PREFIX}'."
+                            return
+                        }
+
+                        echo "Found ${clusters.size()} cluster(s):"
+                        clusters.each { clusterName ->
+                            def info = sh(
+                                script: """
+                                    CREATED=\$(aws eks describe-cluster --name ${clusterName} --region ${env.REGION} \
+                                        --query 'cluster.createdAt' --output text)
+                                    CREATED_EPOCH=\$(date -d "\${CREATED}" +%s)
+                                    AGE_HOURS=\$(( ( \$(date +%s) - CREATED_EPOCH ) / 3600 ))
+                                    echo "\${CREATED}|\${AGE_HOURS}"
+                                """,
+                                returnStdout: true
+                            ).trim()
+                            def parts = info.split('\\|')
+                            echo "* ${clusterName} | Created: ${parts[0]} | Age: ${parts[1]}h"
+                        }
+                    }
                 }
             }
         }
@@ -87,15 +115,22 @@ pipeline {
             when { expression { env.ACTION == 'DELETE_CLUSTER' } }
             steps {
                 withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
-                    sh '''
-                        if ! aws eks describe-cluster --region "${REGION}" --name "${CLUSTER_NAME}" >/dev/null 2>&1; then
-                            echo "Cluster '${CLUSTER_NAME}' not found in region '${REGION}'."
-                            exit 0
-                        fi
-
-                        eksctl delete cluster --region "${REGION}" --name "${CLUSTER_NAME}" \
-                            --disable-nodegroup-eviction --wait
-                    '''
+                    script {
+                        def clusterExists = sh(
+                            script: "aws eks describe-cluster --region ${REGION} --name ${params.CLUSTER_NAME} >/dev/null 2>&1",
+                            returnStatus: true
+                        ) == 0
+
+                        if (clusterExists) {
+                            pmmHaEks.deleteCluster(
+                                clusterName: params.CLUSTER_NAME,
+                                region: env.REGION,
+                                r53ZoneName: env.R53_ZONE_NAME
+                            )
+                        } else {
+                            echo "Cluster '${params.CLUSTER_NAME}' not found in region '${REGION}'."
+                        }
+                    }
                 }
             }
         }
@@ -104,20 +139,14 @@ pipeline {
             when { expression { env.ACTION == 'DELETE_ALL' } }
             steps {
                 withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
-                    sh '''
-                        CLUSTERS=$(aws eks list-clusters --region "$REGION" \
-                            --query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" --output text)
-
-                        if [ -z "$CLUSTERS" ]; then
-                            echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
-                            exit 0
-                        fi
-
-                        for c in $CLUSTERS; do
-                            eksctl delete cluster --region "$REGION" --name "$c" \
-                                --disable-nodegroup-eviction --wait
-                        done
-                    '''
+                    script {
+                        pmmHaEks.deleteAllClusters(
+                            region: env.REGION,
+                            r53ZoneName: env.R53_ZONE_NAME,
+                            skipNewest: params.SKIP_NEWEST,
+                            maxAgeHours: 0  // Delete all regardless of age
+                        )
+                    }
                 }
             }
         }
@@ -126,36 +155,25 @@ pipeline {
             when { expression { env.ACTION == 'DELETE_OLD' } }
             steps {
                 withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
-                    sh '''
-                        CLUSTERS=$(aws eks list-clusters --region "$REGION" \
-                            --query "clusters[?starts_with(@, '${CLUSTER_PREFIX}')]" --output text)
-
-                        if [ -z "$CLUSTERS" ]; then
-                            echo "No clusters found with prefix '${CLUSTER_PREFIX}'."
-                            exit 0
-                        fi
-
-                        CUTOFF=$(date -d "1 day ago" +%s)
-
-                        for c in $CLUSTERS; do
-                            CREATED=$(aws eks describe-cluster --name "$c" --region "$REGION" \
-                                --query "cluster.createdAt" --output text 2>/dev/null || true)
-
-                            if [ -z "$CREATED" ] || [ "$CREATED" == "None" ]; then
-                                echo "Unable to fetch creation time for $c — skipping."
-                                continue
-                            fi
-
-                            CREATED_EPOCH=$(date -d "$CREATED" +%s)
+                    script {
+                        pmmHaEks.deleteAllClusters(
+                            region: env.REGION,
+                            r53ZoneName: env.R53_ZONE_NAME,
+                            skipNewest: true,  // Always protect newest during cron
+                            maxAgeHours: 24    // Only delete clusters older than 24h
+                        )
+                    }
+                }
+            }
+        }
 
-                            if [ "$CREATED_EPOCH" -lt "$CUTOFF" ]; then
-                                eksctl delete cluster --region "$REGION" --name "$c" \
-                                    --disable-nodegroup-eviction --wait
-                            else
-                                echo "Skipping recent cluster: $c (created within last 24h)"
-                            fi
-                        done
-                    '''
+        stage('Cleanup Orphan Resources') {
+            when { expression { env.ACTION == 'CLEANUP_ORPHANS' } }
+            steps {
+                withCredentials([aws(credentialsId: 'pmm-staging-slave')]) {
+                    script {
+                        pmmHaEks.cleanupOrphans(region: env.REGION)
+                    }
                 }
             }
         }