Terraform Destroy Workflow #36
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Terraform Destroy Workflow | |
| on: | |
| workflow_dispatch: | |
| permissions: | |
| id-token: write | |
| contents: read | |
| actions: read | |
| jobs: | |
| terraform-destroy: | |
| name: Terraform Destroy | |
| runs-on: ubuntu-latest | |
| environment: production | |
| steps: | |
| - name: Checkout Repository | |
| uses: actions/checkout@v5 | |
| - name: Verify Variables Available | |
| run: | | |
| echo "CLUSTER_NAME: ${{ vars.CLUSTER_NAME }}" | |
| echo "NAMESPACE: ${{ vars.APP_NAMESPACE }}" | |
| echo "MONITORING_NAMESPACE: ${{ vars.MONITORING_NAMESPACE }}" | |
| echo "ARGOCD_NAMESPACE: ${{ vars.ARGOCD_NAMESPACE }}" | |
| echo "APP_NAME: ${{ vars.APP_NAME }}" | |
| echo "KARPENTER_NODEPOOL_NAME: ${{ vars.KARPENTER_NODEPOOL_NAME }}" | |
| echo "KARPENTER_NODECLASS_NAME: ${{ vars.KARPENTER_NODECLASS_NAME }}" | |
| echo "KARPENTER_NODE_ROLE: ${{ vars.KARPENTER_NODE_ROLE }}" | |
| echo "KARPENTER_INSTANCE_PROFILE: ${{ vars.KARPENTER_INSTANCE_PROFILE }}" | |
| echo "KARPENTER_NAMESPACE: ${{ vars.KARPENTER_NAMESPACE }}" | |
| if [[ -z "${{ vars.CLUSTER_NAME }}" ]]; then | |
| echo "ERROR: CLUSTER_NAME variable not found. Infrastructure may not be deployed." | |
| exit 1 | |
| fi | |
| if [[ -z "${{ vars.APP_NAMESPACE }}" ]]; then | |
| echo "ERROR: APP_NAMESPACE variable not found. Infrastructure may not be deployed." | |
| exit 1 | |
| fi | |
| - name: Configure AWS credentials via OIDC | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/GitHubActionsInfraRole | |
| aws-region: us-east-1 | |
| - name: Setup Terraform | |
| uses: hashicorp/[email protected] | |
| with: | |
| terraform_version: 1.5.7 | |
| - name: Update kubeconfig | |
| run: aws eks update-kubeconfig --name ${{ vars.CLUSTER_NAME }} --region us-east-1 | |
| continue-on-error: true | |
| - name: Install Helm | |
| uses: azure/[email protected] | |
| with: | |
| version: v3.14.0 | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 1: PRE-CLEANUP - GET KARPENTER VERSION INFO | |
| # ================================================== | |
| - name: Detect Karpenter Version and Resources | |
| id: karpenter-info | |
| run: | | |
| echo "🔍 Detecting Karpenter installation and version..." | |
| # Check if Karpenter is installed | |
| KARPENTER_INSTALLED=false | |
| if kubectl get namespace karpenter 2>/dev/null; then | |
| KARPENTER_INSTALLED=true | |
| echo "karpenter_installed=true" >> $GITHUB_OUTPUT | |
| # Try to get Karpenter version | |
| KARPENTER_VERSION=$(kubectl get deployment karpenter -n karpenter -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null | cut -d':' -f2 || echo "unknown") | |
| echo "Detected Karpenter version: $KARPENTER_VERSION" | |
| echo "karpenter_version=$KARPENTER_VERSION" >> $GITHUB_OUTPUT | |
| # Check which CRDs exist to determine API version | |
| if kubectl get crd provisioners.karpenter.sh 2>/dev/null; then | |
| echo "Found legacy Provisioner CRD (v1alpha5)" | |
| echo "api_version=legacy" >> $GITHUB_OUTPUT | |
| elif kubectl get crd nodepools.karpenter.sh 2>/dev/null; then | |
| echo "Found new NodePool CRD (v1beta1)" | |
| echo "api_version=new" >> $GITHUB_OUTPUT | |
| else | |
| echo "No Karpenter CRDs found" | |
| echo "api_version=none" >> $GITHUB_OUTPUT | |
| fi | |
| else | |
| echo "karpenter_installed=false" >> $GITHUB_OUTPUT | |
| echo "api_version=none" >> $GITHUB_OUTPUT | |
| fi | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 2: DELETE APPLICATIONS (ArgoCD first) | |
| # ================================================== | |
| - name: Delete ArgoCD Applications | |
| run: | | |
| echo "🔥 Deleting ArgoCD Applications..." | |
| kubectl delete application ${{ vars.APP_NAME }} -n ${{ vars.ARGOCD_NAMESPACE }} --ignore-not-found --timeout=60s || true | |
| kubectl delete application kube-prometheus-stack -n ${{ vars.ARGOCD_NAMESPACE }} --ignore-not-found --timeout=60s || true | |
| kubectl delete application --all -n ${{ vars.ARGOCD_NAMESPACE }} --ignore-not-found --timeout=60s || true | |
| echo "✅ ArgoCD applications cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 3: SCALE DOWN AND DELETE WORKLOADS + MISSING RESOURCES | |
| # ================================================== | |
| - name: Scale Down and Delete All Workloads | |
| run: | | |
| echo "📉 Comprehensive workload and resource cleanup..." | |
| # Get target namespaces | |
| TARGET_NAMESPACES=("${{ vars.APP_NAMESPACE }}" "${{ vars.MONITORING_NAMESPACE }}" "${{ vars.ARGOCD_NAMESPACE }}" "ingress-nginx" "${{ vars.KARPENTER_NAMESPACE }}") | |
| # Scale down ALL deployments, statefulsets, daemonsets | |
| for ns in "${TARGET_NAMESPACES[@]}"; do | |
| if [[ -n "$ns" ]] && kubectl get namespace $ns --ignore-not-found 2>/dev/null; then | |
| echo "Processing namespace: $ns" | |
| # Scale down workloads | |
| kubectl scale deployment --all --replicas=0 -n $ns --timeout=30s || true | |
| kubectl scale statefulset --all --replicas=0 -n $ns --timeout=30s || true | |
| # Delete workloads | |
| kubectl delete deployment,replicaset,statefulset,daemonset,job,cronjob --all -n $ns --ignore-not-found --timeout=60s || true | |
| # Delete network resources | |
| kubectl delete ingress --all -n $ns --ignore-not-found --timeout=30s || true | |
| kubectl delete networkpolicy --all -n $ns --ignore-not-found --timeout=30s || true | |
| # Delete storage | |
| kubectl delete pvc --all -n $ns --ignore-not-found --timeout=60s || true | |
| fi | |
| done | |
| # Delete ALL services of type LoadBalancer IMMEDIATELY | |
| echo "🔌 Deleting ALL LoadBalancer services..." | |
| kubectl get services --all-namespaces -o json | jq -r '.items[]? | select(.spec.type=="LoadBalancer") | "\(.metadata.namespace) \(.metadata.name)"' | while read namespace service; do | |
| if [[ -n "$namespace" && -n "$service" ]]; then | |
| echo " Deleting LoadBalancer service: $service in $namespace" | |
| kubectl delete service $service -n $namespace --ignore-not-found --timeout=60s || true | |
| fi | |
| done | |
| echo "⏳ Waiting for LoadBalancers to be cleaned up..." | |
| sleep 60 | |
| echo "✅ Workload and service cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 4: KARPENTER-SPECIFIC CLEANUP WITH VERSION DETECTION | |
| # ================================================== | |
| - name: Delete Karpenter Resources (Version-Aware) | |
| run: | | |
| echo "🚀 Starting Karpenter cleanup..." | |
| if [[ "${{ steps.karpenter-info.outputs.karpenter_installed }}" == "true" ]]; then | |
| API_VERSION="${{ steps.karpenter-info.outputs.api_version }}" | |
| echo "Detected Karpenter API version: $API_VERSION" | |
| # Force delete all Karpenter-managed nodes first | |
| echo "🗑️ Force deleting Karpenter nodes..." | |
| kubectl get nodes -l karpenter.sh/cluster=${{ vars.CLUSTER_NAME }} --no-headers -o custom-columns=":metadata.name" | xargs -r kubectl delete node --force --grace-period=0 || true | |
| kubectl get nodes -l node.kubernetes.io/instance-type --no-headers -o custom-columns=":metadata.name" | while read node; do | |
| if kubectl describe node $node 2>/dev/null | grep -q "karpenter"; then | |
| echo "Force deleting Karpenter node: $node" | |
| kubectl delete node $node --force --grace-period=0 || true | |
| fi | |
| done || true | |
| # Clean up based on detected API version | |
| if [[ "$API_VERSION" == "legacy" ]]; then | |
| echo "🧹 Cleaning up LEGACY Karpenter resources..." | |
| # Delete legacy Provisioners and AWSNodeTemplates | |
| kubectl delete provisioner --all --ignore-not-found --timeout=60s || true | |
| kubectl delete awsnodetemplate --all --ignore-not-found --timeout=60s || true | |
| # Remove finalizers from legacy resources | |
| kubectl get provisioner -o name | xargs -r -I {} kubectl patch {} -p '{"metadata":{"finalizers":[]}}' --type=merge || true | |
| kubectl get awsnodetemplate -o name | xargs -r -I {} kubectl patch {} -p '{"metadata":{"finalizers":[]}}' --type=merge || true | |
| elif [[ "$API_VERSION" == "new" ]]; then | |
| echo "🧹 Cleaning up NEW Karpenter resources..." | |
| # Delete new API resources | |
| kubectl delete nodepool --all --ignore-not-found --timeout=60s || true | |
| kubectl delete ec2nodeclass --all --ignore-not-found --timeout=60s || true | |
| kubectl delete nodeclaim --all --ignore-not-found --timeout=60s || true | |
| # Remove finalizers from new resources | |
| kubectl get nodepool -o name | xargs -r -I {} kubectl patch {} -p '{"metadata":{"finalizers":[]}}' --type=merge || true | |
| kubectl get ec2nodeclass -o name | xargs -r -I {} kubectl patch {} -p '{"metadata":{"finalizers":[]}}' --type=merge || true | |
| kubectl get nodeclaim -o name | xargs -r -I {} kubectl patch {} -p '{"metadata":{"finalizers":[]}}' --type=merge || true | |
| fi | |
| # Clean up Karpenter webhooks and CRDs | |
| echo "🕸️ Cleaning up Karpenter webhooks and CRDs..." | |
| kubectl delete validatingwebhookconfiguration validation.webhook.karpenter.sh --ignore-not-found || true | |
| kubectl delete validatingwebhookconfiguration defaulting.webhook.karpenter.sh --ignore-not-found || true | |
| kubectl delete mutatingwebhookconfiguration defaulting.webhook.karpenter.sh --ignore-not-found || true | |
| # Delete all Karpenter CRDs | |
| kubectl get crd -o name | grep karpenter | xargs -r kubectl delete --ignore-not-found --timeout=60s || true | |
| # Wait for cleanup | |
| sleep 30 | |
| else | |
| echo "ℹ️ Karpenter not installed, skipping Karpenter cleanup" | |
| fi | |
| echo "✅ Karpenter cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 5: AWS RESOURCE CLEANUP (CRITICAL ADDITION) | |
| # ================================================== | |
| - name: Clean Up AWS Resources Created by Karpenter | |
| run: | | |
| echo "☁️ Cleaning up AWS resources that might prevent Terraform destroy..." | |
| # Get cluster name for filtering | |
| CLUSTER_NAME="${{ vars.CLUSTER_NAME }}" | |
| # Terminate any EC2 instances with Karpenter tags | |
| echo "🖥️ Terminating Karpenter-managed EC2 instances..." | |
| INSTANCE_IDS=$(aws ec2 describe-instances \ | |
| --filters "Name=tag:karpenter.sh/cluster,Values=$CLUSTER_NAME" "Name=instance-state-name,Values=running,pending" \ | |
| --query 'Reservations[].Instances[].InstanceId' --output text || echo "") | |
| if [[ -n "$INSTANCE_IDS" && "$INSTANCE_IDS" != "None" ]]; then | |
| echo "Found Karpenter instances to terminate: $INSTANCE_IDS" | |
| aws ec2 terminate-instances --instance-ids $INSTANCE_IDS || true | |
| echo "⏳ Waiting for instances to terminate..." | |
| aws ec2 wait instance-terminated --instance-ids $INSTANCE_IDS --cli-read-timeout 300 || true | |
| else | |
| echo "No Karpenter-managed instances found" | |
| fi | |
| # Delete Launch Templates created by Karpenter | |
| echo "🚀 Deleting Karpenter Launch Templates..." | |
| aws ec2 describe-launch-templates \ | |
| --filters "Name=tag:karpenter.sh/cluster,Values=$CLUSTER_NAME" \ | |
| --query 'LaunchTemplates[].LaunchTemplateName' --output text | tr '\t' '\n' | while read template_name; do | |
| if [[ -n "$template_name" && "$template_name" != "None" ]]; then | |
| echo "Deleting launch template: $template_name" | |
| aws ec2 delete-launch-template --launch-template-name "$template_name" || true | |
| fi | |
| done || true | |
| # Clean up Auto Scaling Groups | |
| echo "📏 Checking for Karpenter Auto Scaling Groups..." | |
| aws autoscaling describe-auto-scaling-groups \ | |
| --query "AutoScalingGroups[?contains(Tags[?Key=='karpenter.sh/cluster'].Value, '$CLUSTER_NAME')].AutoScalingGroupName" \ | |
| --output text | tr '\t' '\n' | while read asg_name; do | |
| if [[ -n "$asg_name" && "$asg_name" != "None" ]]; then | |
| echo "Deleting ASG: $asg_name" | |
| aws autoscaling delete-auto-scaling-group --auto-scaling-group-name "$asg_name" --force-delete || true | |
| fi | |
| done || true | |
| echo "✅ AWS resource cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 6: UNINSTALL HELM RELEASES | |
| # ================================================== | |
| - name: Uninstall All Helm Releases | |
| run: | | |
| echo "📦 Uninstalling all Helm releases..." | |
| # Uninstall main application | |
| echo "Uninstalling application: ${{ vars.APP_NAME }}" | |
| helm uninstall ${{ vars.APP_NAME }} -n ${{ vars.APP_NAMESPACE }} --timeout=300s || true | |
| # Uninstall monitoring stack | |
| echo "Uninstalling monitoring stack..." | |
| helm uninstall kube-prometheus-stack -n ${{ vars.MONITORING_NAMESPACE }} --timeout=300s || true | |
| # Uninstall ingress controller | |
| echo "Uninstalling ingress controller..." | |
| helm uninstall ingress-nginx -n ingress-nginx --timeout=300s || true | |
| # Uninstall ArgoCD | |
| echo "Uninstalling ArgoCD..." | |
| helm uninstall argocd -n ${{ vars.ARGOCD_NAMESPACE }} --timeout=300s || true | |
| # Uninstall Karpenter | |
| echo "Uninstalling Karpenter..." | |
| helm uninstall karpenter -n karpenter --timeout=300s || true | |
| echo "⏳ Waiting for Helm releases to be fully removed..." | |
| sleep 30 | |
| echo "✅ Helm releases uninstalled" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 7: DELETE ALL CRDs (COMPREHENSIVE) | |
| # ================================================== | |
| - name: Delete All Custom Resource Definitions | |
| run: | | |
| echo "🗂️ Comprehensive CRD cleanup..." | |
| # Delete monitoring CRDs | |
| echo "Deleting monitoring CRDs..." | |
| kubectl get crd -o name | grep -E 'prometheus|grafana|alertmanager|servicemonitor|prometheusrule|podmonitor|thanosruler|coreos' | xargs -r kubectl delete --timeout=60s || true | |
| # Delete ArgoCD CRDs | |
| echo "Deleting ArgoCD CRDs..." | |
| kubectl get crd -o name | grep 'argoproj.io' | xargs -r kubectl delete --timeout=60s || true | |
| # Delete ingress CRDs | |
| echo "Deleting ingress CRDs..." | |
| kubectl get crd -o name | grep -E 'ingress|nginx' | xargs -r kubectl delete --timeout=60s || true | |
| # Delete ALL Karpenter CRDs (both old and new) | |
| echo "Deleting ALL Karpenter CRDs..." | |
| kubectl get crd -o name | grep karpenter | xargs -r kubectl delete --timeout=60s || true | |
| # Remove finalizers from ALL stuck CRDs | |
| echo "Removing finalizers from stuck CRDs..." | |
| kubectl get crd -o json | jq -r '.items[]? | select(.metadata.finalizers) | .metadata.name' | while read crd_name; do | |
| if [[ -n "$crd_name" ]]; then | |
| echo " Removing finalizers from CRD: $crd_name" | |
| kubectl patch crd $crd_name -p '{"metadata":{"finalizers":[]}}' --type=merge || true | |
| fi | |
| done | |
| echo "✅ CRDs cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 8: CLEANUP PERSISTENT STORAGE | |
| # ================================================== | |
| - name: Cleanup Persistent Storage | |
| run: | | |
| echo "💾 Cleaning up persistent storage..." | |
| # Delete PVCs in target namespaces first | |
| TARGET_NAMESPACES=("${{ vars.APP_NAMESPACE }}" "${{ vars.MONITORING_NAMESPACE }}" "${{ vars.ARGOCD_NAMESPACE }}" "ingress-nginx" "${{ vars.KARPENTER_NAMESPACE }}") | |
| for ns in "${TARGET_NAMESPACES[@]}"; do | |
| if [[ -n "$ns" ]] && kubectl get namespace $ns --ignore-not-found 2>/dev/null; then | |
| echo "Deleting PVCs in namespace: $ns" | |
| kubectl delete pvc --all -n $ns --timeout=120s || true | |
| fi | |
| done | |
| # Delete remaining PVCs across all namespaces | |
| echo "Deleting remaining PVCs across all namespaces..." | |
| kubectl delete pvc --all -A --timeout=120s || true | |
| # Delete PVs | |
| echo "Deleting Persistent Volumes..." | |
| kubectl delete pv --all --timeout=120s || true | |
| echo "✅ Persistent storage cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 9: AGGRESSIVE PRE-NAMESPACE CLEANUP | |
| # ================================================== | |
| - name: Aggressive Resource Cleanup Before Namespace Deletion | |
| run: | | |
| echo "🧹 Performing aggressive cleanup of resources that might block namespace deletion..." | |
| # List of namespaces to clean | |
| NAMESPACES=("${{ vars.APP_NAMESPACE }}" "${{ vars.MONITORING_NAMESPACE }}" "${{ vars.ARGOCD_NAMESPACE }}" "ingress-nginx" "${{ vars.KARPENTER_NAMESPACE }}") | |
| for ns in "${NAMESPACES[@]}"; do | |
| if [[ -n "$ns" ]] && kubectl get namespace $ns --ignore-not-found 2>/dev/null; then | |
| echo "========== Aggressively cleaning namespace: $ns ==========" | |
| # Force delete all pods immediately | |
| echo "Force deleting all pods..." | |
| kubectl delete pods --all -n $ns --force --grace-period=0 --ignore-not-found || true | |
| # Delete ALL resource types that could have finalizers | |
| echo "Deleting all remaining resources..." | |
| kubectl delete deployment,replicaset,statefulset,daemonset,job,cronjob --all -n $ns --ignore-not-found --timeout=30s || true | |
| kubectl delete svc --all -n $ns --ignore-not-found --timeout=30s || true | |
| kubectl delete ingress --all -n $ns --ignore-not-found --timeout=30s || true | |
| kubectl delete configmap --all -n $ns --ignore-not-found --timeout=30s || true | |
| kubectl delete secret --all -n $ns --ignore-not-found --timeout=30s || true | |
| # Delete monitoring-specific resources | |
| kubectl delete servicemonitor,prometheusrule,podmonitor --all -n $ns --ignore-not-found --timeout=30s || true | |
| kubectl delete prometheus,alertmanager,grafana --all -n $ns --ignore-not-found --timeout=30s || true | |
| # Delete ArgoCD-specific resources | |
| kubectl delete application,appproject --all -n $ns --ignore-not-found --timeout=30s || true | |
| # Remove finalizers from ALL remaining resources | |
| echo "Removing finalizers from remaining resources in $ns..." | |
| for resource_type in $(kubectl api-resources --verbs=list --namespaced -o name 2>/dev/null | grep -v events); do | |
| kubectl get $resource_type -n $ns -o json 2>/dev/null | \ | |
| jq -r '.items[]? | select(.metadata.finalizers) | .metadata.name' 2>/dev/null | \ | |
| while read resource_name; do | |
| if [[ -n "$resource_name" ]]; then | |
| echo " Patching $resource_type/$resource_name" | |
| kubectl patch $resource_type $resource_name -n $ns -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true | |
| fi | |
| done | |
| done | |
| echo "✅ Completed aggressive cleanup for namespace: $ns" | |
| fi | |
| done | |
| echo "⏳ Waiting for cleanup to propagate..." | |
| sleep 30 | |
| echo "✅ Pre-namespace cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 10: DELETE NAMESPACES WITH ENHANCED FORCE CLEANUP | |
| # ================================================== | |
| - name: Delete Namespaces with Enhanced Force Cleanup | |
| run: | | |
| #!/bin/bash | |
| echo "🗑️ Starting enhanced namespace deletion process..." | |
| # Function to completely force delete a namespace | |
| force_delete_namespace() { | |
| local ns=$1 | |
| echo "========== Processing namespace: $ns ==========" | |
| if ! kubectl get namespace "$ns" &>/dev/null; then | |
| echo "✅ Namespace $ns does not exist, skipping..." | |
| return 0 | |
| fi | |
| echo "📋 Current namespace status:" | |
| kubectl get namespace $ns -o wide || true | |
| # Step 1: Final resource cleanup in the namespace | |
| echo "🧹 Final cleanup of all resources in namespace $ns..." | |
| # Remove finalizers from all resources in the namespace | |
| for resource_type in $(kubectl api-resources --verbs=list --namespaced -o name 2>/dev/null | grep -v events); do | |
| kubectl get $resource_type -n $ns -o json 2>/dev/null | \ | |
| jq -r '.items[]? | select(.metadata.finalizers) | .metadata.name' 2>/dev/null | \ | |
| while read resource_name; do | |
| if [[ -n "$resource_name" ]]; then | |
| echo " Removing finalizers from $resource_type/$resource_name" | |
| kubectl patch $resource_type $resource_name -n $ns -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true | |
| fi | |
| done | |
| done | |
| # Step 2: Try graceful deletion first | |
| echo "🔄 Attempting graceful namespace deletion..." | |
| kubectl delete namespace $ns --timeout=60s --ignore-not-found & | |
| DELETE_PID=$! | |
| # Wait for graceful deletion | |
| sleep 30 | |
| # Step 3: If still exists, force delete | |
| if kubectl get namespace $ns --ignore-not-found 2>/dev/null; then | |
| echo "⚡ Graceful deletion failed, forcing deletion..." | |
| # Kill the background delete process | |
| kill $DELETE_PID 2>/dev/null || true | |
| # Get current namespace JSON and remove finalizers | |
| kubectl get namespace $ns -o json | \ | |
| jq 'del(.spec.finalizers[])' | \ | |
| kubectl replace --raw "/api/v1/namespaces/$ns/finalize" -f - 2>/dev/null || true | |
| # Alternative approach - patch the namespace directly | |
| kubectl patch namespace $ns -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true | |
| # Wait a bit more | |
| sleep 15 | |
| # Final check and force if needed | |
| if kubectl get namespace $ns --ignore-not-found 2>/dev/null; then | |
| echo "🚨 Trying nuclear option - direct deletion..." | |
| # Delete the namespace object directly | |
| kubectl delete namespace $ns --force --grace-period=0 2>/dev/null || true | |
| # Patch with empty spec | |
| kubectl patch namespace $ns -p '{"spec":{"finalizers":[]}}' --type=merge 2>/dev/null || true | |
| kubectl patch namespace $ns -p '{"metadata":{"finalizers":[]}}' --type=merge 2>/dev/null || true | |
| sleep 10 | |
| fi | |
| fi | |
| # Final verification | |
| if kubectl get namespace $ns --ignore-not-found 2>/dev/null; then | |
| echo "❌ WARNING: Namespace $ns still exists after all attempts" | |
| echo "📋 Final namespace details:" | |
| kubectl get namespace $ns -o yaml || true | |
| return 1 | |
| else | |
| echo "✅ Successfully deleted namespace $ns" | |
| return 0 | |
| fi | |
| } | |
| # Array of namespaces to delete | |
| NAMESPACES=("${{ vars.APP_NAMESPACE }}" "${{ vars.MONITORING_NAMESPACE }}" "${{ vars.ARGOCD_NAMESPACE }}" "ingress-nginx" "${{ vars.KARPENTER_NAMESPACE }}") | |
| # Delete each namespace | |
| FAILED_NAMESPACES=() | |
| for ns in "${NAMESPACES[@]}"; do | |
| if [[ -n "$ns" ]]; then | |
| if ! force_delete_namespace "$ns"; then | |
| FAILED_NAMESPACES+=("$ns") | |
| fi | |
| echo "" | |
| fi | |
| done | |
| # Summary | |
| echo "========== NAMESPACE CLEANUP SUMMARY ==========" | |
| echo "📊 Remaining namespaces:" | |
| kubectl get namespaces || true | |
| if [[ ${#FAILED_NAMESPACES[@]} -eq 0 ]]; then | |
| echo "✅ All target namespaces successfully deleted!" | |
| else | |
| echo "❌ Failed to delete namespaces: ${FAILED_NAMESPACES[*]}" | |
| echo "⚠️ You may need to check these manually after terraform destroy completes" | |
| # Don't fail the workflow for namespace cleanup issues | |
| fi | |
| echo "✅ Namespace deletion process completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 11: VERIFICATION BEFORE TERRAFORM DESTROY | |
| # ================================================== | |
| - name: Pre-Terraform Destroy Verification | |
| run: | | |
| echo "🔍 Performing comprehensive verification before Terraform destroy..." | |
| echo "⏳ Final waiting period for resource cleanup..." | |
| sleep 60 | |
| echo "📊 COMPREHENSIVE CLUSTER STATE VERIFICATION:" | |
| echo "1. Remaining pods in target namespaces:" | |
| kubectl get pods -A | grep -E "${{ vars.APP_NAMESPACE }}|${{ vars.MONITORING_NAMESPACE }}|${{ vars.ARGOCD_NAMESPACE }}|ingress-nginx|${{ vars.KARPENTER_NAMESPACE }}" || echo "✅ No pods found in target namespaces" | |
| echo "2. Remaining namespaces:" | |
| kubectl get namespaces | grep -E "${{ vars.APP_NAMESPACE }}|${{ vars.MONITORING_NAMESPACE }}|${{ vars.ARGOCD_NAMESPACE }}|ingress-nginx|${{ vars.KARPENTER_NAMESPACE }}" || echo "✅ No target namespaces found" | |
| echo "3. Remaining LoadBalancer services:" | |
| kubectl get svc -A --field-selector spec.type=LoadBalancer | grep -v "NAMESPACE" || echo "✅ No LoadBalancer services found" | |
| echo "4. Remaining Karpenter CRDs:" | |
| kubectl get crd | grep karpenter || echo "✅ No Karpenter CRDs found" | |
| echo "5. Remaining PVCs:" | |
| kubectl get pvc -A | grep -v "NAMESPACE" || echo "✅ No PVCs found" | |
| echo "6. Remaining custom resources:" | |
| kubectl get application -A 2>/dev/null || echo "✅ No ArgoCD applications found" | |
| kubectl get prometheus -A 2>/dev/null || echo "✅ No Prometheus resources found" | |
| echo "7. AWS LoadBalancers check:" | |
| aws elbv2 describe-load-balancers --query 'LoadBalancers[?starts_with(LoadBalancerName, `k8s-`)].LoadBalancerName' --output text || echo "✅ No Kubernetes LoadBalancers found in AWS" | |
| echo "✅ Pre-Terraform verification completed - proceeding to Terraform destroy" | |
| # ================================================== | |
| # PHASE 12: TERRAFORM DESTROY | |
| # ================================================== | |
| - name: Terraform Init | |
| run: terraform init | |
| working-directory: ./Terraform | |
| - name: Terraform Destroy Plan | |
| run: terraform plan -destroy | |
| working-directory: ./Terraform | |
| - name: Terraform Destroy | |
| run: terraform destroy -auto-approve | |
| working-directory: ./Terraform | |
| # ================================================== | |
| # PHASE 13: POST-TERRAFORM AWS CLEANUP (SAFETY NET) | |
| # ================================================== | |
| - name: Post-Terraform AWS Resource Cleanup | |
| run: | | |
| echo "🧹 Post-Terraform cleanup - catching any remaining AWS resources..." | |
| CLUSTER_NAME="${{ vars.CLUSTER_NAME }}" | |
| # Clean up any remaining LoadBalancers that might have been missed | |
| echo "🔌 Final LoadBalancer cleanup..." | |
| aws elbv2 describe-load-balancers --query "LoadBalancers[?contains(LoadBalancerName, 'k8s-$CLUSTER_NAME') || contains(LoadBalancerName, 'k8s-ingress')].LoadBalancerArn" --output text | tr '\t' '\n' | while read lb_arn; do | |
| if [[ -n "$lb_arn" && "$lb_arn" != "None" ]]; then | |
| echo "Deleting remaining LoadBalancer: $lb_arn" | |
| aws elbv2 delete-load-balancer --load-balancer-arn "$lb_arn" || true | |
| fi | |
| done || true | |
| # Clean up any remaining Security Groups with cluster tags | |
| echo "🛡️ Cleaning up remaining Security Groups..." | |
| aws ec2 describe-security-groups --filters "Name=tag:kubernetes.io/cluster/$CLUSTER_NAME,Values=owned" --query 'SecurityGroups[].GroupId' --output text | tr '\t' '\n' | while read sg_id; do | |
| if [[ -n "$sg_id" && "$sg_id" != "None" ]]; then | |
| echo "Deleting security group: $sg_id" | |
| aws ec2 delete-security-group --group-id "$sg_id" || true | |
| fi | |
| done || true | |
| # Clean up SQS Queue created for Karpenter | |
| echo "📬 Cleaning up Karpenter SQS queue..." | |
| aws sqs delete-queue --queue-url "https://sqs.us-east-1.amazonaws.com/${{ secrets.AWS_ACCOUNT_ID }}/karpenter-interruption-queue-$CLUSTER_NAME" || true | |
| echo "✅ Post-Terraform AWS cleanup completed" | |
| continue-on-error: true | |
| # ================================================== | |
| # PHASE 14: CLEANUP GITHUB VARIABLES | |
| # ================================================== | |
| - name: Remove GitHub Repository Variables | |
| run: | | |
| echo "🧹 Cleaning up GitHub repository variables..." | |
| gh variable delete CLUSTER_NAME --repo $GITHUB_REPOSITORY || true | |
| gh variable delete APP_NAMESPACE --repo $GITHUB_REPOSITORY || true | |
| gh variable delete MONITORING_NAMESPACE --repo $GITHUB_REPOSITORY || true | |
| gh variable delete ARGOCD_NAMESPACE --repo $GITHUB_REPOSITORY || true | |
| gh variable delete APP_NAME --repo $GITHUB_REPOSITORY || true | |
| gh variable delete KARPENTER_NODEPOOL_NAME --repo $GITHUB_REPOSITORY || true | |
| gh variable delete KARPENTER_NODECLASS_NAME --repo $GITHUB_REPOSITORY || true | |
| gh variable delete KARPENTER_NODE_ROLE --repo $GITHUB_REPOSITORY || true | |
| gh variable delete KARPENTER_INSTANCE_PROFILE --repo $GITHUB_REPOSITORY || true | |
| gh variable delete KARPENTER_NAMESPACE --repo $GITHUB_REPOSITORY || true | |
| echo "✅ GitHub variables cleanup completed" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.PAT_GITHUB }} | |
| continue-on-error: true | |
| # ================================================== | |
| # FINAL SUCCESS MESSAGE WITH MANUAL CLEANUP INSTRUCTIONS | |
| # ================================================== | |
| - name: Destroy Complete with Manual Cleanup Guide | |
| run: | | |
| echo "🎉 =========================================" | |
| echo "🎉 TERRAFORM DESTROY WORKFLOW COMPLETED!" | |
| echo "🎉 =========================================" | |
| echo "✅ All resources have been cleaned up" | |
| echo "✅ Infrastructure has been destroyed" | |
| echo "✅ GitHub variables have been removed" | |
| echo "" | |
| echo "🔍 MANUAL VERIFICATION RECOMMENDED:" | |
| echo "1. Check AWS Console for any remaining:" | |
| echo " - EC2 instances with tags containing '${{ vars.CLUSTER_NAME }}'" | |
| echo " - LoadBalancers starting with 'k8s-'" | |
| echo " - Security Groups with cluster tags" | |
| echo " - Launch Templates with Karpenter tags" | |
| echo " - Auto Scaling Groups" | |
| echo "" | |
| echo "2. If any namespaces are still stuck, manually run:" | |
| echo " kubectl get namespace <namespace-name> -o json | jq 'del(.spec.finalizers[])' | kubectl replace --raw \"/api/v1/namespaces/<namespace-name>/finalize\" -f -" | |
| echo "" | |
| echo "3. Check S3 bucket 'solar-system-terraform-state-123456' - Terraform state should be updated to show no resources" | |
| echo "" | |
| echo "🚨 If you see any AWS charges after this, check the resources listed above!" |