Clean Up AWS Resources After Cluster Deletion (#1396)

* clean sgroup, oidc while delete cluster

Signed-off-by: Vivek Reddy <[email protected]>

* adding account_id access

* review comments

Signed-off-by: vivekr-splunk <[email protected]>

* review comments

Signed-off-by: vivekr-splunk <[email protected]>

* adding wait for network rsr deletion

Signed-off-by: vivekr-splunk <[email protected]>

---------

Signed-off-by: Vivek Reddy <[email protected]>
Signed-off-by: vivekr-splunk <[email protected]>

Author: vivekr-splunk

test/deploy-eks-cluster.sh

@@ -21,32 +21,64 @@ if [[ -z "${EKS_CLUSTER_K8_VERSION}" ]]; then
 fi
 
 function deleteCluster() {
+  echo "Cleanup role, security-group, open-id ${TEST_CLUSTER_NAME}"
+  account_id=$(aws sts get-caller-identity --query "Account" --output text)
+  rolename=$(echo ${TEST_CLUSTER_NAME} | awk -F- '{print "EBS_" $(NF-1) "_" $(NF)}')
+  
+  # Detach role policies
+  role_attached_policies=$(aws iam list-attached-role-policies --role-name $rolename --query 'AttachedPolicies[*].PolicyArn' --output text)
+  for policy_arn in ${role_attached_policies}; do
+    aws iam detach-role-policy --role-name ${rolename} --policy-arn ${policy_arn}
+  done
+
+  # Delete IAM role
+  aws iam delete-role --role-name ${rolename}
+
+  # Delete OIDC provider
+  oidc_id=$(aws eks describe-cluster --name ${TEST_CLUSTER_NAME} --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)
+  aws iam delete-open-id-connect-provider --open-id-connect-provider-arn arn:aws:iam::${account_id}:oidc-provider/oidc.eks.us-west-2.amazonaws.com/id/${oidc_id}
+
+  # Get security group ID
+  security_group_id=$(aws eks describe-cluster --name ${TEST_CLUSTER_NAME} --query "cluster.resourcesVpcConfig.securityGroupIds[0]" --output text)
+
+  # Cleanup remaining PVCs on the EKS Cluster
   echo "Cleanup remaining PVC on the EKS Cluster ${TEST_CLUSTER_NAME}"
   tools/cleanup.sh
+
+  # Get node group
   NODE_GROUP=$(eksctl get nodegroup --cluster=${TEST_CLUSTER_NAME} | sed -n 4p | awk '{ print $2 }')
-  if [[  ! -z "${NODE_GROUP}" ]]; then
-    eksctl delete nodegroup --cluster=${TEST_CLUSTER_NAME} --name=${NODE_GROUP}
-    if [ $? -ne 0 ]; then
-      echo "Unable to delete Nodegroup ${NODE_GROUP}. For Cluster - ${TEST_CLUSTER_NAME}"
-    fi
-  fi
-  eksctl delete cluster --name=${TEST_CLUSTER_NAME}
+
+  # Delete the node group to ensure no EC2 instances are using the security group
+  echo "Deleting node group - ${NODE_GROUP}"
+  eksctl delete nodegroup --cluster=${TEST_CLUSTER_NAME} --name=${NODE_GROUP}
+
+  # Delete cluster
+  echo "Deleting cluster - ${TEST_CLUSTER_NAME}"
+  eksctl delete cluster --name ${TEST_CLUSTER_NAME}
+
   if [ $? -ne 0 ]; then
     echo "Unable to delete cluster - ${TEST_CLUSTER_NAME}"
     return 1
   fi
-  rolename=$(echo ${TEST_CLUSTER_NAME} | awk -F- '{print "EBS_" $(NF-1) "_" $(NF)}')
-  role_attached_policies=$(aws iam list-attached-role-policies --role-name $rolename --query 'AttachedPolicies[*].PolicyArn' --output text)
-  for policy_arn in ${role_attached_policies};
-  do
-    aws iam detach-role-policy --role-name ${rolename} --policy-arn ${policy_arn}
+
+  # Wait for the cluster resources to be fully released before deleting security group
+  echo "Waiting for resources to be detached from security group - ${security_group_id}"
+  while true; do
+    ENIs=$(aws ec2 describe-network-interfaces --filters "Name=group-id,Values=${security_group_id}" --query "NetworkInterfaces[*].NetworkInterfaceId" --output text)
+    if [ -z "${ENIs}" ]; then
+      break
+    fi
+    echo "ENIs still attached to security group: ${ENIs}. Waiting for cleanup..."
+    sleep 10
   done
 
-  aws iam delete-role --role-name ${rolename}
+  # Delete security group
+  aws ec2 delete-security-group --group-id ${security_group_id}
 
   return 0
 }
 
+
 function createCluster() {
   # Deploy eksctl cluster if not deploy
   rc=$(which eksctl)