Auto Cleanup ECS inactive TaskDefs (#1800)

Co-authored-by: Akansha Agarwal <[email protected]>

Author: agarakan

.github/workflows/clean-aws-resources.yml

@@ -159,7 +159,7 @@ jobs:
         working-directory: tool/clean
         run: go run ./clean_host/clean_host.go cn-north-1
 
-  clean-ecs-clusters:
+  clean-ecs-resources:
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -174,9 +174,9 @@ jobs:
           role-to-assume: ${{ secrets.TERRAFORM_AWS_ASSUME_ROLE }}
           aws-region: us-west-2
 
-      - name: Clean old ecs cluster
+      - name: Clean old ecs resources
         working-directory: tool/clean
-        run: go run ./clean_ecs/clean_ecs.go --tags=clean
+        run: go run --tags=clean ./clean_ecs/clean_ecs.go us-west-2
 
   clean-eks-clusters:
     runs-on: ubuntu-latest

tool/clean/clean_ecs/clean_ecs.go

@@ -9,12 +9,14 @@ package main
 import (
 	"context"
 	"log"
+	"os"
 	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/ecs"
+	"github.com/aws/aws-sdk-go-v2/service/ecs/types"
 
 	"github.com/aws/amazon-cloudwatch-agent/tool/clean"
 )
@@ -23,17 +25,20 @@ import (
 
 var expirationTimeOneWeek = time.Now().UTC().Add(-clean.KeepDurationOneWeek)
 
-const cwaIntegTestClusterPrefix = "cwagent-integ-test-cluster-"
+const clusterPrefix = "cwagent-integ-test-cluster-"
+
+var taskdefPrefixes = []string{"cwagent-integ-test-", "extra-apps-family-", "cwagent-task-family-"}
 
 func main() {
 	ctx := context.Background()
-	defaultConfig, err := config.LoadDefaultConfig(ctx)
+	defaultConfig, err := config.LoadDefaultConfig(ctx, config.WithRegion(os.Args[1]))
 	if err != nil {
 		log.Fatalf("Error loading AWS config for ECS cleanup: %v", err)
 	}
 
 	ecsClient := ecs.NewFromConfig(defaultConfig)
 	terminateClusters(ctx, ecsClient)
+	deleteInactiveTaskDefinitions(ctx, ecsClient)
 }
 
 func terminateClusters(ctx context.Context, client *ecs.Client) {
@@ -65,7 +70,7 @@ func terminateClusters(ctx context.Context, client *ecs.Client) {
 		*/
 
 		for _, cluster := range describeClustersOutput.Clusters {
-			if !strings.HasPrefix(*cluster.ClusterName, cwaIntegTestClusterPrefix) {
+			if !strings.HasPrefix(*cluster.ClusterName, clusterPrefix) {
 				continue
 			}
 			if cluster.RunningTasksCount == 0 && cluster.PendingTasksCount == 0 {
@@ -86,6 +91,10 @@ func terminateClusters(ctx context.Context, client *ecs.Client) {
 		ecsListClusterInput.NextToken = listClusterOutput.NextToken
 	}
 
+	if len(clusterIds) == 0 {
+		log.Print("No clusters to delete.")
+	}
+
 	// Deletion Logic
 	for _, clusterId := range clusterIds {
 		log.Printf("Cluster to terminate: %s", *clusterId)
@@ -168,3 +177,68 @@ func isClusterTasksExpired(ctx context.Context, client *ecs.Client, clusterArn *
 	}
 	return true
 }
+
+func deleteInactiveTaskDefinitions(ctx context.Context, client *ecs.Client) {
+	log.Print("Begin cleanup of inactive task definitions")
+
+	taskDefsToDelete := getECSTaskDefsToDelete(ctx, client)
+
+	if len(taskDefsToDelete) == 0 {
+		log.Printf("No inactive task definitions to delete")
+		return
+	}
+
+	log.Printf("Found %d inactive task definitions to delete", len(taskDefsToDelete))
+
+	// Batch delete task definitions (API supports up to 10 at a time)
+	const batchSize = 10
+	totalDeleted := 0
+
+	for i := 0; i < len(taskDefsToDelete); i += batchSize {
+		end := min(i+batchSize, len(taskDefsToDelete))
+		output, err := client.DeleteTaskDefinitions(ctx, &ecs.DeleteTaskDefinitionsInput{
+			TaskDefinitions: taskDefsToDelete[i:end],
+		})
+		if err != nil {
+			log.Printf("Error batch deleting task definitions: %v", err)
+			continue
+		}
+
+		totalDeleted += len(output.TaskDefinitions)
+		for _, failure := range output.Failures {
+			log.Printf("Failed to delete task definition %s: %s", *failure.Arn, *failure.Reason)
+		}
+	}
+
+	log.Printf("Successfully deleted %d task definitions", totalDeleted)
+}
+
+func getECSTaskDefsToDelete(ctx context.Context, client *ecs.Client) []string {
+	taskDefsToDelete := make([]string, 0)
+
+	for _, prefix := range taskdefPrefixes {
+		// List inactive task definitions with integration test prefix
+		listTaskDefsInput := ecs.ListTaskDefinitionsInput{
+			FamilyPrefix: aws.String(prefix),
+			Status:       types.TaskDefinitionStatusInactive,
+		}
+
+		for {
+			listTaskDefsOutput, err := client.ListTaskDefinitions(ctx, &listTaskDefsInput)
+			if err != nil {
+				log.Printf("Error listing task definitions: %v", err)
+				break
+			}
+			if len(listTaskDefsOutput.TaskDefinitionArns) == 0 {
+				break
+			}
+
+			taskDefsToDelete = append(taskDefsToDelete, listTaskDefsOutput.TaskDefinitionArns...)
+			if listTaskDefsOutput.NextToken == nil {
+				break
+			}
+			listTaskDefsInput.NextToken = listTaskDefsOutput.NextToken
+		}
+	}
+	return taskDefsToDelete
+}