Move overlay cleanup to code (#1696)

akshaysngupta · web-flow · commit 494ff1484173 · 2025-02-11T11:27:30.000-08:00
* cleanup using code

* update render tests
diff --git a/cmd/appgw-ingress/main.go b/cmd/appgw-ingress/main.go
@@ -56,6 +56,7 @@ var (
 	resyncPeriod   = flags.Duration("sync-period", resyncPause, "Interval at which to re-list and confirm cloud resources.")
 	versionInfo    = flags.Bool("version", false, "Print version")
 	verbosity      = flags.Int(verbosityFlag, 1, "Set logging verbosity level")
+	cleanupOEC     = flags.Bool("cleanup-oec", false, "Cleanup OverlayExtensionConfig resources")
 )
 
 var allowedSkus = map[n.ApplicationGatewayTier]interface{}{
@@ -105,6 +106,14 @@ func main() {
 		klog.Fatalf("Failed to create controller-runtime client: %v", err)
 	}
 
+	if *cleanupOEC {
+		if err := cni.CleanupOverlayExtensionConfigs(ctrlClient, env.AGICPodNamespace, env.AddonMode); err != nil {
+			klog.Fatalf("Failed to cleanup OverlayExtensionConfig resources: %v", err)
+		}
+		klog.Info("Successfully cleaned up OverlayExtensionConfig resources")
+		return
+	}
+
 	kubeClient := kubernetes.NewForConfigOrDie(apiConfig)
 	k8scontext.IsNetworkingV1PackageSupported = k8scontext.SupportsNetworkingPackage(kubeClient)
 	k8scontext.IsInMultiClusterMode = env.MultiClusterMode
diff --git a/helm/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/templates/cleanup-job.yaml
@@ -17,22 +17,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
 {{- if .Values.addon }}
           - "app.kubernetes.io/managed-by=ingress-appgw-addon"
 {{- else }}
diff --git a/helm/ingress-azure/tests/snapshots/sample-config-addon/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/tests/snapshots/sample-config-addon/ingress-azure/templates/cleanup-job.yaml
@@ -19,22 +19,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: mcr.microsoft.com/azure-application-gateway/kubernetes-ingress:1.6.0
+        imagePullPolicy: Always
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
           - "app.kubernetes.io/managed-by=ingress-appgw-addon"
         securityContext:
           capabilities:
diff --git a/helm/ingress-azure/tests/snapshots/sample-config-empty/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/tests/snapshots/sample-config-empty/ingress-azure/templates/cleanup-job.yaml
@@ -19,22 +19,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: mcr.microsoft.com/azure-application-gateway/kubernetes-ingress:1.6.0
+        imagePullPolicy: Always
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
           - "app.kubernetes.io/managed-by=ingress-appgw-helm"
         securityContext:
           capabilities:
diff --git a/helm/ingress-azure/tests/snapshots/sample-config-existing-secret/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/tests/snapshots/sample-config-existing-secret/ingress-azure/templates/cleanup-job.yaml
@@ -19,22 +19,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: mcr.microsoft.com/azure-application-gateway/kubernetes-ingress:1.6.0
+        imagePullPolicy: Always
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
           - "app.kubernetes.io/managed-by=ingress-appgw-helm"
         securityContext:
           capabilities:
diff --git a/helm/ingress-azure/tests/snapshots/sample-config-prohibited-target/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/tests/snapshots/sample-config-prohibited-target/ingress-azure/templates/cleanup-job.yaml
@@ -19,22 +19,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: mcr.microsoft.com/azure-application-gateway/kubernetes-ingress:1.6.0
+        imagePullPolicy: Always
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
           - "app.kubernetes.io/managed-by=ingress-appgw-helm"
         securityContext:
           capabilities:
diff --git a/helm/ingress-azure/tests/snapshots/sample-config-workload-identity/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/tests/snapshots/sample-config-workload-identity/ingress-azure/templates/cleanup-job.yaml
@@ -19,22 +19,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: mcr.microsoft.com/azure-application-gateway/kubernetes-ingress:1.6.0
+        imagePullPolicy: Always
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
           - "app.kubernetes.io/managed-by=ingress-appgw-helm"
         securityContext:
           capabilities:
diff --git a/helm/ingress-azure/tests/snapshots/sample-config/ingress-azure/templates/cleanup-job.yaml b/helm/ingress-azure/tests/snapshots/sample-config/ingress-azure/templates/cleanup-job.yaml
@@ -19,22 +19,16 @@ spec:
       restartPolicy: OnFailure
       containers:
       - name: cleanup
-        image: "mcr.microsoft.com/oss/kubernetes/kubectl:v1.30.5"
-        imagePullPolicy: IfNotPresent
+        image: mcr.microsoft.com/azure-application-gateway/kubernetes-ingress:1.6.0
+        imagePullPolicy: Always
         env:
         - name: AGIC_POD_NAMESPACE
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
         command:
-          - "kubectl"
-          - "delete"
-          - "--ignore-not-found"
-          - "--wait"
-          - "-n"
-          - "$(AGIC_POD_NAMESPACE)"
-          - "overlayextensionconfigs.acn.azure.com"
-          - "-l"
+          - "/appgw-ingress"
+          - "--cleanup-oec"
           - "app.kubernetes.io/managed-by=ingress-appgw-helm"
         securityContext:
           capabilities:
diff --git a/pkg/cni/cleanup.go b/pkg/cni/cleanup.go
@@ -0,0 +1,68 @@
+package cni
+
+import (
+	"context"
+
+	overlayv1alpha1 "github.com/Azure/azure-container-networking/crd/overlayextensionconfig/api/v1alpha1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func CleanupOverlayExtensionConfigs(k8sClient client.Client, namespace string, addonMode bool) error {
+	// Define a label selector for filtering the resources to delete.
+	managedByValue := ResourceManagedByHelmValue
+	if addonMode {
+		managedByValue = ResourceManagedByAddonValue
+	}
+
+	// Perform cleanup of OverlayExtensionConfig resources.
+	if err := cleanupOverlayExtensionConfigs(k8sClient, namespace, managedByValue); err != nil {
+		klog.Errorf("Error cleaning up OverlayExtensionConfig resources: %v", err)
+		return err
+	}
+
+	klog.Infof("Cleanup completed successfully.")
+	return nil
+}
+
+// cleanupOverlayExtensionConfigs lists and deletes OverlayExtensionConfig resources in the given namespace
+// that match the provided label selector. If the CRD is not present, it logs a warning and returns nil.
+func cleanupOverlayExtensionConfigs(c client.Client, namespace string, label string) error {
+	ctx := context.Background()
+
+	// Create an empty list to hold OverlayExtensionConfig resources.
+	var overlayList overlayv1alpha1.OverlayExtensionConfigList
+
+	// List the resources with the provided namespace and label selector.
+	if err := c.List(ctx, &overlayList,
+		client.InNamespace(namespace),
+		client.MatchingLabels(map[string]string{ResourceManagedByLabel: label})); err != nil {
+		// If the API server does not recognize the CRD, skip cleanup.
+		if meta.IsNoMatchError(err) {
+			klog.Warning("CRD OverlayExtensionConfig not found in the cluster. Skipping cleanup.")
+			return nil
+		}
+		return err
+	}
+
+	// If no resources are found, log and exit.
+	if len(overlayList.Items) == 0 {
+		klog.Infof("No OverlayExtensionConfig resources found in namespace %q with labels %q", namespace, label)
+		return nil
+	}
+
+	// Iterate through and delete each OverlayExtensionConfig resource.
+	var deletionError error
+	for _, item := range overlayList.Items {
+		klog.Infof("Deleting OverlayExtensionConfig: %q", item.Name)
+		if err := c.Delete(ctx, &item, &client.DeleteOptions{}); err != nil {
+			klog.Errorf("Error deleting resource %q: %v", item.Name, err)
+			deletionError = err
+		} else {
+			klog.Infof("Successfully deleted resource %q", item.Name)
+		}
+	}
+
+	return deletionError
+}
diff --git a/pkg/cni/cleanup_test.go b/pkg/cni/cleanup_test.go
