delete AKS extensions when they are removed from spec

Author: nojnhuh

azure/services/aksextensions/aksextensions.go

@@ -17,10 +17,14 @@ limitations under the License.
 package aksextensions
 
 import (
+	"context"
+
 	asokubernetesconfigurationv1 "github.com/Azure/azure-service-operator/v2/api/kubernetesconfiguration/v1api20230501"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	"sigs.k8s.io/cluster-api-provider-azure/azure/services/aso"
+	"sigs.k8s.io/cluster-api-provider-azure/util/slice"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 const serviceName = "extension"
@@ -41,10 +45,17 @@ type Service struct {
 // New creates a new service.
 func New(scope AKSExtensionScope) *Service {
 	svc := aso.NewService[*asokubernetesconfigurationv1.Extension, AKSExtensionScope](serviceName, scope)
+	svc.ListFunc = list
 	svc.Specs = scope.AKSExtensionSpecs()
 	svc.ConditionType = infrav1.AKSExtensionsReadyCondition
 	return &Service{
 		Scope:   scope,
 		Service: svc,
 	}
 }
+
+func list(ctx context.Context, client client.Client, opts ...client.ListOption) ([]*asokubernetesconfigurationv1.Extension, error) {
+	list := &asokubernetesconfigurationv1.ExtensionList{}
+	err := client.List(ctx, list, opts...)
+	return slice.ToPtrs(list.Items), err
+}

test/e2e/aks_marketplace.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/utils/ptr"
 	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
+	"sigs.k8s.io/cluster-api-provider-azure/azure"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/conditions"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -165,6 +166,21 @@ func AKSMarketplaceExtensionSpec(ctx context.Context, inputGetter func() AKSMark
 	ensureAKSExtensionAdded(ctx, input, extensionName, "TraefikLabs.TraefikProxy", extensionClient, amcp)
 	ensureAKSExtensionAdded(ctx, input, officialExtensionName, "microsoft.flux", extensionClient, amcp)
 
+	By("Deleting the AKS Marketplace Extension")
+	Eventually(func(g Gomega) {
+		err = mgmtClient.Get(ctx, client.ObjectKey{
+			Namespace: input.Cluster.Spec.ControlPlaneRef.Namespace,
+			Name:      input.Cluster.Spec.ControlPlaneRef.Name,
+		}, infraControlPlane)
+		g.Expect(err).NotTo(HaveOccurred())
+		infraControlPlane.Spec.Extensions = []infrav1.AKSExtension{}
+		g.Expect(mgmtClient.Update(ctx, infraControlPlane)).To(Succeed())
+	}, input.WaitIntervals...).Should(Succeed())
+
+	By("Ensuring the AKS Marketplace Extension is deleted from the AzureManagedControlPlane")
+	ensureAKSExtensionDeleted(ctx, input, extensionName, extensionClient, amcp)
+	ensureAKSExtensionDeleted(ctx, input, officialExtensionName, extensionClient, amcp)
+
 	By("Restoring initial taints for Windows machine pool")
 	expectedTaints = initialTaints
 	Eventually(func(g Gomega) {
@@ -187,3 +203,10 @@ func ensureAKSExtensionAdded(ctx context.Context, input AKSMarketplaceExtensionS
 		g.Expect(extension.Properties.ExtensionType).To(Equal(ptr.To(extensionType)))
 	}, input.WaitIntervals...).Should(Succeed())
 }
+
+func ensureAKSExtensionDeleted(ctx context.Context, input AKSMarketplaceExtensionSpecInput, extensionName string, extensionClient *armkubernetesconfiguration.ExtensionsClient, amcp *infrav1.AzureManagedControlPlane) {
+	Eventually(func(g Gomega) {
+		_, err := extensionClient.Get(ctx, amcp.Spec.ResourceGroupName, "Microsoft.ContainerService", "managedClusters", input.Cluster.Name, extensionName, nil)
+		g.Expect(azure.ResourceNotFound(err)).To(BeTrue())
+	}, input.WaitIntervals...).Should(Succeed())
+}