@@ -23,12 +23,15 @@ import (
2323 amazoncni "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
2424 appsv1 "k8s.io/api/apps/v1"
2525 corev1 "k8s.io/api/core/v1"
26+ rbacv1 "k8s.io/api/rbac/v1"
2627 apierrors "k8s.io/apimachinery/pkg/api/errors"
28+ "k8s.io/apimachinery/pkg/api/meta"
2729 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2830 "k8s.io/apimachinery/pkg/labels"
2931 "k8s.io/apimachinery/pkg/types"
3032 "k8s.io/klog/v2"
3133 "sigs.k8s.io/controller-runtime/pkg/client"
34+ "sigs.k8s.io/kustomize/api/konfig"
3235
3336 infrav1 "sigs.k8s.io/cluster-api-provider-aws/api/v1beta2"
3437 "sigs.k8s.io/cluster-api-provider-aws/pkg/cloud/awserrors"
@@ -239,26 +242,77 @@ func (s *Service) applyUserProvidedEnvironmentProperties(containerEnv []corev1.E
239242}
240243
241244func (s * Service ) deleteCNI (ctx context.Context , remoteClient client.Client ) error {
242- s .scope .Info ("Ensuring aws-node DaemonSet in cluster is deleted" , "cluster" , klog .KRef (s .scope .Namespace (), s .scope .Name ()))
245+ // EKS has a tendency to pre-install the vpc-cni automagically even if you don't specify it as an addon
246+ // and looks like a kubectl apply from a script of a manifest that looks like this
247+ // https://github.com/aws/amazon-vpc-cni-k8s/blob/master/config/master/aws-k8s-cni.yaml
248+ // and removing these pieces will enable someone to install and alternative CNI. There is also another use
249+ // case where someone would want to remove the vpc-cni and reinstall it via the helm chart located here
250+ // https://github.com/aws/amazon-vpc-cni-k8s/tree/master/charts/aws-vpc-cni meaning we need to account for
251+ // managed-by: Helm label, or we will delete the helm chart resources every reconcile loop. EKS does make
252+ // a CRD for eniconfigs but the default env var on the vpc-cni pod is ENABLE_POD_ENI=false. We will make an
253+ // assumption no CRs are ever created and leave the CRD to reduce complexity of this operation.
254+
255+ s .scope .Info ("Ensuring all resources for AWS VPC CNI in cluster are deleted" , "cluster-name" , s .scope .Name (), "cluster-namespace" , s .scope .Namespace ())
256+
257+ s .scope .Info ("Trying to delete AWS VPC CNI DaemonSet" , "cluster-name" , s .scope .Name (), "cluster-namespace" , s .scope .Namespace ())
258+ if err := s .deleteResource (ctx , remoteClient , types.NamespacedName {
259+ Namespace : awsNodeNamespace ,
260+ Name : awsNodeName ,
261+ }, & appsv1.DaemonSet {}); err != nil {
262+ return err
263+ }
243264
244- ds := & appsv1.DaemonSet {}
245- if err := remoteClient .Get (ctx , types.NamespacedName {Namespace : awsNodeNamespace , Name : awsNodeName }, ds ); err != nil {
246- if apierrors .IsNotFound (err ) {
247- s .scope .V (2 ).Info ("The aws-node DaemonSet is not found, not action" )
248- return nil
249- }
250- return fmt .Errorf ("getting aws-node daemonset: %w" , err )
265+ s .scope .Info ("Trying to delete AWS VPC CNI ServiceAccount" , "cluster-name" , s .scope .Name (), "cluster-namespace" , s .scope .Namespace ())
266+ if err := s .deleteResource (ctx , remoteClient , types.NamespacedName {
267+ Namespace : awsNodeNamespace ,
268+ Name : awsNodeName ,
269+ }, & corev1.ServiceAccount {}); err != nil {
270+ return err
251271 }
252272
253- s .scope .V (2 ).Info ("The aws-node DaemonSet found, deleting" )
254- if err := remoteClient .Delete (ctx , ds , & client.DeleteOptions {}); err != nil {
255- if apierrors .IsNotFound (err ) {
256- s .scope .V (2 ).Info ("The aws-node DaemonSet is not found, not deleted" )
257- return nil
258- }
259- return fmt .Errorf ("deleting aws-node DaemonSet: %w" , err )
273+ s .scope .Info ("Trying to delete AWS VPC CNI ClusterRoleBinding" , "cluster-name" , s .scope .Name (), "cluster-namespace" , s .scope .Namespace ())
274+ if err := s .deleteResource (ctx , remoteClient , types.NamespacedName {
275+ Namespace : string (meta .RESTScopeNameRoot ),
276+ Name : awsNodeName ,
277+ }, & rbacv1.ClusterRoleBinding {}); err != nil {
278+ return err
260279 }
280+
281+ s .scope .Info ("Trying to delete AWS VPC CNI ClusterRole" , "cluster-name" , s .scope .Name (), "cluster-namespace" , s .scope .Namespace ())
282+ if err := s .deleteResource (ctx , remoteClient , types.NamespacedName {
283+ Namespace : string (meta .RESTScopeNameRoot ),
284+ Name : awsNodeName ,
285+ }, & rbacv1.ClusterRole {}); err != nil {
286+ return err
287+ }
288+
261289 record .Eventf (s .scope .InfraCluster (), "DeletedVPCCNI" , "The AWS VPC CNI has been removed from the cluster. Ensure you enable a CNI via another mechanism" )
262290
263291 return nil
264292}
293+
294+ func (s * Service ) deleteResource (ctx context.Context , remoteClient client.Client , key client.ObjectKey , obj client.Object ) error {
295+ if err := remoteClient .Get (ctx , key , obj ); err != nil {
296+ if ! apierrors .IsNotFound (err ) {
297+ return fmt .Errorf ("deleting resource %s: %w" , key , err )
298+ }
299+ s .scope .V (2 ).Info (fmt .Sprintf ("resource %s was not found, no action" , key ))
300+ } else {
301+ // resource found, delete if no label or not managed by helm
302+ if val , ok := obj .GetLabels ()[konfig .ManagedbyLabelKey ]; ! ok || val != "Helm" {
303+ if err := remoteClient .Delete (ctx , obj , & client.DeleteOptions {}); err != nil {
304+ if ! apierrors .IsNotFound (err ) {
305+ return fmt .Errorf ("deleting %s: %w" , key , err )
306+ }
307+ s .scope .V (2 ).Info (fmt .Sprintf (
308+ "resource %s was not found, not deleted" , key ))
309+ } else {
310+ s .scope .V (2 ).Info (fmt .Sprintf ("resource %s was deleted" , key ))
311+ }
312+ } else {
313+ s .scope .V (2 ).Info (fmt .Sprintf ("resource %s is managed by helm, not deleted" , key ))
314+ }
315+ }
316+
317+ return nil
318+ }
0 commit comments