-
Notifications
You must be signed in to change notification settings - Fork 131
[CASCL-647] Add a subcommand to kubectl plugin to uninstall Karpenter #2424
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 7 commits
7448887
f208c10
df49976
101161d
d67a083
f580791
4c5bbde
a2f039c
8578ad3
a86f477
d459e1a
9881734
f8616f7
ea85b71
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| // Package clients provides shared AWS and Kubernetes client initialization | ||
| // for the Karpenter install and uninstall commands. | ||
| package clients | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| awssdk "github.com/aws/aws-sdk-go-v2/aws" | ||
| "github.com/aws/aws-sdk-go-v2/config" | ||
| "github.com/aws/aws-sdk-go-v2/service/cloudformation" | ||
| "github.com/aws/aws-sdk-go-v2/service/ec2" | ||
| "github.com/aws/aws-sdk-go-v2/service/eks" | ||
| "github.com/aws/aws-sdk-go-v2/service/sts" | ||
| karpawsv1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| "k8s.io/apimachinery/pkg/runtime" | ||
| "k8s.io/apimachinery/pkg/runtime/schema" | ||
| "k8s.io/cli-runtime/pkg/genericclioptions" | ||
| "k8s.io/client-go/kubernetes" | ||
| "k8s.io/client-go/kubernetes/scheme" | ||
| "k8s.io/client-go/rest" | ||
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| "sigs.k8s.io/controller-runtime/pkg/client/apiutil" | ||
| karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" | ||
|
|
||
| "github.com/DataDog/datadog-operator/cmd/kubectl-datadog/autoscaling/cluster/install/guess" | ||
| ) | ||
|
|
||
| // Clients holds all AWS and Kubernetes client instances needed for | ||
| // Karpenter installation and uninstallation operations. | ||
| type Clients struct { | ||
| // AWS clients | ||
| Config awssdk.Config | ||
| CloudFormation *cloudformation.Client | ||
| EC2 *ec2.Client | ||
| EKS *eks.Client | ||
| STS *sts.Client | ||
|
|
||
| // Kubernetes clients | ||
| K8sClient client.Client // controller-runtime client | ||
| K8sClientset *kubernetes.Clientset // typed Kubernetes client | ||
| } | ||
|
|
||
| // Build creates AWS and Kubernetes clients for Karpenter operations. | ||
| func Build(ctx context.Context, configFlags *genericclioptions.ConfigFlags, k8sClientset *kubernetes.Clientset) (*Clients, error) { | ||
| awsConfig, err := config.LoadDefaultConfig(ctx) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to load AWS config: %w", err) | ||
| } | ||
|
|
||
| sch := runtime.NewScheme() | ||
|
|
||
| if err = scheme.AddToScheme(sch); err != nil { | ||
| return nil, fmt.Errorf("failed to add base scheme: %w", err) | ||
| } | ||
|
|
||
| sch.AddKnownTypes( | ||
| schema.GroupVersion{Group: "karpenter.sh", Version: "v1"}, | ||
| &karpv1.NodePool{}, | ||
| &karpv1.NodePoolList{}, | ||
| ) | ||
| metav1.AddToGroupVersion(sch, schema.GroupVersion{Group: "karpenter.sh", Version: "v1"}) | ||
|
|
||
| sch.AddKnownTypes( | ||
| schema.GroupVersion{Group: "karpenter.k8s.aws", Version: "v1"}, | ||
| &karpawsv1.EC2NodeClass{}, | ||
| &karpawsv1.EC2NodeClassList{}, | ||
| ) | ||
| metav1.AddToGroupVersion(sch, schema.GroupVersion{Group: "karpenter.k8s.aws", Version: "v1"}) | ||
|
|
||
| restConfig, err := configFlags.ToRESTConfig() | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get REST config: %w", err) | ||
| } | ||
|
|
||
| httpClient, err := rest.HTTPClientFor(restConfig) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("unable to create http client: %w", err) | ||
| } | ||
|
|
||
| mapper, err := apiutil.NewDynamicRESTMapper(restConfig, httpClient) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("unable to instantiate mapper: %w", err) | ||
| } | ||
|
|
||
| k8sClient, err := client.New(restConfig, client.Options{ | ||
| Scheme: sch, | ||
| Mapper: mapper, | ||
| }) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to create Karpenter client: %w", err) | ||
| } | ||
|
|
||
| return &Clients{ | ||
| Config: awsConfig, | ||
| CloudFormation: cloudformation.NewFromConfig(awsConfig), | ||
| EC2: ec2.NewFromConfig(awsConfig), | ||
| EKS: eks.NewFromConfig(awsConfig), | ||
| STS: sts.NewFromConfig(awsConfig), | ||
| K8sClient: k8sClient, | ||
| K8sClientset: k8sClientset, | ||
| }, nil | ||
| } | ||
|
|
||
| // GetClusterNameFromKubeconfig extracts the EKS cluster name from the current kubeconfig context. | ||
| func GetClusterNameFromKubeconfig(ctx context.Context, configFlags *genericclioptions.ConfigFlags) (string, error) { | ||
| kubeRawConfig, err := configFlags.ToRawKubeConfigLoader().RawConfig() | ||
| if err != nil { | ||
| return "", fmt.Errorf("failed to get raw kubeconfig: %w", err) | ||
| } | ||
|
|
||
| kubeContext := "" | ||
| if configFlags.Context != nil { | ||
| kubeContext = *configFlags.Context | ||
| } | ||
|
|
||
| return guess.GetClusterNameFromKubeconfig(ctx, kubeRawConfig, kubeContext), nil | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,13 +7,16 @@ import ( | |
| "errors" | ||
| "fmt" | ||
| "log" | ||
| "os" | ||
| "time" | ||
|
|
||
| "helm.sh/helm/v3/pkg/action" | ||
| "helm.sh/helm/v3/pkg/chart/loader" | ||
| "helm.sh/helm/v3/pkg/cli" | ||
| "helm.sh/helm/v3/pkg/kube" | ||
| "helm.sh/helm/v3/pkg/release" | ||
| "helm.sh/helm/v3/pkg/storage/driver" | ||
| "k8s.io/cli-runtime/pkg/genericclioptions" | ||
| ) | ||
|
|
||
| func CreateOrUpgrade(ctx context.Context, ac *action.Configuration, releaseName, namespace, chartRef, version string, values map[string]any) error { | ||
|
|
@@ -125,3 +128,51 @@ func upgrade(ctx context.Context, ac *action.Configuration, releaseName, namespa | |
|
|
||
| return nil | ||
| } | ||
|
|
||
| func Uninstall(ctx context.Context, ac *action.Configuration, releaseName string) error { | ||
| exist, err := doesExist(ctx, ac, releaseName) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if !exist { | ||
| log.Printf("Helm release %s does not exist, skipping uninstallation.", releaseName) | ||
| return nil | ||
| } | ||
|
|
||
| log.Printf("Uninstalling Helm release %s…", releaseName) | ||
|
|
||
| uninstallAction := action.NewUninstall(ac) | ||
| uninstallAction.Wait = true | ||
| uninstallAction.Timeout = 30 * time.Minute | ||
|
|
||
| response, err := uninstallAction.Run(releaseName) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to uninstall Helm release %s: %w", releaseName, err) | ||
| } | ||
|
|
||
| log.Printf("Uninstalled Helm release %s.", response.Release.Name) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // NewActionConfig creates a new Helm action configuration from kubeconfig flags. | ||
| func NewActionConfig(configFlags *genericclioptions.ConfigFlags, namespace string) (*action.Configuration, error) { | ||
| kubeConfig := "" | ||
| if configFlags.KubeConfig != nil { | ||
| kubeConfig = *configFlags.KubeConfig | ||
| } | ||
| kubeContext := "" | ||
| if configFlags.Context != nil { | ||
| kubeContext = *configFlags.Context | ||
| } | ||
|
|
||
| restClientGetter := kube.GetConfig(kubeConfig, kubeContext, namespace) | ||
|
||
| actionConfig := new(action.Configuration) | ||
|
|
||
| if err := actionConfig.Init(restClientGetter, namespace, os.Getenv("HELM_DRIVER"), log.Printf); err != nil { | ||
| return nil, fmt.Errorf("failed to initialize Helm configuration: %w", err) | ||
| } | ||
|
|
||
| return actionConfig, nil | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -9,7 +9,7 @@ import ( | |
| "sigs.k8s.io/controller-runtime/pkg/client" | ||
| ) | ||
|
|
||
| func createOrUpdate(ctx context.Context, cli client.Client, object client.Object) error { | ||
| func CreateOrUpdate(ctx context.Context, cli client.Client, object client.Object) error { | ||
| resourceVersion, err := getResourceVersion(ctx, cli, object) | ||
| if err != nil { | ||
| return err | ||
|
|
@@ -58,3 +58,59 @@ func update(ctx context.Context, cli client.Client, object client.Object) error | |
|
|
||
| return nil | ||
| } | ||
|
|
||
| func Delete(ctx context.Context, cli client.Client, object client.Object) error { | ||
| log.Printf("Deleting %s %s…", object.GetObjectKind().GroupVersionKind().Kind, object.GetName()) | ||
|
|
||
| if err := cli.Delete(ctx, object); err != nil { | ||
| if apierrors.IsNotFound(err) { | ||
| log.Printf("%s %s not found, skipping deletion.", object.GetObjectKind().GroupVersionKind().Kind, object.GetName()) | ||
| return nil | ||
| } | ||
|
Comment on lines
+66
to
+69
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it a good idea to mask 404? E.g. update bubbles it up.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This is inside the The question might then be: why would we try to delete an object that doesn’t exist? |
||
| return fmt.Errorf("failed to delete %s %s: %w", object.GetObjectKind().GroupVersionKind().Kind, object.GetName(), err) | ||
| } | ||
|
|
||
| log.Printf("Deleted %s %s.", object.GetObjectKind().GroupVersionKind().Kind, object.GetName()) | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func DeleteAllWithLabel(ctx context.Context, cli client.Client, list client.ObjectList, labelSelector client.MatchingLabels) error { | ||
| gvk := list.GetObjectKind().GroupVersionKind() | ||
| kind := gvk.Kind | ||
|
|
||
| log.Printf("Listing %s resources with labels %v…", kind, labelSelector) | ||
|
|
||
| if err := cli.List(ctx, list, labelSelector); err != nil { | ||
| return fmt.Errorf("failed to list %s resources: %w", kind, err) | ||
| } | ||
|
|
||
| items, err := extractListItems(list) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| if len(items) == 0 { | ||
| log.Printf("No %s resources found with labels %v, skipping deletion.", kind, labelSelector) | ||
| return nil | ||
| } | ||
|
|
||
| log.Printf("Found %d %s resource(s) to delete.", len(items), kind) | ||
|
|
||
| for _, item := range items { | ||
| if err := Delete(ctx, cli, item); err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func extractListItems(list client.ObjectList) ([]client.Object, error) { | ||
| switch v := list.(type) { | ||
| case interface{ GetItems() []client.Object }: | ||
| return v.GetItems(), nil | ||
| default: | ||
| return nil, fmt.Errorf("unsupported list type: %T", list) | ||
L3n41c marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: could use slices.DeleteFunc:
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That’s much better! Thanks!
Implemented in f8616f7#diff-75e34b5bb02db7ae8e1635b717b48c7090c87a7f1a2d2a63b955847f3db9acd3