diff --git a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/eks-cluster-class.yaml b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/eks-cluster-class.yaml index 661f22afe..24ce48109 100644 --- a/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/eks-cluster-class.yaml +++ b/charts/cluster-api-runtime-extensions-nutanix/defaultclusterclasses/eks-cluster-class.yaml @@ -31,7 +31,7 @@ spec: bootstrap: ref: apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: EKSConfigTemplate + kind: NodeadmConfigTemplate name: eks-quick-start-worker-configtemplate infrastructure: ref: @@ -73,11 +73,18 @@ metadata: spec: template: spec: + ami: + eksLookupType: AmazonLinux2023 + cloudInit: + insecureSkipSecretsManager: true + instanceMetadataOptions: + httpPutResponseHopLimit: 2 + httpTokens: required instanceType: PLACEHOLDER sshKeyName: "" --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: EKSConfigTemplate +kind: NodeadmConfigTemplate metadata: labels: cluster.x-k8s.io/provider: eks diff --git a/hack/examples/bases/eks/clusterclass/clusterclass.yaml b/hack/examples/bases/eks/clusterclass/clusterclass.yaml index 7033e0363..3950c27fe 100644 --- a/hack/examples/bases/eks/clusterclass/clusterclass.yaml +++ b/hack/examples/bases/eks/clusterclass/clusterclass.yaml @@ -24,7 +24,7 @@ spec: ref: name: "quick-start-worker-configtemplate" apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 - kind: EKSConfigTemplate + kind: NodeadmConfigTemplate infrastructure: ref: name: "quick-start-worker-machinetemplate" @@ -58,10 +58,17 @@ metadata: name: "quick-start-worker-machinetemplate" spec: template: - spec: {} + spec: + cloudInit: + insecureSkipSecretsManager: true + ami: + eksLookupType: AmazonLinux2023 + instanceMetadataOptions: + httpTokens: required + httpPutResponseHopLimit: 2 --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta2 -kind: EKSConfigTemplate +kind: NodeadmConfigTemplate metadata: name: "quick-start-worker-configtemplate" spec: diff --git a/hack/examples/overlays/clusterclasses/eks/kustomizeconfig.yaml b/hack/examples/overlays/clusterclasses/eks/kustomizeconfig.yaml index 2f537d92c..60d50476f 100644 --- a/hack/examples/overlays/clusterclasses/eks/kustomizeconfig.yaml +++ b/hack/examples/overlays/clusterclasses/eks/kustomizeconfig.yaml @@ -16,7 +16,7 @@ nameReference: fieldSpecs: - kind: ClusterClass path: spec/controlPlane/ref/name - - kind: EKSConfigTemplate + - kind: NodeadmConfigTemplate fieldSpecs: - kind: ClusterClass path: spec/workers/machineDeployments/template/bootstrap/ref/name diff --git a/pkg/handlers/eks/mutation/testutils/request.go b/pkg/handlers/eks/mutation/testutils/request.go index 7dce21c79..cc9a9d6b7 100644 --- a/pkg/handlers/eks/mutation/testutils/request.go +++ b/pkg/handlers/eks/mutation/testutils/request.go @@ -47,14 +47,14 @@ func NewEKSControlPlaneRequestItem( ) } -func NewEKSConfigTemplateRequestItem( +func NewNodeadmConfigTemplateRequestItem( uid types.UID, - existingSpec ...eksbootstrapv1.EKSConfigTemplateSpec, + existingSpec ...eksbootstrapv1.NodeadmConfigTemplateSpec, ) runtimehooksv1.GeneratePatchesRequestItem { - eksConfigTemplate := &eksbootstrapv1.EKSConfigTemplate{ + nodeadmConfigTemplate := &eksbootstrapv1.NodeadmConfigTemplate{ TypeMeta: metav1.TypeMeta{ APIVersion: eksbootstrapv1.GroupVersion.String(), - Kind: "EKSConfigTemplate", + Kind: "NodeadmConfigTemplate", }, } @@ -62,13 +62,13 @@ func NewEKSConfigTemplateRequestItem( case 0: // Do nothing. case 1: - eksConfigTemplate.Spec = existingSpec[0] + nodeadmConfigTemplate.Spec = existingSpec[0] default: panic("can only take at most one existing spec") } return request.NewRequestItem( - eksConfigTemplate, + nodeadmConfigTemplate, &runtimehooksv1.HolderReference{ Kind: "MachineDeployment", FieldPath: "spec.template.spec.bootstrap.configRef", diff --git a/pkg/handlers/generic/mutation/ntp/inject.go b/pkg/handlers/generic/mutation/ntp/inject.go index 47ec3058f..df376d666 100644 --- a/pkg/handlers/generic/mutation/ntp/inject.go +++ b/pkg/handlers/generic/mutation/ntp/inject.go @@ -124,12 +124,12 @@ func (h *ntpPatchHandler) Mutate( if err := patches.MutateIfApplicable( obj, vars, &holderRef, - selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "EKSConfigTemplate"), log, - func(obj *eksbootstrapv1.EKSConfigTemplate) error { + selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "NodeadmConfigTemplate"), log, + func(obj *eksbootstrapv1.NodeadmConfigTemplate) error { log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), "patchedObjectName", client.ObjectKeyFromObject(obj), - ).Info("setting users in worker node EKS config template") + ).Info("setting users in worker node NodeadmConfig template") obj.Spec.Template.Spec.NTP = &eksbootstrapv1.NTP{ Enabled: ptr.To(true), Servers: ntp.Servers, diff --git a/pkg/handlers/generic/mutation/ntp/inject_test.go b/pkg/handlers/generic/mutation/ntp/inject_test.go index d367c1361..79f0be7d8 100644 --- a/pkg/handlers/generic/mutation/ntp/inject_test.go +++ b/pkg/handlers/generic/mutation/ntp/inject_test.go @@ -160,8 +160,8 @@ var _ = Describe("Generate NTP patches", func() { }, }, { - Name: "NTP configuration is set for worker nodes with single server for EKSConfigTemplate", - RequestItem: testutils.NewEKSConfigTemplateRequestItem(""), + Name: "NTP configuration is set for worker nodes with single server for NodeadmConfigTemplate", + RequestItem: testutils.NewNodeadmConfigTemplateRequestItem(""), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{ { Operation: "add", diff --git a/pkg/handlers/generic/mutation/taints/inject_worker.go b/pkg/handlers/generic/mutation/taints/inject_worker.go index 22509a618..ba0218743 100644 --- a/pkg/handlers/generic/mutation/taints/inject_worker.go +++ b/pkg/handlers/generic/mutation/taints/inject_worker.go @@ -5,12 +5,14 @@ package taints import ( "context" + "fmt" "strings" "github.com/samber/lo" v1 "k8s.io/api/core/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/ptr" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" ctrl "sigs.k8s.io/controller-runtime" @@ -48,6 +50,10 @@ func newTaintsWorkerPatchHandler( } } +type KubeletRegisterOptions struct { + RegisterWithTaints []v1.Taint `json:"registerWithTaints,omitempty"` +} + func (h *taintsWorkerPatchHandler) Mutate( ctx context.Context, obj *unstructured.Unstructured, @@ -103,25 +109,16 @@ func (h *taintsWorkerPatchHandler) Mutate( if err := patches.MutateIfApplicable( obj, vars, &holderRef, - selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "EKSConfigTemplate"), log, - func(obj *eksbootstrapv1.EKSConfigTemplate) error { + selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "NodeadmConfigTemplate"), log, + func(obj *eksbootstrapv1.NodeadmConfigTemplate) error { log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("adding taints to worker node EKS config template") - if obj.Spec.Template.Spec.KubeletExtraArgs == nil { - obj.Spec.Template.Spec.KubeletExtraArgs = make(map[string]string, 1) - } - - existingTaintsFlagValue := obj.Spec.Template.Spec.KubeletExtraArgs["register-with-taints"] - - newTaintsFlagValue := toEKSConfigTaints(taintsVar) - - if existingTaintsFlagValue != "" { - newTaintsFlagValue = existingTaintsFlagValue + "," + newTaintsFlagValue - } - - obj.Spec.Template.Spec.KubeletExtraArgs["register-with-taints"] = newTaintsFlagValue + ).Info("adding taints to worker NodeadmConfig template") + newTaints := toEKSConfigTaints(taintsVar) + kubeletOptions := ptr.Deref(obj.Spec.Template.Spec.Kubelet, eksbootstrapv1.KubeletOptions{}) + kubeletOptions.Flags = append(kubeletOptions.Flags, fmt.Sprintf("--register-with-taints=%s", newTaints)) + obj.Spec.Template.Spec.Kubelet = &kubeletOptions return nil }); err != nil { return err diff --git a/pkg/handlers/generic/mutation/taints/inject_worker_test.go b/pkg/handlers/generic/mutation/taints/inject_worker_test.go index 081a8bcfd..088162af9 100644 --- a/pkg/handlers/generic/mutation/taints/inject_worker_test.go +++ b/pkg/handlers/generic/mutation/taints/inject_worker_test.go @@ -13,6 +13,7 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + eksbootstrapv1 "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/external/sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/api/v1beta2" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" @@ -59,7 +60,7 @@ var _ = Describe("Generate taints patches for Worker", func() { }}, }, { - Name: "taints for workers set for EKSConfigTemplate", + Name: "taints for workers set for NodeadmConfigTemplate", Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( v1alpha1.WorkerConfigVariableName, @@ -77,15 +78,88 @@ var _ = Describe("Generate taints patches for Worker", func() { }, ), }, - RequestItem: testutils.NewEKSConfigTemplateRequestItem(""), + RequestItem: testutils.NewNodeadmConfigTemplateRequestItem(""), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ Operation: "add", - Path: "/spec/template/spec/kubeletExtraArgs", + Path: "/spec/template/spec/kubelet", ValueMatcher: gomega.HaveKeyWithValue( - "register-with-taints", "key=value:NoExecute", + "flags", + gomega.ContainElement("--register-with-taints=key=value:NoExecute"), ), }}, }, + { + Name: "taints for workers set for NodeadmConfigTemplate with existing flags argument", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.WorkerConfigVariableName, + []v1alpha1.Taint{{ + Key: "key", + Effect: v1alpha1.TaintEffectNoExecute, + Value: "value", + }}, + VariableName, + ), + capitest.VariableWithValue( + "builtin", + apiextensionsv1.JSON{ + Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`), + }, + ), + }, + RequestItem: testutils.NewNodeadmConfigTemplateRequestItem("", eksbootstrapv1.NodeadmConfigTemplateSpec{ + Template: eksbootstrapv1.NodeadmConfigTemplateResource{ + Spec: eksbootstrapv1.NodeadmConfigSpec{ + Kubelet: &eksbootstrapv1.KubeletOptions{ + Flags: []string{ + "--max-pods=110", + }, + }, + }, + }, + }), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ + Operation: "add", + Path: "/spec/template/spec/kubelet/flags/1", + ValueMatcher: gomega.Equal("--register-with-taints=key=value:NoExecute"), + }}, + }, + { + Name: "taints for workers set for NodeadmConfigTemplate with existing flags with register-with-taints ", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.WorkerConfigVariableName, + []v1alpha1.Taint{{ + Key: "key", + Effect: v1alpha1.TaintEffectNoExecute, + Value: "value", + }}, + VariableName, + ), + capitest.VariableWithValue( + "builtin", + apiextensionsv1.JSON{ + Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`), + }, + ), + }, + RequestItem: testutils.NewNodeadmConfigTemplateRequestItem("", eksbootstrapv1.NodeadmConfigTemplateSpec{ + Template: eksbootstrapv1.NodeadmConfigTemplateResource{ + Spec: eksbootstrapv1.NodeadmConfigSpec{ + Kubelet: &eksbootstrapv1.KubeletOptions{ + Flags: []string{ + "--register-with-taints=key1=value1:NoSchedule", + }, + }, + }, + }, + }), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ + Operation: "add", + Path: "/spec/template/spec/kubelet/flags/1", + ValueMatcher: gomega.Equal("--register-with-taints=key=value:NoExecute"), + }}, + }, } // create test node for each case diff --git a/pkg/handlers/generic/mutation/users/inject.go b/pkg/handlers/generic/mutation/users/inject.go index 11ed5dae6..ac663c822 100644 --- a/pkg/handlers/generic/mutation/users/inject.go +++ b/pkg/handlers/generic/mutation/users/inject.go @@ -115,12 +115,12 @@ func (h *usersPatchHandler) Mutate( if err := patches.MutateIfApplicable( obj, vars, &holderRef, - selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "EKSConfigTemplate"), log, - func(obj *eksbootstrapv1.EKSConfigTemplate) error { + selectors.WorkersConfigTemplateSelector(eksbootstrapv1.GroupVersion.String(), "NodeadmConfigTemplate"), log, + func(obj *eksbootstrapv1.NodeadmConfigTemplate) error { log.WithValues( "patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(), "patchedObjectName", ctrlclient.ObjectKeyFromObject(obj), - ).Info("setting users in worker node EKS config template") + ).Info("setting users in worker node NodeadmConfig template") eksBootstrapUsers := make([]eksbootstrapv1.User, 0, len(bootstrapUsers)) for _, user := range bootstrapUsers { var passwdFrom *eksbootstrapv1.PasswdSource diff --git a/pkg/handlers/generic/mutation/users/inject_test.go b/pkg/handlers/generic/mutation/users/inject_test.go index d327ff59a..4e23de61e 100644 --- a/pkg/handlers/generic/mutation/users/inject_test.go +++ b/pkg/handlers/generic/mutation/users/inject_test.go @@ -194,7 +194,7 @@ var _ = Describe("Generate Users patches", func() { }}, }, { - Name: "users set for EKSConfigTemplate generic worker", + Name: "users set for NodeadmConfigTemplate generic worker", Vars: []runtimehooksv1.Variable{ capitest.VariableWithValue( v1alpha1.ClusterConfigVariableName, @@ -210,7 +210,7 @@ var _ = Describe("Generate Users patches", func() { }, ), }, - RequestItem: testutils.NewEKSConfigTemplateRequestItem(""), + RequestItem: testutils.NewNodeadmConfigTemplateRequestItem(""), ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ Operation: "add", Path: "/spec/template/spec/users",