diff --git a/CHANGELOG.md b/CHANGELOG.md index 67895d3b..f3f4b940 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Create karpenter custom resources in workload clusters. + ## [0.20.0] - 2025-06-23 ### Changed diff --git a/Makefile.custom.mk b/Makefile.custom.mk index ac676670..d09d3450 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -26,14 +26,8 @@ crds: controller-gen ## Generate CustomResourceDefinition. generate: controller-gen crds ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. go generate ./... $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... +# We need to run goimports after controller-gen to avoid CI complains about goimports in the generated files + @go run golang.org/x/tools/cmd/goimports -w ./api/v1alpha1 .PHONY: create-acceptance-cluster create-acceptance-cluster: kind @@ -127,7 +121,7 @@ coverage-html: test-unit CONTROLLER_GEN = $(shell pwd)/bin/controller-gen .PHONY: controller-gen controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5) + $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.18.0) ENVTEST = $(shell pwd)/bin/setup-envtest .PHONY: envtest diff --git a/Makefile.gen.app.mk b/Makefile.gen.app.mk index 3f8a89c9..26a4880a 100644 --- a/Makefile.gen.app.mk +++ b/Makefile.gen.app.mk @@ -22,7 +22,6 @@ lint-chart: check-env ## Runs ct against the default chart. rm -rf /tmp/$(APPLICATION)-test mkdir -p /tmp/$(APPLICATION)-test/helm cp -a ./helm/$(APPLICATION) /tmp/$(APPLICATION)-test/helm/ - architect helm template --dir /tmp/$(APPLICATION)-test/helm/$(APPLICATION) docker run -it --rm -v /tmp/$(APPLICATION)-test:/wd --workdir=/wd --name ct $(IMAGE) ct lint --validate-maintainers=false --charts="helm/$(APPLICATION)" rm -rf /tmp/$(APPLICATION)-test diff --git a/Makefile.gen.go.mk b/Makefile.gen.go.mk index a186ca71..b31216a2 100644 --- a/Makefile.gen.go.mk +++ b/Makefile.gen.go.mk @@ -11,7 +11,7 @@ GITSHA1 := $(shell git rev-parse --verify HEAD) MODULE := $(shell go list -m) OS := $(shell go env GOOS) SOURCES := $(shell find . -name '*.go') -VERSION := $(shell architect project version) + ifeq ($(OS), linux) EXTLDFLAGS := -static endif diff --git a/api/v1alpha1/duration.go b/api/v1alpha1/duration.go new file mode 100644 index 00000000..b30d2a8a --- /dev/null +++ b/api/v1alpha1/duration.go @@ -0,0 +1,80 @@ +package v1alpha1 + +import ( + "encoding/json" + "fmt" + "slices" + "time" + + "github.com/samber/lo" +) + +const Never = "Never" + +// NillableDuration is a wrapper around time.Duration which supports correct +// marshaling to YAML and JSON. It uses the value "Never" to signify +// that the duration is disabled and sets the inner duration as nil +type NillableDuration struct { + *time.Duration + + // Raw is used to ensure we remarshal the NillableDuration in the same format it was specified. + // This ensures tools like Flux and ArgoCD don't mistakenly detect drift due to our conversion webhooks. + Raw []byte `hash:"ignore"` +} + +func MustParseNillableDuration(val string) NillableDuration { + nd := NillableDuration{} + // Use %q instead of %s to ensure that we unmarshal the value as a string and not an int + lo.Must0(json.Unmarshal([]byte(fmt.Sprintf("%q", val)), &nd)) + return nd +} + +// UnmarshalJSON implements the json.Unmarshaller interface. +func (d *NillableDuration) UnmarshalJSON(b []byte) error { + var str string + err := json.Unmarshal(b, &str) + if err != nil { + return err + } + if str == Never { + return nil + } + pd, err := time.ParseDuration(str) + if err != nil { + return err + } + d.Raw = slices.Clone(b) + d.Duration = &pd + return nil +} + +// MarshalJSON implements the json.Marshaler interface. +func (d NillableDuration) MarshalJSON() ([]byte, error) { + if d.Raw != nil { + return d.Raw, nil + } + if d.Duration != nil { + return json.Marshal(d.Duration.String()) + } + return json.Marshal(Never) +} + +// ToUnstructured implements the value.UnstructuredConverter interface. +func (d NillableDuration) ToUnstructured() interface{} { + if d.Raw != nil { + // Decode the JSON bytes to get the actual string value + var str string + if err := json.Unmarshal(d.Raw, &str); err == nil { + return str + } + // Fallback to string conversion if unmarshal fails + if d.Duration != nil { + return d.Duration.String() + } + return Never + } + if d.Duration != nil { + return d.Duration.String() + } + return Never +} diff --git a/api/v1alpha1/ec2nodeclass.go b/api/v1alpha1/ec2nodeclass.go new file mode 100644 index 00000000..f5f70322 --- /dev/null +++ b/api/v1alpha1/ec2nodeclass.go @@ -0,0 +1,451 @@ +package v1alpha1 + +import ( + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. +// This will contain configuration necessary to launch instances in AWS. +type EC2NodeClassSpec struct { + // SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="subnetSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term",rule="!self.all(x, has(x.id) && has(x.tags))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SubnetSelectorTerms []SubnetSelectorTerm `json:"subnetSelectorTerms" hash:"ignore"` + // SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="securityGroupSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))" + // +kubebuilder:validation:XValidation:message="'name' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term",rule="!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))" + // +kubebuilder:validation:MaxItems:=30 + // +required + SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` + // CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + // determine the set of eligible capacity reservations. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))" + // +kubebuilder:validation:MaxItems:=30 + // +optional + CapacityReservationSelectorTerms []CapacityReservationSelectorTerm `json:"capacityReservationSelectorTerms" hash:"ignore"` + // AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + // +optional + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` + // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameter']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameter))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))" + // +kubebuilder:validation:XValidation:message="'alias' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms",rule="!(self.exists(x, has(x.alias)) && self.size() != 1)" + // +kubebuilder:validation:MinItems:=1 + // +kubebuilder:validation:MaxItems:=30 + // +required + AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms" hash:"ignore"` + // AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + // This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + // family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + // alias is specified, this field is required. + // NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + // the AMIFamily() helper function + // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Custom,Windows2019,Windows2022} + // +optional + AMIFamily *string `json:"amiFamily,omitempty" hash:"ignore"` + // UserData to be applied to the provisioned nodes. + // It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + // this UserData to ensure nodes are being provisioned with the correct configuration. + // +optional + UserData *string `json:"userData,omitempty"` + // Role is the AWS identity that nodes use. This field is immutable. + // This field is mutually exclusive from instanceProfile. + // Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + // This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + // for the old instance profiles on an update. + // +kubebuilder:validation:XValidation:rule="self != ''",message="role cannot be empty" + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="immutable field changed" + // +optional + Role string `json:"role,omitempty"` + // InstanceProfile is the AWS entity that instances use. + // This field is mutually exclusive from role. + // The instance profile should already have a role assigned to it that Karpenter + // has PassRole permission on for instance launch using this instanceProfile to succeed. + // +kubebuilder:validation:XValidation:rule="self != ''",message="instanceProfile cannot be empty" + // +optional + InstanceProfile *string `json:"instanceProfile,omitempty"` + // Tags to be applied on ec2 resources like instances and launch templates. + // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching eks:eks-cluster-name",rule="self.all(k, k !='eks:eks-cluster-name')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodeclaim",rule="self.all(k, k !='karpenter.sh/nodeclaim')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass",rule="self.all(k, k !='karpenter.k8s.aws/ec2nodeclass')" + // +optional + Tags map[string]string `json:"tags,omitempty"` + // Kubelet defines args to be used when configuring kubelet on provisioned nodes. + // They are a subset of the upstream types, recognizing not all options may be supported. + // Wherever possible, the types and names should reflect the upstream kubelet types. + // +kubebuilder:validation:XValidation:message="imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent",rule="has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true" + // +kubebuilder:validation:XValidation:message="evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod",rule="has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true" + // +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true" + // +optional + Kubelet *KubeletConfiguration `json:"kubelet,omitempty"` + // BlockDeviceMappings to be applied to provisioned nodes. + // +kubebuilder:validation:XValidation:message="must have only one blockDeviceMappings with rootVolume",rule="self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1" + // +kubebuilder:validation:MaxItems:=50 + // +optional + BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` + // InstanceStorePolicy specifies how to handle instance-store disks. + // +optional + InstanceStorePolicy *InstanceStorePolicy `json:"instanceStorePolicy,omitempty"` + // DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + // +optional + DetailedMonitoring *bool `json:"detailedMonitoring,omitempty"` + // MetadataOptions for the generated launch template of provisioned nodes. + // + // This specifies the exposure of the Instance Metadata Service to + // provisioned EC2 nodes. For more information, + // see Instance Metadata and User Data + // (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + // in the Amazon Elastic Compute Cloud User Guide. + // + // Refer to recommended, security best practices + // (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + // for limiting exposure of Instance Metadata and User Data to pods. + // If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + // disabled, with httpPutResponseLimit of 1, and with httpTokens + // required. + // +kubebuilder:default={"httpEndpoint":"enabled","httpProtocolIPv6":"disabled","httpPutResponseHopLimit":1,"httpTokens":"required"} + // +optional + MetadataOptions *MetadataOptions `json:"metadataOptions,omitempty"` + // Context is a Reserved field in EC2 APIs + // https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + // +optional + Context *string `json:"context,omitempty"` +} + +// SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SubnetSelectorTerm struct { + // Tags is a map of key/value tags used to select subnets + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the subnet id in EC2 + // +kubebuilder:validation:Pattern="subnet-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` +} + +// SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type SecurityGroupSelectorTerm struct { + // Tags is a map of key/value tags used to select security groups. + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the security group id in EC2 + // +kubebuilder:validation:Pattern:="sg-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the security group name in EC2. + // This value is the name field, which is different from the name tag. + Name string `json:"name,omitempty"` +} + +type CapacityReservationSelectorTerm struct { + // Tags is a map of key/value tags used to select capacity reservations. + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the capacity reservation id in EC2 + // +kubebuilder:validation:Pattern:="^cr-[0-9a-z]+$" + // +optional + ID string `json:"id,omitempty"` + // Owner is the owner id for the ami. + // +kubebuilder:validation:Pattern:="^[0-9]{12}$" + // +optional + OwnerID string `json:"ownerID,omitempty"` +} + +// AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. +// If multiple fields are used for selection, the requirements are ANDed. +type AMISelectorTerm struct { + // Alias specifies which EKS optimized AMI to select. + // Each alias consists of a family and an AMI version, specified as "family@version". + // Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + // The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + // The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + // Note: The Windows families do **not** support version pinning, and only latest may be used. + // +kubebuilder:validation:XValidation:message="'alias' is improperly formatted, must match the format 'family@version'",rule="self.matches('^[a-zA-Z0-9]+@.+$')" + // +kubebuilder:validation:XValidation:message="family is not supported, must be one of the following: 'al2', 'al2023', 'bottlerocket', 'windows2019', 'windows2022'",rule="self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022']" + // +kubebuilder:validation:XValidation:message="windows families may only specify version 'latest'",rule="self.split('@')[0] in ['windows2019','windows2022'] ? self.split('@')[1] == 'latest' : true" + // +kubebuilder:validation:MaxLength=30 + // +optional + Alias string `json:"alias,omitempty"` + // Tags is a map of key/value tags used to select amis. + // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 + // +optional + Tags map[string]string `json:"tags,omitempty"` + // ID is the ami id in EC2 + // +kubebuilder:validation:Pattern:="ami-[0-9a-z]+" + // +optional + ID string `json:"id,omitempty"` + // Name is the ami name in EC2. + // This value is the name field, which is different from the name tag. + // +optional + Name string `json:"name,omitempty"` + // Owner is the owner for the ami. + // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + // +optional + Owner string `json:"owner,omitempty"` + // SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. + // +optional + SSMParameter string `json:"ssmParameter,omitempty"` +} + +// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. +// They are a subset of the upstream types, recognizing not all options may be supported. +// Wherever possible, the types and names should reflect the upstream kubelet types. +// https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration +// https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53 +type KubeletConfiguration struct { + // clusterDNS is a list of IP addresses for the cluster DNS server. + // Note that not all providers may use all addresses. + // +optional + ClusterDNS []string `json:"clusterDNS,omitempty"` + // MaxPods is an override for the maximum number of pods that can run on + // a worker node instance. + // +kubebuilder:validation:Minimum:=0 + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + // PodsPerCore is an override for the number of pods that can run on a worker node + // instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + // MaxPods is a lower value, that value will be used. + // +kubebuilder:validation:Minimum:=0 + // +optional + PodsPerCore *int32 `json:"podsPerCore,omitempty"` + // SystemReserved contains resources reserved for OS system daemons and kernel memory. + // +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // KubeReserved contains resources reserved for Kubernetes system components. + // +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // EvictionHard is the map of signal names to quantities that define hard eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionHard map[string]string `json:"evictionHard,omitempty"` + // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoftGracePeriod map[string]metav1.Duration `json:"evictionSoftGracePeriod,omitempty"` + // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + // response to soft eviction thresholds being met. + // +optional + EvictionMaxPodGracePeriod *int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // ImageGCHighThresholdPercent is the percent of disk usage after which image + // garbage collection is always run. The percent is calculated by dividing this + // field value by 100, so this field must be between 0 and 100, inclusive. + // When specified, the value must be greater than ImageGCLowThresholdPercent. + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` + // ImageGCLowThresholdPercent is the percent of disk usage before which image + // garbage collection is never run. Lowest disk usage to garbage collect to. + // The percent is calculated by dividing this field value by 100, + // so the field value must be between 0 and 100, inclusive. + // When specified, the value must be less than imageGCHighThresholdPercent + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` + // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + // +optional + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` +} + +// MetadataOptions contains parameters for specifying the exposure of the +// Instance Metadata Service to provisioned EC2 nodes. +type MetadataOptions struct { + // HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + // nodes. If metadata options is non-nil, but this parameter is not specified, + // the default state is "enabled". + // + // If you specify a value of "disabled", instance metadata will not be accessible + // on the node. + // +kubebuilder:default=enabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPEndpoint *string `json:"httpEndpoint,omitempty"` + // HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + // service on provisioned nodes. If metadata options is non-nil, but this parameter + // is not specified, the default state is "disabled". + // +kubebuilder:default=disabled + // +kubebuilder:validation:Enum:={enabled,disabled} + // +optional + HTTPProtocolIPv6 *string `json:"httpProtocolIPv6,omitempty"` + // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + // instance metadata requests. The larger the number, the further instance + // metadata requests can travel. Possible values are integers from 1 to 64. + // If metadata options is non-nil, but this parameter is not specified, the + // default value is 1. + // +kubebuilder:default=1 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=64 + // +optional + HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"` + // HTTPTokens determines the state of token usage for instance metadata + // requests. If metadata options is non-nil, but this parameter is not + // specified, the default state is "required". + // + // If the state is optional, one can choose to retrieve instance metadata with + // or without a signed token header on the request. If one retrieves the IAM + // role credentials without a token, the version 1.0 role credentials are + // returned. If one retrieves the IAM role credentials using a valid signed + // token, the version 2.0 role credentials are returned. + // + // If the state is "required", one must send a signed token header with any + // instance metadata retrieval requests. In this state, retrieving the IAM + // role credentials always returns the version 2.0 credentials; the version + // 1.0 credentials are not available. + // +kubebuilder:default=required + // +kubebuilder:validation:Enum:={required,optional} + // +optional + HTTPTokens *string `json:"httpTokens,omitempty"` +} + +type BlockDeviceMapping struct { + // The device name (for example, /dev/sdh or xvdh). + // +optional + DeviceName *string `json:"deviceName,omitempty"` + // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + // +kubebuilder:validation:XValidation:message="snapshotID or volumeSize must be defined",rule="has(self.snapshotID) || has(self.volumeSize)" + // +kubebuilder:validation:XValidation:message="snapshotID must be set when volumeInitializationRate is set",rule="!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '')" + // +optional + EBS *BlockDevice `json:"ebs,omitempty"` + // RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + // configure at most one root volume in BlockDeviceMappings. + // +optional + RootVolume bool `json:"rootVolume,omitempty"` +} + +type BlockDevice struct { + // DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + // +optional + DeleteOnTermination *bool `json:"deleteOnTermination,omitempty"` + // Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + // be attached to instances that support Amazon EBS encryption. If you are creating + // a volume from a snapshot, you can't specify an encryption value. + // +optional + Encrypted *bool `json:"encrypted,omitempty"` + // IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + // this represents the number of IOPS that are provisioned for the volume. For + // gp2 volumes, this represents the baseline performance of the volume and the + // rate at which the volume accumulates I/O credits for bursting. + // + // The following are the supported values for each volume type: + // + // * gp3: 3,000-16,000 IOPS + // + // * io1: 100-64,000 IOPS + // + // * io2: 100-64,000 IOPS + // + // For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + // on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + // Other instance families guarantee performance up to 32,000 IOPS. + // + // This parameter is supported for io1, io2, and gp3 volumes only. This parameter + // is not supported for gp2, st1, sc1, or standard volumes. + // +optional + IOPS *int64 `json:"iops,omitempty"` + // Identifier (key ID, key alias, key ARN, or alias ARN) of the customer managed KMS key to use for EBS encryption. + // +optional + KMSKeyID *string `json:"kmsKeyID,omitempty"` + // SnapshotID is the ID of an EBS snapshot + // +optional + SnapshotID *string `json:"snapshotID,omitempty"` + // Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + // Valid Range: Minimum value of 125. Maximum value of 1000. + // +optional + Throughput *int64 `json:"throughput,omitempty"` + // VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + // in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + // initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + // predictable and consistent rate after creation. Only allowed if SnapshotID is set. + // Valid Range: Minimum value of 100. Maximum value of 300. + // +kubebuilder:validation:Minimum:=100 + // +kubebuilder:validation:Maximum:=300 + // +optional + VolumeInitializationRate *int32 `json:"volumeInitializationRate,omitempty"` + // VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + // a volume size. The following are the supported volumes sizes for each volume + // type: + // + // * gp2 and gp3: 1-16,384 + // + // * io1 and io2: 4-16,384 + // + // * st1 and sc1: 125-16,384 + // + // * standard: 1-1,024 + // + TODO: Add the CEL resources.quantity type after k8s 1.29 + // + https://github.com/kubernetes/apiserver/commit/b137c256373aec1c5d5810afbabb8932a19ecd2a#diff-838176caa5882465c9d6061febd456397a3e2b40fb423ed36f0cabb1847ecb4dR190 + // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" + // +kubebuilder:validation:Schemaless + // +kubebuilder:validation:Type:=string + // +optional + VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` + // VolumeType of the block device. + // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + // in the Amazon Elastic Compute Cloud User Guide. + // +kubebuilder:validation:Enum:={standard,io1,io2,gp2,sc1,st1,gp3} + // +optional + VolumeType *string `json:"volumeType,omitempty"` +} + +// InstanceStorePolicy enumerates options for configuring instance store disks. +// +kubebuilder:validation:Enum={RAID0} +type InstanceStorePolicy string + +const ( + // InstanceStorePolicyRAID0 configures a RAID-0 array that includes all ephemeral NVMe instance storage disks. + // The containerd and kubelet state directories (`/var/lib/containerd` and `/var/lib/kubelet`) will then use the + // ephemeral storage for more and faster node ephemeral-storage. The node's ephemeral storage can be shared among + // pods that request ephemeral storage and container images that are downloaded to the node. + InstanceStorePolicyRAID0 InstanceStorePolicy = "RAID0" +) + +// // EC2NodeClassSpec defines the configuration for a Karpenter EC2NodeClass +// type EC2NodeClassSpec struct { +// // Name is the ami name in EC2. +// // This value is the name field, which is different from the name tag. +// AMIName string `json:"amiName,omitempty"` +// // Owner is the owner for the ami. +// // You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" +// AMIOwner string `json:"amiOwner,omitempty"` +// +// // SecurityGroups specifies the security groups to use +// // +optional +// SecurityGroups map[string]string `json:"securityGroups,omitempty"` +// +// // Subnets specifies the subnets to use +// // +optional +// Subnets map[string]string `json:"subnets,omitempty"` +// } diff --git a/api/v1alpha1/karpentermachinepool_types.go b/api/v1alpha1/karpentermachinepool_types.go index f1c8a793..1ae569fa 100644 --- a/api/v1alpha1/karpentermachinepool_types.go +++ b/api/v1alpha1/karpentermachinepool_types.go @@ -18,14 +18,19 @@ package v1alpha1 import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + capi "sigs.k8s.io/cluster-api/api/v1beta1" ) // KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. type KarpenterMachinePoolSpec struct { - // The name or the Amazon Resource Name (ARN) of the instance profile associated - // with the IAM role for the instance. The instance profile contains the IAM - // role. - IamInstanceProfile string `json:"iamInstanceProfile,omitempty"` + // NodePool specifies the configuration for the Karpenter NodePool + // +optional + NodePool *NodePoolSpec `json:"nodePool,omitempty"` + + // EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass + // +optional + EC2NodeClass *EC2NodeClassSpec `json:"ec2NodeClass,omitempty"` + // ProviderIDList are the identification IDs of machine instances provided by the provider. // This field must match the provider IDs as seen on the node objects corresponding to a machine pool's machine instances. // +optional @@ -34,13 +39,17 @@ type KarpenterMachinePoolSpec struct { // KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. type KarpenterMachinePoolStatus struct { - // Ready is true when the provider resource is ready. + // Ready denotes that the KarpenterMachinePool is ready and fulfilling the infrastructure contract. // +optional Ready bool `json:"ready"` // Replicas is the most recently observed number of replicas // +optional Replicas int32 `json:"replicas"` + + // Conditions defines current service state of the KarpenterMachinePool. + // +optional + Conditions capi.Conditions `json:"conditions,omitempty"` } // +kubebuilder:object:root=true @@ -72,3 +81,11 @@ type KarpenterMachinePoolList struct { func init() { SchemeBuilder.Register(&KarpenterMachinePool{}, &KarpenterMachinePoolList{}) } + +func (in *KarpenterMachinePool) GetConditions() capi.Conditions { + return in.Status.Conditions +} + +func (in *KarpenterMachinePool) SetConditions(conditions capi.Conditions) { + in.Status.Conditions = conditions +} diff --git a/api/v1alpha1/nodepool.go b/api/v1alpha1/nodepool.go new file mode 100644 index 00000000..6ad9f961 --- /dev/null +++ b/api/v1alpha1/nodepool.go @@ -0,0 +1,256 @@ +package v1alpha1 + +import ( + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NodePoolSpec defines the configuration for a Karpenter NodePool +type NodePoolSpec struct { + // Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + // NodeClaims launched from this NodePool will often be further constrained than the template specifies. + // +required + Template NodeClaimTemplate `json:"template"` + + // Disruption contains the parameters that relate to Karpenter's disruption logic + // +kubebuilder:default:={consolidateAfter: "0s"} + // +optional + Disruption Disruption `json:"disruption"` + + // Limits define a set of bounds for provisioning capacity. + // +optional + Limits Limits `json:"limits,omitempty"` + + // Weight is the priority given to the nodepool during scheduling. A higher + // numerical weight indicates that this nodepool will be ordered + // ahead of other nodepools with lower weights. A nodepool with no weight + // will be treated as if it is a nodepool with a weight of 0. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=100 + // +optional + Weight *int32 `json:"weight,omitempty"` +} + +type NodeClaimTemplate struct { + ObjectMeta `json:"metadata,omitempty"` + // +required + Spec NodeClaimTemplateSpec `json:"spec"` +} + +type ObjectMeta struct { + // Map of string keys and values that can be used to organize and categorize + // (scope and select) objects. May match selectors of replication controllers + // and services. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + // +optional + Labels map[string]string `json:"labels,omitempty"` + + // Annotations is an unstructured key value map stored with a resource that may be + // set by external tools to store and retrieve arbitrary metadata. They are not + // queryable and should be preserved when modifying objects. + // More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + // +optional + Annotations map[string]string `json:"annotations,omitempty"` +} + +// NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool +// NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since +// users are not able to set resource requests in the NodePool. +type NodeClaimTemplateSpec struct { + // Taints will be applied to the NodeClaim's node. + // +optional + Taints []v1.Taint `json:"taints,omitempty"` + // StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + // within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + // daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + // purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + // +optional + StartupTaints []v1.Taint `json:"startupTaints,omitempty"` + // Requirements are layered with GetLabels and applied to every node. + // +kubebuilder:validation:XValidation:message="requirements with operator 'In' must have a value defined",rule="self.all(x, x.operator == 'In' ? x.values.size() != 0 : true)" + // +kubebuilder:validation:XValidation:message="requirements operator 'Gt' or 'Lt' must have a single positive integer value",rule="self.all(x, (x.operator == 'Gt' || x.operator == 'Lt') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)" + // +kubebuilder:validation:XValidation:message="requirements with 'minValues' must have at least that many values specified in the 'values' field",rule="self.all(x, (x.operator == 'In' && has(x.minValues)) ? x.values.size() >= x.minValues : true)" + // +kubebuilder:validation:MaxItems:=100 + // +required + Requirements []NodeSelectorRequirementWithMinValues `json:"requirements" hash:"ignore"` + // TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + // + // Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + // + // This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + // When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + // + // Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + // If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + // that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + // + // The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + // If left undefined, the controller will wait indefinitely for pods to be drained. + // +kubebuilder:validation:Pattern=`^([0-9]+(s|m|h))+$` + // +kubebuilder:validation:Type="string" + // +optional + TerminationGracePeriod *metav1.Duration `json:"terminationGracePeriod,omitempty"` + // ExpireAfter is the duration the controller will wait + // before terminating a node, measured from when the node is created. This + // is useful to implement features like eventually consistent node upgrade, + // memory leak protection, and disruption testing. + // +kubebuilder:default:="720h" + // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` + // +kubebuilder:validation:Type="string" + // +kubebuilder:validation:Schemaless + // +optional + ExpireAfter NillableDuration `json:"expireAfter,omitempty"` +} + +// A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values +// and minValues that represent the requirement to have at least that many values. +type NodeSelectorRequirementWithMinValues struct { + v1.NodeSelectorRequirement `json:",inline"` + // This field is ALPHA and can be dropped or replaced at any time + // MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=50 + // +optional + MinValues *int `json:"minValues,omitempty"` +} + +type NodeClassReference struct { + // Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds" + // +kubebuilder:validation:XValidation:rule="self != ''",message="kind may not be empty" + // +required + Kind string `json:"kind"` + // Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names + // +kubebuilder:validation:XValidation:rule="self != ''",message="name may not be empty" + // +required + Name string `json:"name"` + // API version of the referent + // +kubebuilder:validation:XValidation:rule="self != ''",message="group may not be empty" + // +kubebuilder:validation:Pattern=`^[^/]*$` + // +required + Group string `json:"group"` +} + +type Limits v1.ResourceList + +type ConsolidationPolicy string + +const ( + ConsolidationPolicyWhenEmpty ConsolidationPolicy = "WhenEmpty" + ConsolidationPolicyWhenEmptyOrUnderutilized ConsolidationPolicy = "WhenEmptyOrUnderutilized" +) + +// DisruptionReason defines valid reasons for disruption budgets. +// +kubebuilder:validation:Enum={Underutilized,Empty,Drifted} +type DisruptionReason string + +// Budget defines when Karpenter will restrict the +// number of Node Claims that can be terminating simultaneously. +type Budget struct { + // Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + // Otherwise, this will apply to each reason defined. + // allowed reasons are Underutilized, Empty, and Drifted. + // +optional + Reasons []DisruptionReason `json:"reasons,omitempty"` + // Nodes dictates the maximum number of NodeClaims owned by this NodePool + // that can be terminating at once. This is calculated by counting nodes that + // have a deletion timestamp set, or are actively being deleted by Karpenter. + // This field is required when specifying a budget. + // This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + // checking for int nodes for IntOrString nodes. + // Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + // +kubebuilder:validation:Pattern:="^((100|[0-9]{1,2})%|[0-9]+)$" + // +kubebuilder:default:="10%" + Nodes string `json:"nodes" hash:"ignore"` + // Schedule specifies when a budget begins being active, following + // the upstream cronjob syntax. If omitted, the budget is always active. + // Timezones are not supported. + // This field is required if Duration is set. + // +kubebuilder:validation:Pattern:=`^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$` + // +optional + Schedule *string `json:"schedule,omitempty" hash:"ignore"` + // Duration determines how long a Budget is active since each Schedule hit. + // Only minutes and hours are accepted, as cron does not work in seconds. + // If omitted, the budget is always active. + // This is required if Schedule is set. + // This regex has an optional 0s at the end since the duration.String() always adds + // a 0s at the end. + // +kubebuilder:validation:Pattern=`^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$` + // +kubebuilder:validation:Type="string" + // +optional + Duration *metav1.Duration `json:"duration,omitempty" hash:"ignore"` +} + +type Disruption struct { + // ConsolidateAfter is the duration the controller will wait + // before attempting to terminate nodes that are underutilized. + // Refer to ConsolidationPolicy for how underutilization is considered. + // +kubebuilder:validation:Pattern=`^(([0-9]+(s|m|h))+|Never)$` + // +kubebuilder:validation:Type="string" + // +kubebuilder:validation:Schemaless + // +required + ConsolidateAfter NillableDuration `json:"consolidateAfter"` + // ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + // algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + // +kubebuilder:default:="WhenEmptyOrUnderutilized" + // +kubebuilder:validation:Enum:={WhenEmpty,WhenEmptyOrUnderutilized} + // +optional + ConsolidationPolicy ConsolidationPolicy `json:"consolidationPolicy,omitempty"` + // Budgets is a list of Budgets. + // If there are multiple active budgets, Karpenter uses + // the most restrictive value. If left undefined, + // this will default to one budget with a value to 10%. + // +kubebuilder:validation:XValidation:message="'schedule' must be set with 'duration'",rule="self.all(x, has(x.schedule) == has(x.duration))" + // +kubebuilder:default:={{nodes: "10%"}} + // +kubebuilder:validation:MaxItems=50 + // +optional + Budgets []Budget `json:"budgets,omitempty" hash:"ignore"` +} + +// ConsolidateUnderSpec defines when to consolidate under +type ConsolidateUnderSpec struct { + // CPUUtilization specifies the CPU utilization threshold + // +optional + CPUUtilization *string `json:"cpuUtilization,omitempty"` + + // MemoryUtilization specifies the memory utilization threshold + // +optional + MemoryUtilization *string `json:"memoryUtilization,omitempty"` +} + +// LimitsSpec defines the limits for a NodePool +type LimitsSpec struct { + // CPU specifies the CPU limit + // +optional + CPU *resource.Quantity `json:"cpu,omitempty"` + + // Memory specifies the memory limit + // +optional + Memory *resource.Quantity `json:"memory,omitempty"` +} + +// RequirementSpec defines a requirement for a NodePool +type RequirementSpec struct { + // Key specifies the requirement key + Key string `json:"key"` + + // Operator specifies the requirement operator + Operator string `json:"operator"` + + // Values specifies the requirement values + // +optional + Values []string `json:"values,omitempty"` +} + +// TaintSpec defines a taint for a NodePool +type TaintSpec struct { + // Key specifies the taint key + Key string `json:"key"` + + // Value specifies the taint value + // +optional + Value *string `json:"value,omitempty"` + + // Effect specifies the taint effect + Effect string `json:"effect"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 42853805..8bb0928b 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,16 +21,334 @@ limitations under the License. package v1alpha1 import ( + timex "time" + + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/cluster-api/api/v1beta1" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AMISelectorTerm) DeepCopyInto(out *AMISelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AMISelectorTerm. +func (in *AMISelectorTerm) DeepCopy() *AMISelectorTerm { + if in == nil { + return nil + } + out := new(AMISelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDevice) DeepCopyInto(out *BlockDevice) { + *out = *in + if in.DeleteOnTermination != nil { + in, out := &in.DeleteOnTermination, &out.DeleteOnTermination + *out = new(bool) + **out = **in + } + if in.Encrypted != nil { + in, out := &in.Encrypted, &out.Encrypted + *out = new(bool) + **out = **in + } + if in.IOPS != nil { + in, out := &in.IOPS, &out.IOPS + *out = new(int64) + **out = **in + } + if in.KMSKeyID != nil { + in, out := &in.KMSKeyID, &out.KMSKeyID + *out = new(string) + **out = **in + } + if in.SnapshotID != nil { + in, out := &in.SnapshotID, &out.SnapshotID + *out = new(string) + **out = **in + } + if in.Throughput != nil { + in, out := &in.Throughput, &out.Throughput + *out = new(int64) + **out = **in + } + if in.VolumeInitializationRate != nil { + in, out := &in.VolumeInitializationRate, &out.VolumeInitializationRate + *out = new(int32) + **out = **in + } + if in.VolumeSize != nil { + in, out := &in.VolumeSize, &out.VolumeSize + x := (*in).DeepCopy() + *out = &x + } + if in.VolumeType != nil { + in, out := &in.VolumeType, &out.VolumeType + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDevice. +func (in *BlockDevice) DeepCopy() *BlockDevice { + if in == nil { + return nil + } + out := new(BlockDevice) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BlockDeviceMapping) DeepCopyInto(out *BlockDeviceMapping) { + *out = *in + if in.DeviceName != nil { + in, out := &in.DeviceName, &out.DeviceName + *out = new(string) + **out = **in + } + if in.EBS != nil { + in, out := &in.EBS, &out.EBS + *out = new(BlockDevice) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BlockDeviceMapping. +func (in *BlockDeviceMapping) DeepCopy() *BlockDeviceMapping { + if in == nil { + return nil + } + out := new(BlockDeviceMapping) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Budget) DeepCopyInto(out *Budget) { + *out = *in + if in.Reasons != nil { + in, out := &in.Reasons, &out.Reasons + *out = make([]DisruptionReason, len(*in)) + copy(*out, *in) + } + if in.Schedule != nil { + in, out := &in.Schedule, &out.Schedule + *out = new(string) + **out = **in + } + if in.Duration != nil { + in, out := &in.Duration, &out.Duration + *out = new(v1.Duration) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Budget. +func (in *Budget) DeepCopy() *Budget { + if in == nil { + return nil + } + out := new(Budget) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapacityReservationSelectorTerm) DeepCopyInto(out *CapacityReservationSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapacityReservationSelectorTerm. +func (in *CapacityReservationSelectorTerm) DeepCopy() *CapacityReservationSelectorTerm { + if in == nil { + return nil + } + out := new(CapacityReservationSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConsolidateUnderSpec) DeepCopyInto(out *ConsolidateUnderSpec) { + *out = *in + if in.CPUUtilization != nil { + in, out := &in.CPUUtilization, &out.CPUUtilization + *out = new(string) + **out = **in + } + if in.MemoryUtilization != nil { + in, out := &in.MemoryUtilization, &out.MemoryUtilization + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConsolidateUnderSpec. +func (in *ConsolidateUnderSpec) DeepCopy() *ConsolidateUnderSpec { + if in == nil { + return nil + } + out := new(ConsolidateUnderSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Disruption) DeepCopyInto(out *Disruption) { + *out = *in + in.ConsolidateAfter.DeepCopyInto(&out.ConsolidateAfter) + if in.Budgets != nil { + in, out := &in.Budgets, &out.Budgets + *out = make([]Budget, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Disruption. +func (in *Disruption) DeepCopy() *Disruption { + if in == nil { + return nil + } + out := new(Disruption) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { + *out = *in + if in.SubnetSelectorTerms != nil { + in, out := &in.SubnetSelectorTerms, &out.SubnetSelectorTerms + *out = make([]SubnetSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.SecurityGroupSelectorTerms != nil { + in, out := &in.SecurityGroupSelectorTerms, &out.SecurityGroupSelectorTerms + *out = make([]SecurityGroupSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.CapacityReservationSelectorTerms != nil { + in, out := &in.CapacityReservationSelectorTerms, &out.CapacityReservationSelectorTerms + *out = make([]CapacityReservationSelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } + if in.AMISelectorTerms != nil { + in, out := &in.AMISelectorTerms, &out.AMISelectorTerms + *out = make([]AMISelectorTerm, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AMIFamily != nil { + in, out := &in.AMIFamily, &out.AMIFamily + *out = new(string) + **out = **in + } + if in.UserData != nil { + in, out := &in.UserData, &out.UserData + *out = new(string) + **out = **in + } + if in.InstanceProfile != nil { + in, out := &in.InstanceProfile, &out.InstanceProfile + *out = new(string) + **out = **in + } + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Kubelet != nil { + in, out := &in.Kubelet, &out.Kubelet + *out = new(KubeletConfiguration) + (*in).DeepCopyInto(*out) + } + if in.BlockDeviceMappings != nil { + in, out := &in.BlockDeviceMappings, &out.BlockDeviceMappings + *out = make([]*BlockDeviceMapping, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(BlockDeviceMapping) + (*in).DeepCopyInto(*out) + } + } + } + if in.InstanceStorePolicy != nil { + in, out := &in.InstanceStorePolicy, &out.InstanceStorePolicy + *out = new(InstanceStorePolicy) + **out = **in + } + if in.DetailedMonitoring != nil { + in, out := &in.DetailedMonitoring, &out.DetailedMonitoring + *out = new(bool) + **out = **in + } + if in.MetadataOptions != nil { + in, out := &in.MetadataOptions, &out.MetadataOptions + *out = new(MetadataOptions) + (*in).DeepCopyInto(*out) + } + if in.Context != nil { + in, out := &in.Context, &out.Context + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EC2NodeClassSpec. +func (in *EC2NodeClassSpec) DeepCopy() *EC2NodeClassSpec { + if in == nil { + return nil + } + out := new(EC2NodeClassSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterMachinePool) DeepCopyInto(out *KarpenterMachinePool) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterMachinePool. @@ -86,6 +404,16 @@ func (in *KarpenterMachinePoolList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterMachinePoolSpec) DeepCopyInto(out *KarpenterMachinePoolSpec) { *out = *in + if in.NodePool != nil { + in, out := &in.NodePool, &out.NodePool + *out = new(NodePoolSpec) + (*in).DeepCopyInto(*out) + } + if in.EC2NodeClass != nil { + in, out := &in.EC2NodeClass, &out.EC2NodeClass + *out = new(EC2NodeClassSpec) + (*in).DeepCopyInto(*out) + } if in.ProviderIDList != nil { in, out := &in.ProviderIDList, &out.ProviderIDList *out = make([]string, len(*in)) @@ -106,6 +434,13 @@ func (in *KarpenterMachinePoolSpec) DeepCopy() *KarpenterMachinePoolSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KarpenterMachinePoolStatus) DeepCopyInto(out *KarpenterMachinePoolStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(v1beta1.Conditions, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KarpenterMachinePoolStatus. @@ -117,3 +452,431 @@ func (in *KarpenterMachinePoolStatus) DeepCopy() *KarpenterMachinePoolStatus { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { + *out = *in + if in.ClusterDNS != nil { + in, out := &in.ClusterDNS, &out.ClusterDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } + if in.PodsPerCore != nil { + in, out := &in.PodsPerCore, &out.PodsPerCore + *out = new(int32) + **out = **in + } + if in.SystemReserved != nil { + in, out := &in.SystemReserved, &out.SystemReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.KubeReserved != nil { + in, out := &in.KubeReserved, &out.KubeReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionHard != nil { + in, out := &in.EvictionHard, &out.EvictionHard + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoft != nil { + in, out := &in.EvictionSoft, &out.EvictionSoft + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoftGracePeriod != nil { + in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod + *out = make(map[string]v1.Duration, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionMaxPodGracePeriod != nil { + in, out := &in.EvictionMaxPodGracePeriod, &out.EvictionMaxPodGracePeriod + *out = new(int32) + **out = **in + } + if in.ImageGCHighThresholdPercent != nil { + in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent + *out = new(int32) + **out = **in + } + if in.ImageGCLowThresholdPercent != nil { + in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent + *out = new(int32) + **out = **in + } + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. +func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { + if in == nil { + return nil + } + out := new(KubeletConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in Limits) DeepCopyInto(out *Limits) { + { + in := &in + *out = make(Limits, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Limits. +func (in Limits) DeepCopy() Limits { + if in == nil { + return nil + } + out := new(Limits) + in.DeepCopyInto(out) + return *out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LimitsSpec) DeepCopyInto(out *LimitsSpec) { + *out = *in + if in.CPU != nil { + in, out := &in.CPU, &out.CPU + x := (*in).DeepCopy() + *out = &x + } + if in.Memory != nil { + in, out := &in.Memory, &out.Memory + x := (*in).DeepCopy() + *out = &x + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LimitsSpec. +func (in *LimitsSpec) DeepCopy() *LimitsSpec { + if in == nil { + return nil + } + out := new(LimitsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetadataOptions) DeepCopyInto(out *MetadataOptions) { + *out = *in + if in.HTTPEndpoint != nil { + in, out := &in.HTTPEndpoint, &out.HTTPEndpoint + *out = new(string) + **out = **in + } + if in.HTTPProtocolIPv6 != nil { + in, out := &in.HTTPProtocolIPv6, &out.HTTPProtocolIPv6 + *out = new(string) + **out = **in + } + if in.HTTPPutResponseHopLimit != nil { + in, out := &in.HTTPPutResponseHopLimit, &out.HTTPPutResponseHopLimit + *out = new(int64) + **out = **in + } + if in.HTTPTokens != nil { + in, out := &in.HTTPTokens, &out.HTTPTokens + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetadataOptions. +func (in *MetadataOptions) DeepCopy() *MetadataOptions { + if in == nil { + return nil + } + out := new(MetadataOptions) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NillableDuration) DeepCopyInto(out *NillableDuration) { + *out = *in + if in.Duration != nil { + in, out := &in.Duration, &out.Duration + *out = new(timex.Duration) + **out = **in + } + if in.Raw != nil { + in, out := &in.Raw, &out.Raw + *out = make([]byte, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NillableDuration. +func (in *NillableDuration) DeepCopy() *NillableDuration { + if in == nil { + return nil + } + out := new(NillableDuration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimTemplate) DeepCopyInto(out *NodeClaimTemplate) { + *out = *in + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimTemplate. +func (in *NodeClaimTemplate) DeepCopy() *NodeClaimTemplate { + if in == nil { + return nil + } + out := new(NodeClaimTemplate) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClaimTemplateSpec) DeepCopyInto(out *NodeClaimTemplateSpec) { + *out = *in + if in.Taints != nil { + in, out := &in.Taints, &out.Taints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.StartupTaints != nil { + in, out := &in.StartupTaints, &out.StartupTaints + *out = make([]corev1.Taint, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Requirements != nil { + in, out := &in.Requirements, &out.Requirements + *out = make([]NodeSelectorRequirementWithMinValues, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.TerminationGracePeriod != nil { + in, out := &in.TerminationGracePeriod, &out.TerminationGracePeriod + *out = new(v1.Duration) + **out = **in + } + in.ExpireAfter.DeepCopyInto(&out.ExpireAfter) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClaimTemplateSpec. +func (in *NodeClaimTemplateSpec) DeepCopy() *NodeClaimTemplateSpec { + if in == nil { + return nil + } + out := new(NodeClaimTemplateSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeClassReference) DeepCopyInto(out *NodeClassReference) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeClassReference. +func (in *NodeClassReference) DeepCopy() *NodeClassReference { + if in == nil { + return nil + } + out := new(NodeClassReference) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodePoolSpec) DeepCopyInto(out *NodePoolSpec) { + *out = *in + in.Template.DeepCopyInto(&out.Template) + in.Disruption.DeepCopyInto(&out.Disruption) + if in.Limits != nil { + in, out := &in.Limits, &out.Limits + *out = make(Limits, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + if in.Weight != nil { + in, out := &in.Weight, &out.Weight + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodePoolSpec. +func (in *NodePoolSpec) DeepCopy() *NodePoolSpec { + if in == nil { + return nil + } + out := new(NodePoolSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NodeSelectorRequirementWithMinValues) DeepCopyInto(out *NodeSelectorRequirementWithMinValues) { + *out = *in + in.NodeSelectorRequirement.DeepCopyInto(&out.NodeSelectorRequirement) + if in.MinValues != nil { + in, out := &in.MinValues, &out.MinValues + *out = new(int) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeSelectorRequirementWithMinValues. +func (in *NodeSelectorRequirementWithMinValues) DeepCopy() *NodeSelectorRequirementWithMinValues { + if in == nil { + return nil + } + out := new(NodeSelectorRequirementWithMinValues) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ObjectMeta) DeepCopyInto(out *ObjectMeta) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ObjectMeta. +func (in *ObjectMeta) DeepCopy() *ObjectMeta { + if in == nil { + return nil + } + out := new(ObjectMeta) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RequirementSpec) DeepCopyInto(out *RequirementSpec) { + *out = *in + if in.Values != nil { + in, out := &in.Values, &out.Values + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequirementSpec. +func (in *RequirementSpec) DeepCopy() *RequirementSpec { + if in == nil { + return nil + } + out := new(RequirementSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SecurityGroupSelectorTerm) DeepCopyInto(out *SecurityGroupSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityGroupSelectorTerm. +func (in *SecurityGroupSelectorTerm) DeepCopy() *SecurityGroupSelectorTerm { + if in == nil { + return nil + } + out := new(SecurityGroupSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SubnetSelectorTerm) DeepCopyInto(out *SubnetSelectorTerm) { + *out = *in + if in.Tags != nil { + in, out := &in.Tags, &out.Tags + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SubnetSelectorTerm. +func (in *SubnetSelectorTerm) DeepCopy() *SubnetSelectorTerm { + if in == nil { + return nil + } + out := new(SubnetSelectorTerm) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TaintSpec) DeepCopyInto(out *TaintSpec) { + *out = *in + if in.Value != nil { + in, out := &in.Value, &out.Value + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TaintSpec. +func (in *TaintSpec) DeepCopy() *TaintSpec { + if in == nil { + return nil + } + out := new(TaintSpec) + in.DeepCopyInto(out) + return out +} diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 69c3e422..eeee82cf 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 helm.sh/resource-policy: keep labels: cluster.x-k8s.io/v1beta1: v1alpha1 @@ -50,12 +50,915 @@ spec: spec: description: KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. properties: - iamInstanceProfile: - description: |- - The name or the Amazon Resource Name (ARN) of the instance profile associated - with the IAM role for the instance. The instance profile contains the IAM - role. - type: string + ec2NodeClass: + description: EC2NodeClass specifies the configuration for the Karpenter + EC2NodeClass + properties: + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match + the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the + following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', + ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] + ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM + parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', + 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || + has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned + nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically + set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the + EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, + or alias ARN) of the customer managed KMS key to use + for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate + is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) + && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 + capacityReservationSelectorTerms: + description: |- + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along + with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring + is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal + names to quantities that define grace periods for each eviction + signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + description: KubeReserved contains resources reserved for + Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + description: SystemReserved contains resources reserved for + OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in + self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security + group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with + a combination of other fields in a security group selector + term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector + terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances + and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + nodePool: + description: NodePool specifies the configuration for the Karpenter + NodePool + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to + Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons + for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + requirements: + description: Requirements are layered with GetLabels and + applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a + value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have + a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator + == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) + >= 0) : true)' + - message: requirements with 'minValues' must have at + least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's + node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object providerIDList: description: |- ProviderIDList are the identification IDs of machine instances provided by the provider. @@ -68,8 +971,54 @@ spec: description: KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. properties: + conditions: + description: Conditions defines current service state of the KarpenterMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array ready: - description: Ready is true when the provider resource is ready. + description: Ready denotes that the KarpenterMachinePool is ready + and fulfilling the infrastructure contract. type: boolean replicas: description: Replicas is the most recently observed number of replicas diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index fa80df42..031d28f7 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -39,6 +39,7 @@ import ( capi "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" @@ -89,8 +90,10 @@ var _ = BeforeSuite(func() { testEnv = &envtest.Environment{ CRDDirectoryPaths: []string{ filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", fmt.Sprintf("cluster-api@%s", capiModule[0].Module.Version), "config", "crd", "bases"), + filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", fmt.Sprintf("cluster-api@%s", capiModule[0].Module.Version), "controlplane", "kubeadm", "config", "crd", "bases"), filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", "cluster-api-provider-aws", fmt.Sprintf("v2@%s", capaModule[0].Module.Version), "config", "crd", "bases"), crdPath, + filepath.Join("..", "config", "crd", "bases"), }, ErrorIfCRDPathMissing: true, } @@ -111,6 +114,8 @@ var _ = BeforeSuite(func() { k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) Expect(err).NotTo(HaveOccurred()) Expect(k8sClient).NotTo(BeNil()) + + komega.SetClient(k8sClient) }) var _ = AfterSuite(func() { diff --git a/controllers/controllersfakes/fake_awscluster_client.go b/controllers/controllersfakes/fake_awscluster_client.go index b670ad9d..76442c27 100644 --- a/controllers/controllersfakes/fake_awscluster_client.go +++ b/controllers/controllersfakes/fake_awscluster_client.go @@ -795,26 +795,6 @@ func (fake *FakeAWSClusterClient) UpdateStatusReturnsOnCall(i int, result1 error func (fake *FakeAWSClusterClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addFinalizerMutex.RLock() - defer fake.addFinalizerMutex.RUnlock() - fake.getAWSClusterMutex.RLock() - defer fake.getAWSClusterMutex.RUnlock() - fake.getClusterMutex.RLock() - defer fake.getClusterMutex.RUnlock() - fake.getIdentityMutex.RLock() - defer fake.getIdentityMutex.RUnlock() - fake.getOwnerMutex.RLock() - defer fake.getOwnerMutex.RUnlock() - fake.markConditionTrueMutex.RLock() - defer fake.markConditionTrueMutex.RUnlock() - fake.patchClusterMutex.RLock() - defer fake.patchClusterMutex.RUnlock() - fake.removeFinalizerMutex.RLock() - defer fake.removeFinalizerMutex.RUnlock() - fake.unpauseMutex.RLock() - defer fake.unpauseMutex.RUnlock() - fake.updateStatusMutex.RLock() - defer fake.updateStatusMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/controllers/controllersfakes/fake_cluster_client.go b/controllers/controllersfakes/fake_cluster_client.go index 5db00477..03fb5c8d 100644 --- a/controllers/controllersfakes/fake_cluster_client.go +++ b/controllers/controllersfakes/fake_cluster_client.go @@ -944,30 +944,6 @@ func (fake *FakeClusterClient) UnpauseReturnsOnCall(i int, result1 error) { func (fake *FakeClusterClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addAWSClusterFinalizerMutex.RLock() - defer fake.addAWSClusterFinalizerMutex.RUnlock() - fake.addAWSManagedControlPlaneFinalizerMutex.RLock() - defer fake.addAWSManagedControlPlaneFinalizerMutex.RUnlock() - fake.addClusterFinalizerMutex.RLock() - defer fake.addClusterFinalizerMutex.RUnlock() - fake.getAWSClusterMutex.RLock() - defer fake.getAWSClusterMutex.RUnlock() - fake.getAWSManagedControlPlaneMutex.RLock() - defer fake.getAWSManagedControlPlaneMutex.RUnlock() - fake.getClusterMutex.RLock() - defer fake.getClusterMutex.RUnlock() - fake.getIdentityMutex.RLock() - defer fake.getIdentityMutex.RUnlock() - fake.markConditionTrueMutex.RLock() - defer fake.markConditionTrueMutex.RUnlock() - fake.removeAWSClusterFinalizerMutex.RLock() - defer fake.removeAWSClusterFinalizerMutex.RUnlock() - fake.removeAWSManagedControlPlaneFinalizerMutex.RLock() - defer fake.removeAWSManagedControlPlaneFinalizerMutex.RUnlock() - fake.removeClusterFinalizerMutex.RLock() - defer fake.removeClusterFinalizerMutex.RUnlock() - fake.unpauseMutex.RLock() - defer fake.unpauseMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/controllers/karpentermachinepool_controller.go b/controllers/karpentermachinepool_controller.go index b087f1a7..57e2c577 100644 --- a/controllers/karpentermachinepool_controller.go +++ b/controllers/karpentermachinepool_controller.go @@ -3,6 +3,7 @@ package controllers import ( "context" "crypto/sha256" + "encoding/json" "errors" "fmt" "path" @@ -10,15 +11,19 @@ import ( "github.com/go-logr/logr" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" capalogger "sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger" capi "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/controllers/remote" + capiexp "sigs.k8s.io/cluster-api/exp/api/v1beta1" capiutilexp "sigs.k8s.io/cluster-api/exp/util" capiutil "sigs.k8s.io/cluster-api/util" "sigs.k8s.io/cluster-api/util/annotations" + "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/cluster-api/util/predicates" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -27,23 +32,32 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "github.com/aws-resolver-rules-operator/api/v1alpha1" + "github.com/aws-resolver-rules-operator/pkg/conditions" "github.com/aws-resolver-rules-operator/pkg/resolver" + "github.com/aws-resolver-rules-operator/pkg/versionskew" ) const ( + // BootstrapDataHashAnnotation stores the SHA256 hash of the bootstrap data + // to detect when userdata changes and needs to be re-uploaded to S3 BootstrapDataHashAnnotation = "giantswarm.io/userdata-hash" - KarpenterFinalizer = "capa-operator.finalizers.giantswarm.io/karpenter-controller" - S3ObjectPrefix = "karpenter-machine-pool" - // KarpenterNodePoolReadyCondition reports on current status of the autoscaling group. Ready indicates the group is provisioned. - KarpenterNodePoolReadyCondition capi.ConditionType = "KarpenterNodePoolReadyCondition" - // WaitingForBootstrapDataReason used when machine is waiting for bootstrap data to be ready before proceeding. - WaitingForBootstrapDataReason = "WaitingForBootstrapData" + + // EC2NodeClassAPIGroup is the API group for Karpenter EC2NodeClass resources + EC2NodeClassAPIGroup = "karpenter.k8s.aws" + + // KarpenterFinalizer ensures proper cleanup of Karpenter resources and EC2 instances + // before allowing the KarpenterMachinePool to be deleted + KarpenterFinalizer = "capa-operator.finalizers.giantswarm.io/karpenter-controller" + + // S3ObjectPrefix is the S3 path prefix where bootstrap data is stored + // Format: s3://// + S3ObjectPrefix = "karpenter-machine-pool" ) type KarpenterMachinePoolReconciler struct { awsClients resolver.AWSClients client client.Client - // clusterClientGetter is used to create a client targeting the workload cluster + // clusterClientGetter is used to create a kubernetes client targeting the workload cluster clusterClientGetter remote.ClusterClientGetter } @@ -51,8 +65,12 @@ func NewKarpenterMachinepoolReconciler(client client.Client, clusterClientGetter return &KarpenterMachinePoolReconciler{awsClients: awsClients, client: client, clusterClientGetter: clusterClientGetter} } -// Reconcile will upload to S3 the Ignition configuration for the reconciled node pool. -// It will also take care of deleting EC2 instances created by karpenter when the cluster is being removed. +// Reconcile reconciles KarpenterMachinePool CRs, which represent cluster node pools that will be managed by karpenter. +// KarpenterMachinePoolReconciler reconciles KarpenterMachinePool objects by: +// 1. Creating Karpenter NodePool and EC2NodeClass resources in workload clusters +// 2. Managing bootstrap data upload to S3 for node initialization +// 3. Enforcing Kubernetes version skew policies +// 4. Cleaning up EC2 instances when clusters are deleted func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { logger := log.FromContext(ctx) logger.Info("Reconciling") @@ -63,8 +81,28 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return reconcile.Result{}, client.IgnoreNotFound(err) } + patchHelper, err := patch.NewHelper(karpenterMachinePool, r.client) + if err != nil { + return reconcile.Result{}, fmt.Errorf("failed to init patch helper: %w", err) + } + defer func() { + if err := patchHelper.Patch(ctx, karpenterMachinePool, patch.WithOwnedConditions{ + Conditions: []capi.ConditionType{ + conditions.ReadyCondition, + conditions.NodePoolCreatedCondition, + conditions.EC2NodeClassCreatedCondition, + conditions.BootstrapDataReadyCondition, + conditions.VersionSkewPolicySatisfiedCondition, + }, + }, patch.WithForceOverwriteConditions{}); err != nil { + logger.Error(err, "failed to patch KarpenterMachinePool") + } + }() + machinePool, err := capiutilexp.GetOwnerMachinePool(ctx, r.client, karpenterMachinePool.ObjectMeta) if err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get MachinePool owning the KarpenterMachinePool: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get MachinePool owning the KarpenterMachinePool: %w", err) } if machinePool == nil { @@ -75,7 +113,11 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco } logger = logger.WithValues("machinePool", machinePool.Name) + // Bootstrap data must be available before we can proceed with creating Karpenter resources if machinePool.Spec.Template.Spec.Bootstrap.DataSecretName == nil { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretMissingReferenceReason, "Bootstrap data secret reference is not yet available in MachinePool") + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, "Bootstrap data secret reference is not yet available in MachinePool") + karpenterMachinePool.Status.Ready = false logger.Info("Bootstrap data secret reference is not yet available") return reconcile.Result{RequeueAfter: time.Duration(10) * time.Second}, nil } @@ -84,6 +126,8 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco cluster, err := capiutil.GetClusterFromMetadata(ctx, r.client, machinePool.ObjectMeta) if err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get Cluster owning the MachinePool: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get Cluster owning the MachinePool that owns the KarpenterMachinePool: %w", err) } @@ -96,6 +140,8 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco awsCluster := &capa.AWSCluster{} if err := r.client.Get(ctx, client.ObjectKey{Namespace: cluster.Spec.InfrastructureRef.Namespace, Name: cluster.Spec.InfrastructureRef.Name}, awsCluster); err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get AWSCluster: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get AWSCluster referenced in Cluster.spec.infrastructureRef: %w", err) } @@ -104,107 +150,164 @@ func (r *KarpenterMachinePoolReconciler) Reconcile(ctx context.Context, req reco return ctrl.Result{}, nil } + // S3 bucket is required for storing bootstrap data that Karpenter nodes will fetch if awsCluster.Spec.S3Bucket == nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, "S3 bucket is required but not configured in AWSCluster.spec.s3Bucket") + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, errors.New("a cluster wide object storage configured at `AWSCluster.spec.s3Bucket` is required") } + // Get AWS credentials for S3 and EC2 operations roleIdentity := &capa.AWSClusterRoleIdentity{} if err = r.client.Get(ctx, client.ObjectKey{Name: awsCluster.Spec.IdentityRef.Name}, roleIdentity); err != nil { + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to get AWSClusterRoleIdentity: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, fmt.Errorf("failed to get AWSClusterRoleIdentity referenced in AWSCluster: %w", err) } - // Create deep copy of the reconciled object so we can change it - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - - // We only remove the finalizer after we've removed the EC2 instances created by karpenter. - // These are normally removed by Karpenter but when deleting a cluster, karpenter may not have enough time to clean them up. + // Handle deletion: cleanup EC2 instances and Karpenter resources if !karpenterMachinePool.GetDeletionTimestamp().IsZero() { - return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity) - } - - bootstrapSecret := &v1.Secret{} - if err := r.client.Get(ctx, client.ObjectKey{Namespace: req.Namespace, Name: *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName}, bootstrapSecret); err != nil { - return reconcile.Result{}, fmt.Errorf("bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName is not found: %w", err) - } - - bootstrapSecretValue, ok := bootstrapSecret.Data["value"] - if !ok { - return reconcile.Result{}, errors.New("error retrieving bootstrap data: secret value key is missing") + return r.reconcileDelete(ctx, logger, cluster, awsCluster, karpenterMachinePool, roleIdentity, patchHelper) } - updated := controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) - if updated { - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { - return reconcile.Result{}, fmt.Errorf("failed to add finalizer to KarpenterMachinePool: %w", err) - } + // Validate version skew: ensure worker nodes don't use newer Kubernetes versions than control plane + allowed, controlPlaneCurrentVersion, nodePoolDesiredVersion, err := r.IsVersionSkewAllowed(ctx, cluster, machinePool) + if err != nil { + return reconcile.Result{}, err } - bootstrapUserDataHash := fmt.Sprintf("%x", sha256.Sum256(bootstrapSecretValue)) - previousHash, annotationHashExists := karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] - if !annotationHashExists || previousHash != bootstrapUserDataHash { - s3Client, err := r.awsClients.NewS3Client(awsCluster.Spec.Region, roleIdentity.Spec.RoleArn) - if err != nil { - return reconcile.Result{}, err - } + if !allowed { + message := fmt.Sprintf("Version skew policy violation: control plane version %s is older than node pool version %s", controlPlaneCurrentVersion, nodePoolDesiredVersion) + logger.Info("Blocking Karpenter custom resources update due to version skew policy", + "controlPlaneCurrentVersion", controlPlaneCurrentVersion, + "nodePoolDesiredVersion", nodePoolDesiredVersion, + "reason", message) - key := path.Join(S3ObjectPrefix, req.Name) + // Mark version skew as invalid and resources as not ready + conditions.MarkVersionSkewInvalid(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) + conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.VersionSkewBlockedReason, message) - logger.Info("Writing userdata to S3", "bucket", awsCluster.Spec.S3Bucket.Name, "key", key) - if err = s3Client.Put(ctx, awsCluster.Spec.S3Bucket.Name, key, bootstrapSecretValue); err != nil { - return reconcile.Result{}, err - } + return reconcile.Result{RequeueAfter: time.Duration(1) * time.Minute}, nil + } - if karpenterMachinePool.Annotations == nil { - karpenterMachinePool.Annotations = make(map[string]string) - } - karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] = bootstrapUserDataHash + // Mark version skew as valid + conditions.MarkVersionSkewPolicySatisfied(karpenterMachinePool) + // Add finalizer to ensure proper cleanup sequence + controllerutil.AddFinalizer(karpenterMachinePool, KarpenterFinalizer) - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.annotations with user data hash", "annotation", BootstrapDataHashAnnotation) - return reconcile.Result{}, err - } + // Create or update Karpenter custom resources in the workload cluster. + if err := r.createOrUpdateKarpenterResources(ctx, logger, cluster, awsCluster, karpenterMachinePool, machinePool); err != nil { + logger.Error(err, "failed to create or update Karpenter custom resources in the workload cluster") + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to create or update Karpenter resources: %v", err)) + karpenterMachinePool.Status.Ready = false + return reconcile.Result{}, err } - providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) - if err != nil { + // Reconcile bootstrap data - fetch secret and upload to S3 if changed + if err := r.reconcileMachinePoolBootstrapUserData(ctx, logger, awsCluster, karpenterMachinePool, *machinePool.Spec.Template.Spec.Bootstrap.DataSecretName, roleIdentity); err != nil { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to reconcile bootstrap data: %v", err)) + karpenterMachinePool.Status.Ready = false return reconcile.Result{}, err } + conditions.MarkBootstrapDataReady(karpenterMachinePool) - if numberOfNodeClaims == 0 { - // Karpenter has not reacted yet, let's requeue - return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil + // Update status with current node information from the workload cluster + if err := r.saveKarpenterInstancesToStatus(ctx, logger, cluster, karpenterMachinePool, machinePool); err != nil { + logger.Error(err, "failed to save Karpenter instances to status") + conditions.MarkKarpenterMachinePoolNotReady(karpenterMachinePool, conditions.NotReadyReason, fmt.Sprintf("Failed to save Karpenter instances to status: %v", err)) + karpenterMachinePool.Status.Ready = false + return reconcile.Result{}, err } - karpenterMachinePool.Status.Replicas = numberOfNodeClaims + // Mark the KarpenterMachinePool as ready when all conditions are satisfied + conditions.MarkKarpenterMachinePoolReady(karpenterMachinePool) karpenterMachinePool.Status.Ready = true - logger.Info("Found NodeClaims in workload cluster, patching KarpenterMachinePool", "numberOfNodeClaims", numberOfNodeClaims) + return reconcile.Result{}, nil +} - if err := r.client.Status().Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.status.Replicas") - return reconcile.Result{}, err +// saveKarpenterInstancesToStatus updates the KarpenterMachinePool and parent MachinePool with current node information +// from the workload cluster, including replica counts and provider ID lists. +func (r *KarpenterMachinePoolReconciler) saveKarpenterInstancesToStatus(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { + providerIDList, numberOfNodeClaims, err := r.computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx, logger, cluster) + if err != nil { + return err } - karpenterMachinePool.Spec.ProviderIDList = providerIDList + logger.Info("Updating MachinePool.spec.replicas, KarpenterMachinePool.spec.ProviderIDList and KarpenterMachinePool.status.Replicas", "numberOfNodeClaims", numberOfNodeClaims, "providerIDList", providerIDList) - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy), client.FieldOwner("karpentermachinepool-controller")); err != nil { - logger.Error(err, "failed to patch karpenterMachinePool.spec.providerIDList") - return reconcile.Result{}, err - } + karpenterMachinePool.Status.Replicas = numberOfNodeClaims + karpenterMachinePool.Spec.ProviderIDList = providerIDList + // Update the parent MachinePool replica count to match actual node claims if machinePool.Spec.Replicas == nil || *machinePool.Spec.Replicas != numberOfNodeClaims { machinePoolCopy := machinePool.DeepCopy() machinePool.Spec.Replicas = &numberOfNodeClaims if err := r.client.Patch(ctx, machinePool, client.MergeFrom(machinePoolCopy), client.FieldOwner("karpenter-machinepool-controller")); err != nil { logger.Error(err, "failed to patch MachinePool.spec.replicas") - return reconcile.Result{}, err + return err } } - return reconcile.Result{}, nil + return nil } -func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, roleIdentity *capa.AWSClusterRoleIdentity) (reconcile.Result, error) { +// reconcileMachinePoolBootstrapUserData handles the bootstrap user data reconciliation process. +// It fetches the bootstrap secret, checks if the data has changed, and uploads it to S3 if needed. +// It also updates the hash annotation to track the current bootstrap data version. +func (r *KarpenterMachinePoolReconciler) reconcileMachinePoolBootstrapUserData(ctx context.Context, logger logr.Logger, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, dataSecretName string, roleIdentity *capa.AWSClusterRoleIdentity) error { + // Get the bootstrap secret containing userdata for node initialization + bootstrapSecret := &v1.Secret{} + if err := r.client.Get(ctx, client.ObjectKey{Namespace: karpenterMachinePool.Namespace, Name: dataSecretName}, bootstrapSecret); err != nil { + if k8serrors.IsNotFound(err) { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretNotFoundReason, fmt.Sprintf("Bootstrap secret %s not found", dataSecretName)) + } else { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataUploadFailedReason, fmt.Sprintf("Failed to get bootstrap secret %s: %v", dataSecretName, err)) + } + return fmt.Errorf("failed to get bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName: %w", err) + } + + bootstrapSecretValue, ok := bootstrapSecret.Data["value"] + if !ok { + conditions.MarkBootstrapDataNotReady(karpenterMachinePool, conditions.BootstrapDataSecretInvalidReason, "Bootstrap secret value key is missing") + return errors.New("error retrieving bootstrap data: secret value key is missing") + } + + // Check if bootstrap data has changed and needs to be re-uploaded to S3 + bootstrapUserDataHash := fmt.Sprintf("%x", sha256.Sum256(bootstrapSecretValue)) + previousHash, annotationHashExists := karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] + if !annotationHashExists || previousHash != bootstrapUserDataHash { + s3Client, err := r.awsClients.NewS3Client(awsCluster.Spec.Region, roleIdentity.Spec.RoleArn) + if err != nil { + return err + } + + key := path.Join(S3ObjectPrefix, karpenterMachinePool.Name) + + logger.Info("Writing userdata to S3", "bucket", awsCluster.Spec.S3Bucket.Name, "key", key) + if err = s3Client.Put(ctx, awsCluster.Spec.S3Bucket.Name, key, bootstrapSecretValue); err != nil { + return err + } + + // Update the hash annotation to track the current bootstrap data version + if karpenterMachinePool.Annotations == nil { + karpenterMachinePool.Annotations = make(map[string]string) + } + karpenterMachinePool.Annotations[BootstrapDataHashAnnotation] = bootstrapUserDataHash + } + + return nil +} + +// reconcileDelete deletes the karpenter custom resources from the workload cluster. +// When the cluster itself is being deleted, it also terminates all EC2 instances +// created by Karpenter to prevent orphaned resources. +func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, roleIdentity *capa.AWSClusterRoleIdentity, patchHelper *patch.Helper) (reconcile.Result, error) { + // We check if the owner Cluster is also being deleted (on top of the `KarpenterMachinePool` being deleted). + // If the Cluster is being deleted, we terminate all the ec2 instances that karpenter may have launched. + // These are normally removed by Karpenter, but when deleting a cluster, karpenter may not have enough time to clean them up. if !cluster.GetDeletionTimestamp().IsZero() { ec2Client, err := r.awsClients.NewEC2Client(awsCluster.Spec.Region, roleIdentity.Spec.RoleArn) if err != nil { @@ -212,55 +315,57 @@ func (r *KarpenterMachinePoolReconciler) reconcileDelete(ctx context.Context, lo } // Terminate EC2 instances with the karpenter.sh/nodepool tag matching the KarpenterMachinePool name - logger.Info("Terminating EC2 instances for KarpenterMachinePool", "karpenterMachinePoolName", karpenterMachinePool.Name) instanceIDs, err := ec2Client.TerminateInstancesByTag(ctx, logger, "karpenter.sh/nodepool", karpenterMachinePool.Name) if err != nil { return reconcile.Result{}, fmt.Errorf("failed to terminate EC2 instances: %w", err) } - logger.Info("Found instances", "instanceIDs", instanceIDs) - - // Requeue if we find instances to terminate. Once there are no instances to terminate, we proceed to remove the finalizer. - // We do this when the cluster is being deleted, to avoid removing the finalizer before karpenter launches a new instance that would be left over. + // Requeue if we find instances to terminate. On the next reconciliation, once there are no instances to terminate, we proceed to remove the finalizer. + // We don't want to remove the finalizer when there may be ec2 instances still around to be cleaned up. if len(instanceIDs) > 0 { return reconcile.Result{RequeueAfter: 30 * time.Second}, nil } } - // Create deep copy of the reconciled object so we can change it - karpenterMachinePoolCopy := karpenterMachinePool.DeepCopy() - - controllerutil.RemoveFinalizer(karpenterMachinePool, KarpenterFinalizer) - if err := r.client.Patch(ctx, karpenterMachinePool, client.MergeFrom(karpenterMachinePoolCopy)); err != nil { - logger.Error(err, "failed to remove finalizer", "finalizer", KarpenterFinalizer) + // Delete Karpenter resources from the workload cluster + if err := r.deleteKarpenterResources(ctx, logger, cluster, karpenterMachinePool); err != nil { + logger.Error(err, "failed to delete Karpenter resources") return reconcile.Result{}, err } + logger.Info("Removing finalizer", "finalizer", KarpenterFinalizer) + controllerutil.RemoveFinalizer(karpenterMachinePool, KarpenterFinalizer) + return reconcile.Result{}, nil } -func getWorkloadClusterNodeClaims(ctx context.Context, ctrlClient client.Client) (*unstructured.UnstructuredList, error) { +// getWorkloadClusterNodeClaims retrieves all NodeClaim resources from the workload cluster. +// NodeClaims represent actual compute resources provisioned by Karpenter. +func (r *KarpenterMachinePoolReconciler) getWorkloadClusterNodeClaims(ctx context.Context, cluster *capi.Cluster) (*unstructured.UnstructuredList, error) { + nodeClaimList := &unstructured.UnstructuredList{} + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) + if err != nil { + return nodeClaimList, err + } + nodeClaimGVR := schema.GroupVersionResource{ Group: "karpenter.sh", Version: "v1", Resource: "nodeclaims", } - nodeClaimList := &unstructured.UnstructuredList{} nodeClaimList.SetGroupVersionKind(nodeClaimGVR.GroupVersion().WithKind("NodeClaimList")) - err := ctrlClient.List(ctx, nodeClaimList) + err = workloadClusterClient.List(ctx, nodeClaimList) return nodeClaimList, err } +// computeProviderIDListFromNodeClaimsInWorkloadCluster extracts provider IDs from NodeClaims +// and returns both the list of provider IDs and the total count of node claims. +// Provider IDs are AWS-specific identifiers like "aws:///us-west-2a/i-1234567890abcdef0" func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWorkloadCluster(ctx context.Context, logger logr.Logger, cluster *capi.Cluster) ([]string, int32, error) { var providerIDList []string - workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) - if err != nil { - return providerIDList, 0, err - } - - nodeClaimList, err := getWorkloadClusterNodeClaims(ctx, workloadClusterClient) + nodeClaimList, err := r.getWorkloadClusterNodeClaims(ctx, cluster) if err != nil { return providerIDList, 0, err } @@ -271,6 +376,7 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo logger.Error(err, "error retrieving nodeClaim.status.providerID", "nodeClaim", nc.GetName()) continue } + if found && providerID != "" { providerIDList = append(providerIDList, providerID) } @@ -280,6 +386,294 @@ func (r *KarpenterMachinePoolReconciler) computeProviderIDListFromNodeClaimsInWo return providerIDList, int32(len(nodeClaimList.Items)), nil } +// getControlPlaneVersion retrieves the current Kubernetes version from the control plane. +// This is used for version skew validation to ensure workers don't run newer versions +// than the control plane, as defined in the version skew policy https://kubernetes.io/releases/version-skew-policy/. +func (r *KarpenterMachinePoolReconciler) getControlPlaneVersion(ctx context.Context, cluster *capi.Cluster) (string, error) { + if cluster.Spec.ControlPlaneRef == nil { + return "", fmt.Errorf("cluster has no control plane reference") + } + + groupVersionKind := schema.GroupVersionKind{ + Group: cluster.Spec.ControlPlaneRef.GroupVersionKind().Group, + Version: cluster.Spec.ControlPlaneRef.GroupVersionKind().Version, + Kind: cluster.Spec.ControlPlaneRef.GroupVersionKind().Kind, + } + controlPlane := &unstructured.Unstructured{} + controlPlane.SetGroupVersionKind(groupVersionKind) + controlPlane.SetName(cluster.Spec.ControlPlaneRef.Name) + controlPlane.SetNamespace(cluster.Spec.ControlPlaneRef.Namespace) + + if err := r.client.Get(ctx, client.ObjectKey{Name: cluster.Spec.ControlPlaneRef.Name, Namespace: cluster.Spec.ControlPlaneRef.Namespace}, controlPlane); err != nil { + return "", fmt.Errorf("failed to get control plane %s: %w", cluster.Spec.ControlPlaneRef.Kind, err) + } + + version, found, err := unstructured.NestedString(controlPlane.Object, "status", "version") + if err != nil { + return "", fmt.Errorf("failed to get current k8s version from control plane: %w", err) + } + if !found { + return "", fmt.Errorf("version not found in control plane spec") + } + + return version, nil +} + +// createOrUpdateKarpenterResources creates or updates the Karpenter NodePool and EC2NodeClass custom resources in the workload cluster. +func (r *KarpenterMachinePoolReconciler) createOrUpdateKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool, machinePool *capiexp.MachinePool) error { + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) + if err != nil { + return fmt.Errorf("failed to get workload cluster client: %w", err) + } + + // Create or update EC2NodeClass + if err := r.createOrUpdateEC2NodeClass(ctx, logger, workloadClusterClient, awsCluster, karpenterMachinePool); err != nil { + conditions.MarkEC2NodeClassNotCreated(karpenterMachinePool, conditions.EC2NodeClassCreationFailedReason, fmt.Sprintf("%v", err)) + return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) + } + conditions.MarkEC2NodeClassCreated(karpenterMachinePool) + + // Create or update NodePool + if err := r.createOrUpdateNodePool(ctx, logger, workloadClusterClient, cluster, karpenterMachinePool); err != nil { + conditions.MarkNodePoolNotCreated(karpenterMachinePool, conditions.NodePoolCreationFailedReason, fmt.Sprintf("%v", err)) + return fmt.Errorf("failed to create or update NodePool: %w", err) + } + conditions.MarkNodePoolCreated(karpenterMachinePool) + + return nil +} + +// createOrUpdateEC2NodeClass creates or updates the EC2NodeClass resource in the workload cluster +// EC2NodeClass defines the EC2-specific configuration for nodes that Karpenter will provision, +// including AMI selection, instance profiles, security groups, and user data. +func (r *KarpenterMachinePoolReconciler) createOrUpdateEC2NodeClass(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, awsCluster *capa.AWSCluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { + ec2NodeClassGVR := schema.GroupVersionResource{ + Group: EC2NodeClassAPIGroup, + Version: "v1", + Resource: "ec2nodeclasses", + } + + ec2NodeClass := &unstructured.Unstructured{} + ec2NodeClass.SetGroupVersionKind(ec2NodeClassGVR.GroupVersion().WithKind("EC2NodeClass")) + ec2NodeClass.SetName(karpenterMachinePool.Name) + ec2NodeClass.SetNamespace("") + ec2NodeClass.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) + + // Generate Ignition user data + userData := r.generateUserData(awsCluster.Spec.S3Bucket.Name, karpenterMachinePool.Name) + + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, ec2NodeClass, func() error { + spec := map[string]interface{}{ + "amiFamily": "Custom", + "amiSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.AMISelectorTerms, + "blockDeviceMappings": karpenterMachinePool.Spec.EC2NodeClass.BlockDeviceMappings, + "instanceProfile": karpenterMachinePool.Spec.EC2NodeClass.InstanceProfile, + "metadataOptions": map[string]interface{}{ + "httpPutResponseHopLimit": karpenterMachinePool.Spec.EC2NodeClass.MetadataOptions.HTTPPutResponseHopLimit, + "httpTokens": karpenterMachinePool.Spec.EC2NodeClass.MetadataOptions.HTTPTokens, + }, + "securityGroupSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SecurityGroupSelectorTerms, + "subnetSelectorTerms": karpenterMachinePool.Spec.EC2NodeClass.SubnetSelectorTerms, + "userData": userData, + "tags": mergeMaps(awsCluster.Spec.AdditionalTags, karpenterMachinePool.Spec.EC2NodeClass.Tags), + } + + ec2NodeClass.Object["spec"] = spec + return nil + }) + + if err != nil { + return fmt.Errorf("failed to create or update EC2NodeClass: %w", err) + } + + switch operation { + case controllerutil.OperationResultCreated: + logger.Info("Created EC2NodeClass in workload cluster") + case controllerutil.OperationResultUpdated: + logger.Info("Updated EC2NodeClass in workload cluster") + } + + return nil +} + +// mergeMaps combines multiple maps, with later maps taking precedence for duplicate keys +func mergeMaps[A comparable, B any](maps ...map[A]B) map[A]B { + result := make(map[A]B) + for _, m := range maps { + for k, v := range m { + result[k] = v + } + } + return result +} + +// createOrUpdateNodePool creates or updates the NodePool resource in the workload cluster. +// NodePool defines the desired state and constraints for nodes that Karpenter should provision, +// including resource limits, disruption policies, and node requirements. +func (r *KarpenterMachinePoolReconciler) createOrUpdateNodePool(ctx context.Context, logger logr.Logger, workloadClusterClient client.Client, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { + nodePoolGVR := schema.GroupVersionResource{ + Group: "karpenter.sh", + Version: "v1", + Resource: "nodepools", + } + + nodePool := &unstructured.Unstructured{} + nodePool.SetGroupVersionKind(nodePoolGVR.GroupVersion().WithKind("NodePool")) + nodePool.SetName(karpenterMachinePool.Name) + nodePool.SetNamespace("") + nodePool.SetLabels(map[string]string{"app.kubernetes.io/managed-by": "aws-resolver-rules-operator"}) + + operation, err := controllerutil.CreateOrUpdate(ctx, workloadClusterClient, nodePool, func() error { + spec := map[string]interface{}{ + "template": map[string]interface{}{ + "metadata": map[string]interface{}{}, + "spec": map[string]interface{}{ + "startupTaints": []interface{}{ + map[string]interface{}{ + "effect": "NoExecute", + "key": "node.cilium.io/agent-not-ready", + "value": "true", + }, + map[string]interface{}{ + "effect": "NoExecute", + "key": "node.cluster.x-k8s.io/uninitialized", + "value": "true", + }, + }, + "nodeClassRef": map[string]interface{}{ + "group": EC2NodeClassAPIGroup, + "kind": "EC2NodeClass", + "name": karpenterMachinePool.Name, + }, + }, + }, + "disruption": map[string]interface{}{}, + } + + if karpenterMachinePool.Spec.NodePool != nil { + dis := spec["disruption"].(map[string]interface{}) + dis["budgets"] = karpenterMachinePool.Spec.NodePool.Disruption.Budgets + dis["consolidateAfter"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidateAfter + dis["consolidationPolicy"] = karpenterMachinePool.Spec.NodePool.Disruption.ConsolidationPolicy + + if karpenterMachinePool.Spec.NodePool.Limits != nil { + spec["limits"] = karpenterMachinePool.Spec.NodePool.Limits + } + + if karpenterMachinePool.Spec.NodePool.Weight != nil { + spec["weight"] = *karpenterMachinePool.Spec.NodePool.Weight + } + + templateMetadata := spec["template"].(map[string]interface{})["metadata"].(map[string]interface{}) + templateMetadata["labels"] = karpenterMachinePool.Spec.NodePool.Template.ObjectMeta.Labels + + templateSpec := spec["template"].(map[string]interface{})["spec"].(map[string]interface{}) + + templateSpec["taints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Taints + templateSpec["startupTaints"] = karpenterMachinePool.Spec.NodePool.Template.Spec.StartupTaints + templateSpec["requirements"] = karpenterMachinePool.Spec.NodePool.Template.Spec.Requirements + templateSpec["expireAfter"] = karpenterMachinePool.Spec.NodePool.Template.Spec.ExpireAfter + + if karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod != nil { + templateSpec["terminationGracePeriod"] = karpenterMachinePool.Spec.NodePool.Template.Spec.TerminationGracePeriod + } + } + + nodePool.Object["spec"] = spec + + return nil + }) + + if err != nil { + return fmt.Errorf("failed to create or update NodePool: %w", err) + } + + switch operation { + case controllerutil.OperationResultCreated: + logger.Info("Created NodePool in workload cluster") + case controllerutil.OperationResultUpdated: + logger.Info("Updated NodePool in workload cluster") + } + + return nil +} + +// deleteKarpenterResources deletes the Karpenter NodePool and EC2NodeClass resources from the workload cluster. +func (r *KarpenterMachinePoolReconciler) deleteKarpenterResources(ctx context.Context, logger logr.Logger, cluster *capi.Cluster, karpenterMachinePool *v1alpha1.KarpenterMachinePool) error { + workloadClusterClient, err := r.clusterClientGetter(ctx, "", r.client, client.ObjectKeyFromObject(cluster)) + if err != nil { + return fmt.Errorf("failed to get workload cluster client: %w", err) + } + + // Delete NodePool + nodePoolGVR := schema.GroupVersionResource{ + Group: "karpenter.sh", + Version: "v1", + Resource: "nodepools", + } + + nodePool := &unstructured.Unstructured{} + nodePool.SetGroupVersionKind(nodePoolGVR.GroupVersion().WithKind("NodePool")) + nodePool.SetName(karpenterMachinePool.Name) + nodePool.SetNamespace("default") + + if err := workloadClusterClient.Delete(ctx, nodePool); err != nil && !k8serrors.IsNotFound(err) && !meta.IsNoMatchError(err) { + logger.Error(err, "failed to delete NodePool", "name", karpenterMachinePool.Name) + return fmt.Errorf("failed to delete NodePool: %w", err) + } + + // Delete EC2NodeClass + ec2NodeClassGVR := schema.GroupVersionResource{ + Group: EC2NodeClassAPIGroup, + Version: "v1", + Resource: "ec2nodeclasses", + } + + ec2NodeClass := &unstructured.Unstructured{} + ec2NodeClass.SetGroupVersionKind(ec2NodeClassGVR.GroupVersion().WithKind("EC2NodeClass")) + ec2NodeClass.SetName(karpenterMachinePool.Name) + ec2NodeClass.SetNamespace("default") + + if err := workloadClusterClient.Delete(ctx, ec2NodeClass); err != nil && !k8serrors.IsNotFound(err) && !meta.IsNoMatchError(err) { + logger.Error(err, "failed to delete EC2NodeClass", "name", karpenterMachinePool.Name) + return fmt.Errorf("failed to delete EC2NodeClass: %w", err) + } + + return nil +} + +// generateUserData generates the user data for Ignition configuration. +func (r *KarpenterMachinePoolReconciler) generateUserData(s3bucketName, karpenterMachinePoolName string) string { + userData := map[string]interface{}{ + "ignition": map[string]interface{}{ + "config": map[string]interface{}{ + "merge": []map[string]interface{}{ + { + "source": fmt.Sprintf("s3://%s/%s/%s", s3bucketName, S3ObjectPrefix, karpenterMachinePoolName), + "verification": map[string]interface{}{}, + }, + }, + "replace": map[string]interface{}{ + "verification": map[string]interface{}{}, + }, + }, + "proxy": map[string]interface{}{}, + "security": map[string]interface{}{ + "tls": map[string]interface{}{}, + }, + "timeouts": map[string]interface{}{}, + "version": "3.4.0", + }, + "kernelArguments": map[string]interface{}{}, + "passwd": map[string]interface{}{}, + "storage": map[string]interface{}{}, + "systemd": map[string]interface{}{}, + } + + userDataBytes, _ := json.Marshal(userData) + return string(userDataBytes) +} + // SetupWithManager sets up the controller with the Manager. func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { logger := capalogger.FromContext(ctx).GetLogger() @@ -290,3 +684,21 @@ func (r *KarpenterMachinePoolReconciler) SetupWithManager(ctx context.Context, m WithEventFilter(predicates.ResourceNotPaused(logger)). Complete(r) } + +// IsVersionSkewAllowed checks if the worker version can be updated based on the control plane version. +// The workers can't use a newer k8s version than the one used by the control plane. +// +// This implements Kubernetes version skew policy https://kubernetes.io/releases/version-skew-policy/ +func (r *KarpenterMachinePoolReconciler) IsVersionSkewAllowed(ctx context.Context, cluster *capi.Cluster, machinePool *capiexp.MachinePool) (bool, string, string, error) { + controlPlaneVersion, err := r.getControlPlaneVersion(ctx, cluster) + if err != nil { + return true, "", "", fmt.Errorf("failed to get current Control Plane k8s version: %w", err) + } + + allowed, err := versionskew.IsSkewAllowed(controlPlaneVersion, *machinePool.Spec.Template.Spec.Version) + if err != nil { + return true, controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, fmt.Errorf("failed to validate version skew: %w", err) + } + + return allowed, controlPlaneVersion, *machinePool.Spec.Template.Spec.Version, nil +} diff --git a/controllers/karpentermachinepool_controller_test.go b/controllers/karpentermachinepool_controller_test.go index 4403aa5a..24e6b067 100644 --- a/controllers/karpentermachinepool_controller_test.go +++ b/controllers/karpentermachinepool_controller_test.go @@ -9,7 +9,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gstruct" + gomegatypes "github.com/onsi/gomega/types" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -17,11 +20,10 @@ import ( "k8s.io/kubectl/pkg/scheme" capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" capi "sigs.k8s.io/cluster-api/api/v1beta1" - "sigs.k8s.io/cluster-api/controllers/remote" capiexp "sigs.k8s.io/cluster-api/exp/api/v1beta1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/envtest/komega" "sigs.k8s.io/controller-runtime/pkg/reconcile" karpenterinfra "github.com/aws-resolver-rules-operator/api/v1alpha1" @@ -30,6 +32,28 @@ import ( "github.com/aws-resolver-rules-operator/pkg/resolver/resolverfakes" ) +const ( + AMIName = "flatcar-stable-4152.2.3-kube-1.29.1-tooling-1.26.0-gs" + AMIOwner = "1234567890" + AWSRegion = "eu-west-1" + ClusterName = "foo" + AWSClusterBucketName = "my-awesome-bucket" + DataSecretName = "foo-mp-12345" + KarpenterMachinePoolName = "foo" + KarpenterNodesInstanceProfile = "karpenter-iam-role" + KubernetesVersion = "v1.29.1" +) + +// findCondition returns the condition with the given type from the list of conditions. +func findCondition(conditions capi.Conditions, conditionType string) *capi.Condition { + for i := range conditions { + if conditions[i].Type == capi.ConditionType(conditionType) { + return &conditions[i] + } + } + return nil +} + var _ = Describe("KarpenterMachinePool reconciler", func() { var ( capiBootstrapSecretContent []byte @@ -37,23 +61,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { dataSecretName string s3Client *resolverfakes.FakeS3Client ec2Client *resolverfakes.FakeEC2Client - fakeCtrlClient client.Client - fakeClusterClientGetter remote.ClusterClientGetter ctx context.Context + instanceProfile = KarpenterNodesInstanceProfile reconciler *controllers.KarpenterMachinePoolReconciler reconcileErr error reconcileResult reconcile.Result ) - const ( - ClusterName = "foo" - AWSClusterBucketName = "my-awesome-bucket" - DataSecretName = "foo-mp-12345" - KarpenterMachinePoolName = "foo" - KarpenterMachinePoolNamespace = "org-bar" - KubernetesVersion = "v1.29.1" - ) - BeforeEach(func() { ctx = context.Background() @@ -72,15 +86,9 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { err = karpenterinfra.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) - fakeCtrlClient = fake.NewClientBuilder(). - WithScheme(scheme.Scheme). - WithStatusSubresource(&karpenterinfra.KarpenterMachinePool{}). - Build() - - // Use the default fake cluster client getter - fakeClusterClientGetter = func(ctx context.Context, _ string, _ client.Client, _ client.ObjectKey) (client.Client, error) { + workloadClusterClientGetter := func(ctx context.Context, _ string, _ client.Client, _ client.ObjectKey) (client.Client, error) { // Return the same client that we're using for the test - return fakeCtrlClient, nil + return k8sClient, nil } s3Client = new(resolverfakes.FakeS3Client) @@ -91,13 +99,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { EC2Client: ec2Client, } - reconciler = controllers.NewKarpenterMachinepoolReconciler(fakeCtrlClient, fakeClusterClientGetter, clientsFactory) + reconciler = controllers.NewKarpenterMachinepoolReconciler(k8sClient, workloadClusterClientGetter, clientsFactory) }) JustBeforeEach(func() { request := ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, }, } @@ -114,11 +122,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, }, } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) + err := k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) It("does nothing", func() { @@ -128,34 +136,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { When("the KarpenterMachinePool is being deleted", func() { BeforeEach(func() { - karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: KarpenterMachinePoolName, - Labels: map[string]string{ - capi.ClusterNameLabel: ClusterName, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "MachinePool", - Name: KarpenterMachinePoolName, - }, - }, - Finalizers: []string{controllers.KarpenterFinalizer}, - }, - } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - - err = fakeCtrlClient.Delete(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - dataSecretName = DataSecretName version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -170,7 +155,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, @@ -178,7 +163,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -187,78 +172,141 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err = fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + Finalizers: []string{controllers.KarpenterFinalizer}, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + err = k8sClient.Delete(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) awsCluster := &capa.AWSCluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, }, Spec: capa.AWSClusterSpec{ IdentityRef: &capa.AWSIdentityReference{ - Name: "default", + Name: "default-delete-test", Kind: capa.ClusterRoleIdentityKind, }, S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, }, } - err = fakeCtrlClient.Create(ctx, awsCluster) + err = k8sClient.Create(ctx, awsCluster) Expect(err).NotTo(HaveOccurred()) clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-kubeconfig", ClusterName), }, } - err = fakeCtrlClient.Create(ctx, clusterKubeconfigSecret) + err = k8sClient.Create(ctx, clusterKubeconfigSecret) Expect(err).NotTo(HaveOccurred()) awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ ObjectMeta: metav1.ObjectMeta{ - Name: "default", + Name: "default-delete-test", + }, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, }, - Spec: capa.AWSClusterRoleIdentitySpec{}, } - err = fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) + err = k8sClient.Create(ctx, awsClusterRoleIdentity) Expect(err).NotTo(HaveOccurred()) bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"value": capiBootstrapSecretContent}, } - err = fakeCtrlClient.Create(ctx, bootstrapSecret) + err = k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) When("the owner cluster is also being deleted", func() { BeforeEach(func() { + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": "v1.21.2", + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err := k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, }, - Finalizers: []string{"something-to-keep-it-around-when-deleting"}, + Finalizers: []string{"giantswarm.io/something-to-keep-it-around-when-deleting"}, }, Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, + Topology: nil, }, } - err := fakeCtrlClient.Create(ctx, cluster) + err = k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) - err = fakeCtrlClient.Delete(ctx, cluster) + err = k8sClient.Delete(ctx, cluster) Expect(err).NotTo(HaveOccurred()) }) // This test is a bit cumbersome because we are deleting CRs, so we can't use different `It` blocks or the CRs would be gone. @@ -277,7 +325,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Expect(reconcileResult.RequeueAfter).To(Equal(30 * time.Second)) karpenterMachinePoolList := &karpenterinfra.KarpenterMachinePoolList{} - err := fakeCtrlClient.List(ctx, karpenterMachinePoolList) + err := k8sClient.List(ctx, karpenterMachinePoolList, client.InNamespace(namespace)) Expect(err).NotTo(HaveOccurred()) // Finalizer should be there blocking the deletion of the CR Expect(karpenterMachinePoolList.Items).To(HaveLen(1)) @@ -286,13 +334,13 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { reconcileResult, reconcileErr = reconciler.Reconcile(ctx, ctrl.Request{ NamespacedName: types.NamespacedName{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, }, }) karpenterMachinePoolList = &karpenterinfra.KarpenterMachinePoolList{} - err = fakeCtrlClient.List(ctx, karpenterMachinePoolList) + err = k8sClient.List(ctx, karpenterMachinePoolList, client.InNamespace(namespace)) Expect(err).NotTo(HaveOccurred()) // Finalizer should've been removed and the CR should be gone Expect(karpenterMachinePoolList.Items).To(HaveLen(0)) @@ -302,27 +350,67 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) When("the KarpenterMachinePool exists and it has a MachinePool owner", func() { - BeforeEach(func() { - karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: KarpenterMachinePoolName, - Labels: map[string]string{ - capi.ClusterNameLabel: ClusterName, + When("the referenced MachinePool does not exist", func() { + BeforeEach(func() { + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: "12345678-1234-1234-1234-123456789012", + }, + }, }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "MachinePool", - Name: KarpenterMachinePoolName, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Name: AMIName, + Owner: AMIOwner, + }, + }, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"my-target-sg": "is-this"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"my-target-subnet": "is-that"}, + }, + }, + }, + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, }, }, - }, - } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - }) - When("the referenced MachinePool does not exist", func() { + } + err := k8sClient.Create(ctx, karpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + }) It("returns an error", func() { Expect(reconcileErr).To(MatchError(ContainSubstring("failed to get MachinePool owning the KarpenterMachinePool"))) }) @@ -332,7 +420,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -340,22 +428,21 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capiexp.MachinePoolSpec{ ClusterName: ClusterName, - // Replicas: nil, Template: capi.MachineTemplateSpec{ ObjectMeta: capi.ObjectMeta{}, Spec: capi.MachineSpec{ - ClusterName: "", + ClusterName: ClusterName, Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -364,11 +451,59 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err := fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + }, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) It("returns early", func() { Expect(reconcileErr).NotTo(HaveOccurred()) + + updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("BootstrapDataReady", v1.ConditionFalse, "BootstrapDataSecretMissingReference", "Bootstrap data secret reference is not yet available in MachinePool")) }) }) When("the referenced MachinePool exists and MachinePool.spec.template.spec.bootstrap.dataSecretName is set", func() { @@ -377,7 +512,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { version := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -393,7 +528,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, @@ -401,7 +536,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, @@ -410,7 +545,116 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, }, } - err := fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + terminationGracePeriod := metav1.Duration{Duration: 30 * time.Second} + weight := int32(1) + deviceName := "/dev/xvda" + volumeSize := resource.MustParse("8Gi") + volumeTypeGp3 := "gp3" + deleteOnTerminationTrue := true + hopLimit := int64(5) + metadataOptionsRequired := "required" + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Name: AMIName, + Owner: AMIOwner, + }, + }, + BlockDeviceMappings: []*karpenterinfra.BlockDeviceMapping{ + { + DeviceName: &deviceName, + EBS: &karpenterinfra.BlockDevice{ + DeleteOnTermination: &deleteOnTerminationTrue, + VolumeSize: &volumeSize, + VolumeType: &volumeTypeGp3, + }, + RootVolume: true, + }, + }, + InstanceProfile: &instanceProfile, + MetadataOptions: &karpenterinfra.MetadataOptions{ + HTTPPutResponseHopLimit: &hopLimit, + HTTPTokens: &metadataOptionsRequired, + }, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"my-target-sg": "is-this"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"my-target-subnet": "is-that"}, + }, + }, + Tags: map[string]string{ + "one-tag": "only-for-karpenter", + }, + }, + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + ExpireAfter: karpenterinfra.MustParseNillableDuration("24h"), + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + StartupTaints: []v1.Taint{ + { + Key: "karpenter.sh/test-startup-taint", + Value: "test-taint-value", + Effect: v1.TaintEffectNoSchedule, + }, + }, + Taints: []v1.Taint{ + { + Key: "karpenter.sh/test-taint", + Value: "test-taint-value", + Effect: v1.TaintEffectNoSchedule, + }, + }, + TerminationGracePeriod: &terminationGracePeriod, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + Limits: map[v1.ResourceName]resource.Quantity{ + v1.ResourceCPU: resource.MustParse("1000m"), + v1.ResourceMemory: resource.MustParse("1000Mi"), + }, + Weight: &weight, + }, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) When("there is no Cluster owning the MachinePool", func() { @@ -422,7 +666,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -430,24 +674,56 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capi.ClusterSpec{ Paused: true, + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, }, } - err := fakeCtrlClient.Create(ctx, cluster) + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": "v1.21.2", + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) clusterKubeconfigSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-kubeconfig", ClusterName), }, } - err = fakeCtrlClient.Create(ctx, clusterKubeconfigSecret) + err = k8sClient.Create(ctx, clusterKubeconfigSecret) Expect(err).NotTo(HaveOccurred()) }) It("returns early", func() { @@ -459,22 +735,54 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, }, }, Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, }, } - err := fakeCtrlClient.Create(ctx, cluster) + err := k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": "v1.21.2", + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) }) When("there is no AWSCluster", func() { @@ -486,7 +794,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { awsCluster := &capa.AWSCluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -494,7 +802,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capa.AWSClusterSpec{}, } - err := fakeCtrlClient.Create(ctx, awsCluster) + err := k8sClient.Create(ctx, awsCluster) Expect(err).NotTo(HaveOccurred()) }) It("returns an error", func() { @@ -502,55 +810,82 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) }) When("the AWSCluster exists and there is a S3 bucket defined on it", func() { - BeforeEach(func() { - awsCluster := &capa.AWSCluster{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: ClusterName, - }, - Spec: capa.AWSClusterSpec{ - IdentityRef: &capa.AWSIdentityReference{ - Name: "default", - Kind: capa.ClusterRoleIdentityKind, - }, - S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, - }, - } - err := fakeCtrlClient.Create(ctx, awsCluster) - Expect(err).NotTo(HaveOccurred()) - }) When("it can't find the identity used by the AWSCluster", func() { + BeforeEach(func() { + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + IdentityRef: &capa.AWSIdentityReference{ + Name: "not-referenced-by-test", + Kind: capa.ClusterRoleIdentityKind, + }, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err := k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + }) It("returns an error", func() { Expect(reconcileErr).To(MatchError(ContainSubstring("failed to get AWSClusterRoleIdentity referenced in AWSCluster"))) }) }) When("it finds the identity used by the AWSCluster", func() { BeforeEach(func() { + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + AdditionalTags: map[string]string{ + "additional-tag-for-all-resources": "custom-tag", + }, + IdentityRef: &capa.AWSIdentityReference{ + Name: "default", + Kind: capa.ClusterRoleIdentityKind, + }, + Region: AWSRegion, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err := k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ ObjectMeta: metav1.ObjectMeta{ Name: "default", }, - Spec: capa.AWSClusterRoleIdentitySpec{}, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, } - err := fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) - Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) }) When("the bootstrap secret referenced in the dataSecretName field does not exist", func() { It("returns an error", func() { - Expect(reconcileErr).To(MatchError(ContainSubstring("bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName is not found"))) + Expect(reconcileErr).To(MatchError(ContainSubstring("failed to get bootstrap secret in MachinePool.spec.template.spec.bootstrap.dataSecretName"))) }) }) When("the bootstrap secret exists but it does not contain the 'value' key", func() { BeforeEach(func() { bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"not-what-we-expect": capiBootstrapSecretContent}, } - err := fakeCtrlClient.Create(ctx, bootstrapSecret) + err := k8sClient.Create(ctx, bootstrapSecret) Expect(err).NotTo(HaveOccurred()) }) It("returns an error", func() { @@ -561,18 +896,129 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { BeforeEach(func() { bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"value": capiBootstrapSecretContent}, } - err := fakeCtrlClient.Create(ctx, bootstrapSecret) + err := k8sClient.Create(ctx, bootstrapSecret) + Expect(err).NotTo(HaveOccurred()) + }) + It("creates karpenter EC2NodeClass object in workload cluster", func() { + Expect(reconcileErr).NotTo(HaveOccurred()) + + ec2nodeclassList := &unstructured.UnstructuredList{} + ec2nodeclassList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: controllers.EC2NodeClassAPIGroup, + Kind: "EC2NodeClassList", + Version: "v1", + }) + + err := k8sClient.List(ctx, ec2nodeclassList) + Expect(err).NotTo(HaveOccurred()) + Expect(ec2nodeclassList.Items).To(HaveLen(1)) + Expect(ec2nodeclassList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "userData").To(Equal(fmt.Sprintf("{\"ignition\":{\"config\":{\"merge\":[{\"source\":\"s3://%s/karpenter-machine-pool/%s\",\"verification\":{}}],\"replace\":{\"verification\":{}}},\"proxy\":{},\"security\":{\"tls\":{}},\"timeouts\":{},\"version\":\"3.4.0\"},\"kernelArguments\":{},\"passwd\":{},\"storage\":{},\"systemd\":{}}", AWSClusterBucketName, KarpenterMachinePoolName))) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "instanceProfile").To(Equal(KarpenterNodesInstanceProfile)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "tags"). + To(HaveKeyWithValue("additional-tag-for-all-resources", "custom-tag")) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "tags"). + To(HaveKeyWithValue("one-tag", "only-for-karpenter")) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "blockDeviceMappings").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "blockDeviceMappings").To( + ContainElement( // slice matcher: at least one element matches + gstruct.MatchAllKeys(gstruct.Keys{ // map matcher: all these keys must match exactly + "deviceName": Equal("/dev/xvda"), + "rootVolume": BeTrue(), + "ebs": gstruct.MatchAllKeys(gstruct.Keys{ + "deleteOnTermination": BeTrue(), + "volumeSize": Equal("8Gi"), + "volumeType": Equal("gp3"), + }), + }), + ), + ) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "metadataOptions"). + To(HaveKeyWithValue("httpTokens", "required")) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "metadataOptions"). + To(HaveKeyWithValue("httpPutResponseHopLimit", int64(5))) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "amiSelectorTerms").To( + ContainElement( // slice matcher: at least one element matches + gstruct.MatchAllKeys(gstruct.Keys{ // map matcher: all these keys must match exactly + "name": Equal(AMIName), + "owner": Equal(AMIOwner), + }), + ), + ) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "securityGroupSelectorTerms").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "securityGroupSelectorTerms").To( + ConsistOf( + gstruct.MatchAllKeys(gstruct.Keys{ + // the top-level map has a single "tags" field, + // whose value itself must be a map containing our SG name → value + "tags": gstruct.MatchAllKeys(gstruct.Keys{ + "my-target-sg": Equal("is-this"), + }), + }), + ), + ) + + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "subnetSelectorTerms").To(HaveLen(1)) + ExpectUnstructured(ec2nodeclassList.Items[0], "spec", "subnetSelectorTerms").To( + ConsistOf( + gstruct.MatchAllKeys(gstruct.Keys{ + "tags": gstruct.MatchAllKeys(gstruct.Keys{ + "my-target-subnet": Equal("is-that"), + }), + }), + ), + ) + }) + It("creates karpenter NodePool object in workload cluster", func() { + nodepoolList := &unstructured.UnstructuredList{} + nodepoolList.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "karpenter.sh", + Kind: "NodePoolList", + Version: "v1", + }) + + err := k8sClient.List(ctx, nodepoolList) Expect(err).NotTo(HaveOccurred()) + Expect(nodepoolList.Items).To(HaveLen(1)) + Expect(nodepoolList.Items[0].GetName()).To(Equal(KarpenterMachinePoolName)) + + ExpectUnstructured(nodepoolList.Items[0], "spec", "disruption", "consolidateAfter").To(Equal("30s")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "disruption", "consolidationPolicy").To(BeEquivalentTo(karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized)) + ExpectUnstructured(nodepoolList.Items[0], "spec", "limits").To(HaveKeyWithValue("cpu", "1")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "limits").To(HaveKeyWithValue("memory", "1000Mi")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "weight").To(BeEquivalentTo(int64(1))) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "expireAfter").To(BeEquivalentTo("24h")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "terminationGracePeriod").To(BeEquivalentTo("30s")) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "startupTaints").To(BeEquivalentTo([]interface{}{ + map[string]interface{}{ + "key": "karpenter.sh/test-startup-taint", + "value": "test-taint-value", + "effect": "NoSchedule", + }, + })) + ExpectUnstructured(nodepoolList.Items[0], "spec", "template", "spec", "taints").To(BeEquivalentTo([]interface{}{ + map[string]interface{}{ + "key": "karpenter.sh/test-taint", + "value": "test-taint-value", + "effect": "NoSchedule", + }, + })) }) It("adds the finalizer to the KarpenterMachinePool", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.GetFinalizers()).To(ContainElement(controllers.KarpenterFinalizer)) }) @@ -587,22 +1033,10 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { It("writes annotation containing bootstrap data hash", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) Expect(updatedKarpenterMachinePool.Annotations).To(HaveKeyWithValue(controllers.BootstrapDataHashAnnotation, Equal(capiBootstrapSecretHash))) }) - When("there are no NodeClaim in the workload cluster yet", func() { - It("requeues to try again soon", func() { - Expect(reconcileErr).NotTo(HaveOccurred()) - Expect(reconcileResult.RequeueAfter).To(Equal(1 * time.Minute)) - updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) - Expect(err).NotTo(HaveOccurred()) - Expect(updatedKarpenterMachinePool.Status.Ready).To(BeFalse()) - Expect(updatedKarpenterMachinePool.Status.Replicas).To(BeZero()) - Expect(updatedKarpenterMachinePool.Spec.ProviderIDList).To(BeEmpty()) - }) - }) When("there are NodeClaim resources in the workload cluster", func() { BeforeEach(func() { nodeClaim1 := &unstructured.Unstructured{} @@ -610,9 +1044,19 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { "metadata": map[string]interface{}{ "name": fmt.Sprintf("%s-z9y8x", KarpenterMachinePoolName), }, - "spec": map[string]interface{}{}, - "status": map[string]interface{}{ - "providerID": "aws:///us-west-2a/i-1234567890abcdef0", + "spec": map[string]interface{}{ + "nodeClassRef": map[string]interface{}{ + "group": "karpenter.k8s.aws", + "kind": "EC2NodeClass", + "name": "default", + }, + "requirements": []interface{}{ + map[string]interface{}{ + "key": "kubernetes.io/arch", + "operator": "In", + "values": []string{"amd64"}, + }, + }, }, } nodeClaim1.SetGroupVersionKind(schema.GroupVersionKind{ @@ -620,7 +1064,11 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "NodeClaim", Version: "v1", }) - err := fakeCtrlClient.Create(ctx, nodeClaim1) + err := k8sClient.Create(ctx, nodeClaim1) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(nodeClaim1.Object, map[string]interface{}{"providerID": "aws:///us-west-2a/i-1234567890abcdef0"}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, nodeClaim1) Expect(err).NotTo(HaveOccurred()) nodeClaim2 := &unstructured.Unstructured{} @@ -628,9 +1076,19 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { "metadata": map[string]interface{}{ "name": fmt.Sprintf("%s-m0n1o", KarpenterMachinePoolName), }, - "spec": map[string]interface{}{}, - "status": map[string]interface{}{ - "providerID": "aws:///us-west-2a/i-09876543219fedcba", + "spec": map[string]interface{}{ + "nodeClassRef": map[string]interface{}{ + "group": "karpenter.k8s.aws", + "kind": "EC2NodeClass", + "name": "default", + }, + "requirements": []interface{}{ + map[string]interface{}{ + "key": "kubernetes.io/arch", + "operator": "In", + "values": []string{"amd64"}, + }, + }, }, } nodeClaim2.SetGroupVersionKind(schema.GroupVersionKind{ @@ -638,15 +1096,24 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Kind: "NodeClaim", Version: "v1", }) - err = fakeCtrlClient.Create(ctx, nodeClaim2) + err = k8sClient.Create(ctx, nodeClaim2) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(nodeClaim2.Object, map[string]interface{}{"providerID": "aws:///us-west-2a/i-09876543219fedcba"}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, nodeClaim2) Expect(err).NotTo(HaveOccurred()) }) It("updates the KarpenterMachinePool spec and status accordingly", func() { Expect(reconcileErr).NotTo(HaveOccurred()) updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} - err := fakeCtrlClient.Get(ctx, types.NamespacedName{Namespace: KarpenterMachinePoolNamespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) Expect(err).NotTo(HaveOccurred()) + + // Check that the Ready condition is True + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("Ready", v1.ConditionTrue, "Ready", "")) Expect(updatedKarpenterMachinePool.Status.Ready).To(BeTrue()) + + // Check karpenter machine pool spec and status Expect(updatedKarpenterMachinePool.Status.Replicas).To(Equal(int32(2))) Expect(updatedKarpenterMachinePool.Spec.ProviderIDList).To(ContainElements("aws:///us-west-2a/i-1234567890abcdef0", "aws:///us-west-2a/i-09876543219fedcba")) }) @@ -668,33 +1135,448 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }) When("the KarpenterMachinePool exists with a hash annotation signaling unchanged bootstrap data", func() { BeforeEach(func() { - karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + dataSecretName := DataSecretName + kubernetesVersion := KubernetesVersion + machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, - Annotations: map[string]string{ - controllers.BootstrapDataHashAnnotation: capiBootstrapSecretHash, - }, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: "cluster.x-k8s.io/v1beta1", - Kind: "MachinePool", - Name: KarpenterMachinePoolName, + }, + Spec: capiexp.MachinePoolSpec{ + ClusterName: ClusterName, + Template: capi.MachineTemplateSpec{ + ObjectMeta: capi.ObjectMeta{}, + Spec: capi.MachineSpec{ + ClusterName: ClusterName, + Bootstrap: capi.Bootstrap{ + ConfigRef: &v1.ObjectReference{ + Kind: "KubeadmConfig", + Namespace: namespace, + Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), + APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", + }, + DataSecretName: &dataSecretName, + }, + InfrastructureRef: v1.ObjectReference{ + Kind: "KarpenterMachinePool", + Namespace: namespace, + Name: KarpenterMachinePoolName, + APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", + }, + Version: &kubernetesVersion, }, }, }, } - err := fakeCtrlClient.Create(ctx, karpenterMachinePool) + err := k8sClient.Create(ctx, machinePool) Expect(err).NotTo(HaveOccurred()) - dataSecretName := DataSecretName - version := KubernetesVersion + Eventually(komega.Get(machinePool), time.Second*10, time.Millisecond*250).Should(Succeed()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Annotations: map[string]string{ + controllers.BootstrapDataHashAnnotation: capiBootstrapSecretHash, + }, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Name: AMIName, + Owner: AMIOwner, + }, + }, + InstanceProfile: &instanceProfile, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"my-target-sg": "is-this"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"my-target-subnet": "is-that"}, + }, + }, + }, + NodePool: &karpenterinfra.NodePoolSpec{ + Template: karpenterinfra.NodeClaimTemplate{ + Spec: karpenterinfra.NodeClaimTemplateSpec{ + Requirements: []karpenterinfra.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/os", + Operator: v1.NodeSelectorOpIn, + Values: []string{"linux"}, + }, + }, + }, + }, + }, + Disruption: karpenterinfra.Disruption{ + ConsolidateAfter: karpenterinfra.MustParseNillableDuration("30s"), + ConsolidationPolicy: karpenterinfra.ConsolidationPolicyWhenEmptyOrUnderutilized, + }, + }, + }, + } + err = k8sClient.Create(ctx, karpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": KubernetesVersion, + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + + cluster := &capi.Cluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, + InfrastructureRef: &v1.ObjectReference{ + Kind: "AWSCluster", + Namespace: namespace, + Name: ClusterName, + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + }, + }, + } + err = k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + clusterKubeconfigSecret := &v1.Secret{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: fmt.Sprintf("%s-kubeconfig", ClusterName), + }, + } + err = k8sClient.Create(ctx, clusterKubeconfigSecret) + Expect(err).NotTo(HaveOccurred()) + + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + IdentityRef: &capa.AWSIdentityReference{ + Name: "default", + Kind: capa.ClusterRoleIdentityKind, + }, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err = k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + + awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, + } + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) + + bootstrapSecret := &v1.Secret{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: DataSecretName, + }, + Data: map[string][]byte{"value": capiBootstrapSecretContent}, + } + err = k8sClient.Create(ctx, bootstrapSecret) + Expect(err).NotTo(HaveOccurred()) + }) + It("doesn't write the user data to S3 again", func() { + Expect(reconcileErr).NotTo(HaveOccurred()) + Expect(s3Client.PutCallCount()).To(Equal(0)) + }) + }) + + When("creating the NodePool fails", func() { + BeforeEach(func() { + dataSecretName = DataSecretName + kubernetesVersion := KubernetesVersion machinePool := &capiexp.MachinePool{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capiexp.MachinePoolSpec{ + ClusterName: ClusterName, + Template: capi.MachineTemplateSpec{ + Spec: capi.MachineSpec{ + ClusterName: ClusterName, + Version: &kubernetesVersion, + Bootstrap: capi.Bootstrap{ + DataSecretName: &dataSecretName, + }, + }, + }, + }, + } + err := k8sClient.Create(ctx, machinePool) + Expect(err).NotTo(HaveOccurred()) + + // Get the created machinePool to access its UID + err = k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, machinePool) + Expect(err).NotTo(HaveOccurred()) + + deviceName := "/dev/xvda" + volumeType := "gp3" + deleteOnTermination := true + volumeSize, _ := resource.ParseQuantity("8Gi") + kmp := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.UID, + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{ + EC2NodeClass: &karpenterinfra.EC2NodeClassSpec{ + InstanceProfile: &instanceProfile, + AMISelectorTerms: []karpenterinfra.AMISelectorTerm{ + { + Alias: "al2@latest", + }, + }, + BlockDeviceMappings: []*karpenterinfra.BlockDeviceMapping{ + { + DeviceName: &deviceName, + RootVolume: true, + EBS: &karpenterinfra.BlockDevice{ + VolumeSize: &volumeSize, + VolumeType: &volumeType, + DeleteOnTermination: &deleteOnTermination, + }, + }, + }, + SecurityGroupSelectorTerms: []karpenterinfra.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"Name": "foo"}, + }, + }, + SubnetSelectorTerms: []karpenterinfra.SubnetSelectorTerm{ + { + Tags: map[string]string{"Name": "foo"}, + }, + }, + Tags: map[string]string{ + "one-tag": "only-for-karpenter", + }, + }, + // NodePool spec omitted to focus on testing condition persistence + // NodePool: nil, + }, + } + err = k8sClient.Create(ctx, kmp) + Expect(err).NotTo(HaveOccurred()) + + // Create Cluster resource + cluster := &capi.Cluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + }, + Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, + InfrastructureRef: &v1.ObjectReference{ + Kind: "AWSCluster", + Namespace: namespace, + Name: ClusterName, + APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", + }, + }, + } + err = k8sClient.Create(ctx, cluster) + Expect(err).NotTo(HaveOccurred()) + + // Create AWSCluster resource + awsCluster := &capa.AWSCluster{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: ClusterName, + }, + Spec: capa.AWSClusterSpec{ + AdditionalTags: map[string]string{ + "additional-tag-for-all-resources": "custom-tag", + }, + IdentityRef: &capa.AWSIdentityReference{ + Name: "default", + Kind: capa.ClusterRoleIdentityKind, + }, + Region: AWSRegion, + S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, + }, + } + err = k8sClient.Create(ctx, awsCluster) + Expect(err).NotTo(HaveOccurred()) + + // Create AWSClusterRoleIdentity resource + awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, + }, + } + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) + + // Create bootstrap secret for successful reconciliation + bootstrapSecret := &v1.Secret{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: DataSecretName, + }, + Data: map[string][]byte{"value": capiBootstrapSecretContent}, + } + err = k8sClient.Create(ctx, bootstrapSecret) + Expect(err).NotTo(HaveOccurred()) + + // Create control plane with same version as machine pool for successful version skew validation + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": KubernetesVersion, + }, + } + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": KubernetesVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + + reconcileResult, reconcileErr = reconciler.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + }, + }) + }) + + It("updates conditions even when reconciliation fails", func() { + Expect(reconcileErr).To(HaveOccurred()) + Expect(reconcileErr.Error()).To(ContainSubstring("failed to create or update NodePool")) + + // Get the updated KarpenterMachinePool + updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + // This condition should be properly persisted even though reconciliation failed + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("EC2NodeClassCreated", v1.ConditionTrue, "EC2NodeClassCreated", "")) + + // Version skew should be valid since we use the same version + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("VersionSkewPolicySatisfied", v1.ConditionTrue, "VersionSkewValid", "")) + + // NodePoolCreated should be False since creation failed + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("NodePoolCreated", v1.ConditionFalse, "NodePoolCreationFailed", "failed to create or update NodePool")) + }) + }) + + When("version skew validation fails (node pool version newer than control plane)", func() { + var controlPlaneVersion = "v1.29.0" + var nodePoolVersion = "v1.30.0" // Newer than control plane - should violate version skew policy + + BeforeEach(func() { + dataSecretName = DataSecretName + machinePool := &capiexp.MachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, Name: KarpenterMachinePoolName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, @@ -702,7 +1584,6 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, Spec: capiexp.MachinePoolSpec{ ClusterName: ClusterName, - // Replicas: nil, Template: capi.MachineTemplateSpec{ ObjectMeta: capi.ObjectMeta{}, Spec: capi.MachineSpec{ @@ -710,7 +1591,7 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { Bootstrap: capi.Bootstrap{ ConfigRef: &v1.ObjectReference{ Kind: "KubeadmConfig", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: fmt.Sprintf("%s-1a2b3c", KarpenterMachinePoolName), APIVersion: "bootstrap.cluster.x-k8s.io/v1beta1", }, @@ -718,85 +1599,198 @@ var _ = Describe("KarpenterMachinePool reconciler", func() { }, InfrastructureRef: v1.ObjectReference{ Kind: "KarpenterMachinePool", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: KarpenterMachinePoolName, APIVersion: "infrastructure.cluster.x-k8s.io/v1alpha1", }, - Version: &version, + Version: &nodePoolVersion, // Newer version than control plane }, }, }, } - err = fakeCtrlClient.Create(ctx, machinePool) + err := k8sClient.Create(ctx, machinePool) Expect(err).NotTo(HaveOccurred()) cluster := &capi.Cluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, Labels: map[string]string{ capi.ClusterNameLabel: ClusterName, }, }, Spec: capi.ClusterSpec{ + ControlPlaneRef: &v1.ObjectReference{ + Kind: "KubeadmControlPlane", + Namespace: namespace, + Name: ClusterName, + APIVersion: "controlplane.cluster.x-k8s.io/v1beta1", + }, InfrastructureRef: &v1.ObjectReference{ Kind: "AWSCluster", - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, APIVersion: "infrastructure.cluster.x-k8s.io/v1beta2", }, }, } - err = fakeCtrlClient.Create(ctx, cluster) + err = k8sClient.Create(ctx, cluster) Expect(err).NotTo(HaveOccurred()) - clusterKubeconfigSecret := &v1.Secret{ - ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, - Name: fmt.Sprintf("%s-kubeconfig", ClusterName), + // Create control plane with OLDER version than node pool + kubeadmControlPlane := &unstructured.Unstructured{} + kubeadmControlPlane.Object = map[string]interface{}{ + "metadata": map[string]interface{}{ + "name": ClusterName, + "namespace": namespace, + }, + "spec": map[string]interface{}{ + "kubeadmConfigSpec": map[string]interface{}{}, + "machineTemplate": map[string]interface{}{ + "infrastructureRef": map[string]interface{}{}, + }, + "version": "v1.21.2", // Irrelevant for this test }, } - err = fakeCtrlClient.Create(ctx, clusterKubeconfigSecret) + kubeadmControlPlane.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "controlplane.cluster.x-k8s.io", + Kind: "KubeadmControlPlane", + Version: "v1beta1", + }) + err = k8sClient.Create(ctx, kubeadmControlPlane) + Expect(err).NotTo(HaveOccurred()) + + // Set control plane status with OLDER version than node pool + err = unstructured.SetNestedField(kubeadmControlPlane.Object, map[string]interface{}{"version": controlPlaneVersion}, "status") + Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Status().Update(ctx, kubeadmControlPlane) Expect(err).NotTo(HaveOccurred()) awsCluster := &capa.AWSCluster{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: ClusterName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, }, Spec: capa.AWSClusterSpec{ + Region: AWSRegion, + S3Bucket: &capa.S3Bucket{ + Name: AWSClusterBucketName, + }, IdentityRef: &capa.AWSIdentityReference{ - Name: "default", - Kind: capa.ClusterRoleIdentityKind, + Kind: "AWSClusterRoleIdentity", + Name: "aws-cluster-role-identity", + }, + AdditionalTags: map[string]string{ + "additional-tag-for-all-resources": "custom-tag", }, - S3Bucket: &capa.S3Bucket{Name: AWSClusterBucketName}, }, } - err = fakeCtrlClient.Create(ctx, awsCluster) + err = k8sClient.Create(ctx, awsCluster) Expect(err).NotTo(HaveOccurred()) awsClusterRoleIdentity := &capa.AWSClusterRoleIdentity{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", + ObjectMeta: ctrl.ObjectMeta{ + Name: "aws-cluster-role-identity", + }, + Spec: capa.AWSClusterRoleIdentitySpec{ + AWSRoleSpec: capa.AWSRoleSpec{ + RoleArn: "arn:aws:iam::123456789012:role/test-role", + }, }, - Spec: capa.AWSClusterRoleIdentitySpec{}, } - err = fakeCtrlClient.Create(ctx, awsClusterRoleIdentity) - Expect(err).NotTo(HaveOccurred()) + err = k8sClient.Create(ctx, awsClusterRoleIdentity) + Expect(err).To(SatisfyAny( + BeNil(), + MatchError(ContainSubstring("already exists")), + )) + // Create bootstrap secret bootstrapSecret := &v1.Secret{ ObjectMeta: ctrl.ObjectMeta{ - Namespace: KarpenterMachinePoolNamespace, + Namespace: namespace, Name: DataSecretName, }, Data: map[string][]byte{"value": capiBootstrapSecretContent}, } - err = fakeCtrlClient.Create(ctx, bootstrapSecret) + err = k8sClient.Create(ctx, bootstrapSecret) + Expect(err).NotTo(HaveOccurred()) + + karpenterMachinePool := &karpenterinfra.KarpenterMachinePool{ + ObjectMeta: ctrl.ObjectMeta{ + Namespace: namespace, + Name: KarpenterMachinePoolName, + Labels: map[string]string{ + capi.ClusterNameLabel: ClusterName, + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "cluster.x-k8s.io/v1beta1", + Kind: "MachinePool", + Name: KarpenterMachinePoolName, + UID: machinePool.GetUID(), + }, + }, + }, + Spec: karpenterinfra.KarpenterMachinePoolSpec{}, + } + err = k8sClient.Create(ctx, karpenterMachinePool) Expect(err).NotTo(HaveOccurred()) }) - It("doesn't write the user data to S3 again", func() { - Expect(reconcileErr).NotTo(HaveOccurred()) - Expect(s3Client.PutCallCount()).To(Equal(0)) + + It("returns a version skew error", func() { + Expect(reconcileResult.RequeueAfter).To(Equal(60 * time.Second)) + }) + + It("persists the version skew conditions to the Kubernetes API", func() { + // This test verifies that conditions ARE saved even when errors occur + updatedKarpenterMachinePool := &karpenterinfra.KarpenterMachinePool{} + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: namespace, Name: KarpenterMachinePoolName}, updatedKarpenterMachinePool) + Expect(err).NotTo(HaveOccurred()) + + // Verify that version skew condition was persisted with the correct state + // Expect(versionSkewCondition.Message).To(ContainSubstring("control plane version v1.29.0 is older than node pool version v1.30.0")) + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("VersionSkewPolicySatisfied", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) + + // Verify that EC2NodeClass condition was persisted with error state + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("EC2NodeClassCreated", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) + + // Verify that NodePool condition was persisted with error state + Expect(updatedKarpenterMachinePool.Status.Conditions).To(HaveCondition("NodePoolCreated", v1.ConditionFalse, "VersionSkewBlocked", "Version skew policy violation: control plane version v1.29.0 is older than node pool version v1.30.0")) }) }) }) + +// ExpectUnstructured digs into u.Object at the given path, asserts that it was found and error‐free, and returns +// a GomegaAssertion on the raw interface{} value. +func ExpectUnstructured(u unstructured.Unstructured, fields ...string) Assertion { + v, found, err := unstructured.NestedFieldNoCopy(u.Object, fields...) + Expect(found).To(BeTrue(), "expected to find field %v", fields) + Expect(err).NotTo(HaveOccurred(), "error retrieving %v: %v", fields, err) + return Expect(v) +} + +// HaveCondition checks for a Condition with the given Type, Status, and Reason. +func HaveCondition(condType capi.ConditionType, status v1.ConditionStatus, reason, message string) gomegatypes.GomegaMatcher { + return WithTransform(func(conditions capi.Conditions) *capi.Condition { + for i := range conditions { + if conditions[i].Type == condType { + return &conditions[i] + } + } + return nil + }, And( + Not(BeNil()), + WithTransform(func(c *capi.Condition) v1.ConditionStatus { + return c.Status + }, Equal(status)), + WithTransform(func(c *capi.Condition) string { + return c.Reason + }, Equal(reason)), + WithTransform(func(c *capi.Condition) string { + return c.Message + }, ContainSubstring(message)), + )) +} diff --git a/controllers/route_controller_test.go b/controllers/route_controller_test.go index 012d1c02..525f3cbd 100644 --- a/controllers/route_controller_test.go +++ b/controllers/route_controller_test.go @@ -40,7 +40,7 @@ var _ = Describe("RouteReconciler", func() { transitGatewayID = "tgw-019120b363d1e81e4" prefixListARN = fmt.Sprintf("arn:aws:ec2:eu-north-1:123456789012:prefix-list/%s", prefixlistID) transitGatewayARN = fmt.Sprintf("arn:aws:ec2:eu-north-1:123456789012:transit-gateway/%s", transitGatewayID) - //subnets []string + // subnets []string ) getActualCluster := func() *capa.AWSCluster { @@ -75,7 +75,7 @@ var _ = Describe("RouteReconciler", func() { prefixListARN, ) requestResourceName = awsCluster.Name - //subnets = []string{awsCluster.Spec.NetworkSpec.Subnets[0].ID} + // subnets = []string{awsCluster.Spec.NetworkSpec.Subnets[0].ID} }) JustBeforeEach(func() { @@ -136,7 +136,7 @@ var _ = Describe("RouteReconciler", func() { }) }) - //There is no difference between GiantswarmManaged and UserManaged mode + // There is no difference between GiantswarmManaged and UserManaged mode Describe("GiantswarmManaged Mode", func() { When("There is an error while adding routes", func() { BeforeEach(func() { diff --git a/go.mod b/go.mod index e7a6adb4..d320b57f 100644 --- a/go.mod +++ b/go.mod @@ -1,9 +1,12 @@ module github.com/aws-resolver-rules-operator -go 1.23.0 +go 1.24.0 + +toolchain go1.24.3 require ( github.com/aws/aws-sdk-go v1.55.7 + github.com/blang/semver/v4 v4.0.0 github.com/giantswarm/k8smetadata v0.25.0 github.com/go-logr/logr v1.4.3 github.com/google/go-cmp v0.7.0 @@ -14,6 +17,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.22.0 + github.com/samber/lo v1.51.0 go.uber.org/zap v1.27.0 golang.org/x/tools v0.35.0 k8s.io/api v0.31.2 @@ -26,12 +30,9 @@ require ( sigs.k8s.io/yaml v1.6.0 ) -replace google.golang.org/protobuf v1.31.0 => google.golang.org/protobuf v1.33.0 - require ( github.com/apparentlymart/go-cidr v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect @@ -77,7 +78,6 @@ require ( golang.org/x/time v0.5.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/protobuf v1.36.6 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -90,3 +90,5 @@ require ( sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) + +replace google.golang.org/protobuf v1.31.0 => google.golang.org/protobuf v1.33.0 diff --git a/go.sum b/go.sum index 5cbb326c..a12fe4d1 100644 --- a/go.sum +++ b/go.sum @@ -154,6 +154,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/samber/lo v1.51.0 h1:kysRYLbHy/MB7kQZf5DSN50JHmMsNEdeY24VzJFu7wI= +github.com/samber/lo v1.51.0/go.mod h1:4+MXEGsJzbKGaUEQFKBq2xtfuznW9oz/WrgyzMzRoM0= github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= diff --git a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml index 69c3e422..eeee82cf 100644 --- a/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml +++ b/helm/aws-resolver-rules-operator/templates/infrastructure.cluster.x-k8s.io_karpentermachinepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.5 + controller-gen.kubebuilder.io/version: v0.18.0 helm.sh/resource-policy: keep labels: cluster.x-k8s.io/v1beta1: v1alpha1 @@ -50,12 +50,915 @@ spec: spec: description: KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool. properties: - iamInstanceProfile: - description: |- - The name or the Amazon Resource Name (ARN) of the instance profile associated - with the IAM role for the instance. The instance profile contains the IAM - role. - type: string + ec2NodeClass: + description: EC2NodeClass specifies the configuration for the Karpenter + EC2NodeClass + properties: + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. + The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match + the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the + following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', + ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] + ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM + parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', + 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || + has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) + || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with + a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned + nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically + set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the + EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, + or alias ARN) of the customer managed KMS key to use + for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate + is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) + && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 + capacityReservationSelectorTerms: + description: |- + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along + with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring + is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store + disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement + for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + description: EvictionHard is the map of signal names to quantities + that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + description: EvictionSoft is the map of signal names to quantities + that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal + names to quantities that define grace periods for each eviction + signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + description: KubeReserved contains resources reserved for + Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: kubeReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + description: SystemReserved contains resources reserved for + OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' + || x=='pid') + - message: systemReserved value cannot be a negative resource + quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in + self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching + evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, + (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security + group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with + a combination of other fields in a security group selector + term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector + terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a + combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances + and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + nodePool: + description: NodePool specifies the configuration for the Karpenter + NodePool + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to + Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons + for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + requirements: + description: Requirements are layered with GetLabels and + applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies + to. + type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a + value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() + != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have + a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator + == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) + >= 0) : true)' + - message: requirements with 'minValues' must have at + least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) + ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's + node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied + to a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the + taint key. + type: string + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object providerIDList: description: |- ProviderIDList are the identification IDs of machine instances provided by the provider. @@ -68,8 +971,54 @@ spec: description: KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool. properties: + conditions: + description: Conditions defines current service state of the KarpenterMachinePool. + items: + description: Condition defines an observation of a Cluster API resource + operational state. + properties: + lastTransitionTime: + description: |- + Last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when + the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + A human readable message indicating details about the transition. + This field may be empty. + type: string + reason: + description: |- + The reason for the condition's last transition in CamelCase. + The specific API may choose whether or not this field is considered a guaranteed API. + This field may not be empty. + type: string + severity: + description: |- + Severity provides an explicit classification of Reason code, so the users or machines can immediately + understand the current situation and act accordingly. + The Severity field MUST be set only when Status=False. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: |- + Type of condition in CamelCase or in foo.example.com/CamelCase. + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions + can be useful (see .node.status.conditions), the ability to deconflict is important. + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array ready: - description: Ready is true when the provider resource is ready. + description: Ready denotes that the KarpenterMachinePool is ready + and fulfilling the infrastructure contract. type: boolean replicas: description: Replicas is the most recently observed number of replicas diff --git a/helm/aws-resolver-rules-operator/templates/rbac.yaml b/helm/aws-resolver-rules-operator/templates/rbac.yaml index d3a5d75b..5abd5efd 100644 --- a/helm/aws-resolver-rules-operator/templates/rbac.yaml +++ b/helm/aws-resolver-rules-operator/templates/rbac.yaml @@ -20,10 +20,12 @@ rules: - list - patch - watch + - update - apiGroups: - controlplane.cluster.x-k8s.io resources: - awsmanagedcontrolplanes + - kubeadmcontrolplanes verbs: - get - list diff --git a/pkg/conditions/conditions.go b/pkg/conditions/conditions.go index 7773a3da..dbea1ad6 100644 --- a/pkg/conditions/conditions.go +++ b/pkg/conditions/conditions.go @@ -13,6 +13,48 @@ const ( TransitGatewayCreated capi.ConditionType = "TransitGatewayCreated" TransitGatewayAttached capi.ConditionType = "TransitGatewayAttached" PrefixListEntriesReady capi.ConditionType = "PrefixListEntriesReady" + + // NodePoolCreatedCondition indicates whether the NodePool resource has been successfully + // created or updated in the workload cluster. This doesn't mean the NodePool is ready + // to provision nodes, just that the resource exists. + NodePoolCreatedCondition capi.ConditionType = "NodePoolCreated" + + // EC2NodeClassCreatedCondition indicates whether the EC2NodeClass resource has been + // successfully created or updated in the workload cluster. This doesn't mean the + // EC2NodeClass is ready for use, just that the resource exists. + EC2NodeClassCreatedCondition capi.ConditionType = "EC2NodeClassCreated" + + // BootstrapDataReadyCondition indicates whether the bootstrap user data has been + // successfully uploaded to S3 and is ready for use by Karpenter nodes. + BootstrapDataReadyCondition capi.ConditionType = "BootstrapDataReady" + + // VersionSkewPolicySatisfiedCondition indicates whether the Kubernetes version skew policy + // is satisfied (worker nodes don't use newer versions than control plane). + VersionSkewPolicySatisfiedCondition capi.ConditionType = "VersionSkewPolicySatisfied" + + // ReadyCondition indicates the overall readiness of the KarpenterMachinePool. + // This is True when all necessary Karpenter resources are created and configured. + ReadyCondition capi.ConditionType = "Ready" +) + +// Condition reasons used by various controllers +const ( + // Generic reasons used across controllers + ReadyReason = "Ready" + NotReadyReason = "NotReady" + + // KarpenterMachinePool controller reasons + NodePoolCreationFailedReason = "NodePoolCreationFailed" + NodePoolCreationSucceededReason = "NodePoolCreated" + EC2NodeClassCreationFailedReason = "EC2NodeClassCreationFailed" + EC2NodeClassCreationSucceededReason = "EC2NodeClassCreated" + BootstrapDataUploadFailedReason = "BootstrapDataUploadFailed" + BootstrapDataSecretNotFoundReason = "BootstrapDataSecretNotFound" + BootstrapDataSecretInvalidReason = "BootstrapDataSecretInvalid" + BootstrapDataSecretMissingReferenceReason = "BootstrapDataSecretMissingReference" + BootstrapDataUploadSucceededReason = "BootstrapDataUploaded" + VersionSkewBlockedReason = "VersionSkewBlocked" + VersionSkewValidReason = "VersionSkewValid" ) func MarkReady(setter capiconditions.Setter, condition capi.ConditionType) { @@ -42,3 +84,63 @@ func MarkIDNotProvided(cluster *capi.Cluster, id string) { "The %s ID is missing from the annotations", id, ) } + +func MarkNodePoolCreated(setter capiconditions.Setter) { + capiconditions.Set(setter, &capi.Condition{ + Type: NodePoolCreatedCondition, + Status: "True", + Reason: NodePoolCreationSucceededReason, + }) +} + +func MarkNodePoolNotCreated(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, NodePoolCreatedCondition, reason, capi.ConditionSeverityError, "%s", message) +} + +func MarkEC2NodeClassCreated(setter capiconditions.Setter) { + capiconditions.Set(setter, &capi.Condition{ + Type: EC2NodeClassCreatedCondition, + Status: "True", + Reason: EC2NodeClassCreationSucceededReason, + }) +} + +func MarkEC2NodeClassNotCreated(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, EC2NodeClassCreatedCondition, reason, capi.ConditionSeverityError, "%s", message) +} + +func MarkBootstrapDataReady(setter capiconditions.Setter) { + capiconditions.Set(setter, &capi.Condition{ + Type: BootstrapDataReadyCondition, + Status: "True", + Reason: ReadyReason, + }) +} + +func MarkBootstrapDataNotReady(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, BootstrapDataReadyCondition, reason, capi.ConditionSeverityError, "%s", message) +} + +func MarkVersionSkewPolicySatisfied(setter capiconditions.Setter) { + capiconditions.Set(setter, &capi.Condition{ + Type: VersionSkewPolicySatisfiedCondition, + Status: "True", + Reason: VersionSkewValidReason, + }) +} + +func MarkVersionSkewInvalid(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, VersionSkewPolicySatisfiedCondition, reason, capi.ConditionSeverityError, "%s", message) +} + +func MarkKarpenterMachinePoolReady(setter capiconditions.Setter) { + capiconditions.Set(setter, &capi.Condition{ + Type: ReadyCondition, + Status: "True", + Reason: ReadyReason, + }) +} + +func MarkKarpenterMachinePoolNotReady(setter capiconditions.Setter, reason, message string) { + capiconditions.MarkFalse(setter, ReadyCondition, reason, capi.ConditionSeverityError, "%s", message) +} diff --git a/pkg/resolver/resolverfakes/fake_ec2client.go b/pkg/resolver/resolverfakes/fake_ec2client.go index 290f44af..50fba80a 100644 --- a/pkg/resolver/resolverfakes/fake_ec2client.go +++ b/pkg/resolver/resolverfakes/fake_ec2client.go @@ -261,12 +261,6 @@ func (fake *FakeEC2Client) TerminateInstancesByTagReturnsOnCall(i int, result1 [ func (fake *FakeEC2Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.createSecurityGroupForResolverEndpointsMutex.RLock() - defer fake.createSecurityGroupForResolverEndpointsMutex.RUnlock() - fake.deleteSecurityGroupForResolverEndpointsMutex.RLock() - defer fake.deleteSecurityGroupForResolverEndpointsMutex.RUnlock() - fake.terminateInstancesByTagMutex.RLock() - defer fake.terminateInstancesByTagMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_prefix_list_client.go b/pkg/resolver/resolverfakes/fake_prefix_list_client.go index f85f8adc..11b350f4 100644 --- a/pkg/resolver/resolverfakes/fake_prefix_list_client.go +++ b/pkg/resolver/resolverfakes/fake_prefix_list_client.go @@ -319,14 +319,6 @@ func (fake *FakePrefixListClient) DeleteEntryReturnsOnCall(i int, result1 error) func (fake *FakePrefixListClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.applyMutex.RLock() - defer fake.applyMutex.RUnlock() - fake.applyEntryMutex.RLock() - defer fake.applyEntryMutex.RUnlock() - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - fake.deleteEntryMutex.RLock() - defer fake.deleteEntryMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_ramclient.go b/pkg/resolver/resolverfakes/fake_ramclient.go index 3f30b598..a1a2cc86 100644 --- a/pkg/resolver/resolverfakes/fake_ramclient.go +++ b/pkg/resolver/resolverfakes/fake_ramclient.go @@ -164,10 +164,6 @@ func (fake *FakeRAMClient) DeleteResourceShareReturnsOnCall(i int, result1 error func (fake *FakeRAMClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.applyResourceShareMutex.RLock() - defer fake.applyResourceShareMutex.RUnlock() - fake.deleteResourceShareMutex.RLock() - defer fake.deleteResourceShareMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_resolver_client.go b/pkg/resolver/resolverfakes/fake_resolver_client.go index 5fcff0cd..2a5c6938 100644 --- a/pkg/resolver/resolverfakes/fake_resolver_client.go +++ b/pkg/resolver/resolverfakes/fake_resolver_client.go @@ -583,20 +583,6 @@ func (fake *FakeResolverClient) GetResolverRuleByNameReturnsOnCall(i int, result func (fake *FakeResolverClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.associateResolverRuleWithContextMutex.RLock() - defer fake.associateResolverRuleWithContextMutex.RUnlock() - fake.createResolverRuleMutex.RLock() - defer fake.createResolverRuleMutex.RUnlock() - fake.deleteResolverRuleMutex.RLock() - defer fake.deleteResolverRuleMutex.RUnlock() - fake.disassociateResolverRuleWithContextMutex.RLock() - defer fake.disassociateResolverRuleWithContextMutex.RUnlock() - fake.findResolverRuleIdsAssociatedWithVPCIdMutex.RLock() - defer fake.findResolverRuleIdsAssociatedWithVPCIdMutex.RUnlock() - fake.findResolverRulesByAWSAccountIdMutex.RLock() - defer fake.findResolverRulesByAWSAccountIdMutex.RUnlock() - fake.getResolverRuleByNameMutex.RLock() - defer fake.getResolverRuleByNameMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_route53client.go b/pkg/resolver/resolverfakes/fake_route53client.go index 8f2dbd3a..8cc7ab47 100644 --- a/pkg/resolver/resolverfakes/fake_route53client.go +++ b/pkg/resolver/resolverfakes/fake_route53client.go @@ -653,22 +653,6 @@ func (fake *FakeRoute53Client) GetHostedZoneNSRecordReturnsOnCall(i int, result1 func (fake *FakeRoute53Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addDelegationToParentZoneMutex.RLock() - defer fake.addDelegationToParentZoneMutex.RUnlock() - fake.addDnsRecordsToHostedZoneMutex.RLock() - defer fake.addDnsRecordsToHostedZoneMutex.RUnlock() - fake.createHostedZoneMutex.RLock() - defer fake.createHostedZoneMutex.RUnlock() - fake.deleteDelegationFromParentZoneMutex.RLock() - defer fake.deleteDelegationFromParentZoneMutex.RUnlock() - fake.deleteDnsRecordsFromHostedZoneMutex.RLock() - defer fake.deleteDnsRecordsFromHostedZoneMutex.RUnlock() - fake.deleteHostedZoneMutex.RLock() - defer fake.deleteHostedZoneMutex.RUnlock() - fake.getHostedZoneIdByNameMutex.RLock() - defer fake.getHostedZoneIdByNameMutex.RUnlock() - fake.getHostedZoneNSRecordMutex.RLock() - defer fake.getHostedZoneNSRecordMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_route_table_client.go b/pkg/resolver/resolverfakes/fake_route_table_client.go index 84cef529..c63e7ef3 100644 --- a/pkg/resolver/resolverfakes/fake_route_table_client.go +++ b/pkg/resolver/resolverfakes/fake_route_table_client.go @@ -168,10 +168,6 @@ func (fake *FakeRouteTableClient) RemoveRoutesReturnsOnCall(i int, result1 error func (fake *FakeRouteTableClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.addRoutesMutex.RLock() - defer fake.addRoutesMutex.RUnlock() - fake.removeRoutesMutex.RLock() - defer fake.removeRoutesMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_s3client.go b/pkg/resolver/resolverfakes/fake_s3client.go index 2bccf200..dbbe3ae8 100644 --- a/pkg/resolver/resolverfakes/fake_s3client.go +++ b/pkg/resolver/resolverfakes/fake_s3client.go @@ -99,8 +99,6 @@ func (fake *FakeS3Client) PutReturnsOnCall(i int, result1 error) { func (fake *FakeS3Client) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.putMutex.RLock() - defer fake.putMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/resolver/resolverfakes/fake_transit_gateway_client.go b/pkg/resolver/resolverfakes/fake_transit_gateway_client.go index ac9e828a..696dbaa7 100644 --- a/pkg/resolver/resolverfakes/fake_transit_gateway_client.go +++ b/pkg/resolver/resolverfakes/fake_transit_gateway_client.go @@ -319,14 +319,6 @@ func (fake *FakeTransitGatewayClient) DetachReturnsOnCall(i int, result1 error) func (fake *FakeTransitGatewayClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.applyMutex.RLock() - defer fake.applyMutex.RUnlock() - fake.applyAttachmentMutex.RLock() - defer fake.applyAttachmentMutex.RUnlock() - fake.deleteMutex.RLock() - defer fake.deleteMutex.RUnlock() - fake.detachMutex.RLock() - defer fake.detachMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/pkg/versionskew/skew.go b/pkg/versionskew/skew.go new file mode 100644 index 00000000..abb83c33 --- /dev/null +++ b/pkg/versionskew/skew.go @@ -0,0 +1,29 @@ +package versionskew + +import ( + "fmt" + + "github.com/blang/semver/v4" +) + +// IsSkewAllowed checks if the worker version can be updated based on the control plane version. +// The workers can't use a newer k8s version than the one used by the control plane. +// +// This implements Kubernetes version skew policy https://kubernetes.io/releases/version-skew-policy/ +// +// Returns: (allowed bool, error) +func IsSkewAllowed(controlPlaneVersion, workerVersion string) (bool, error) { + // Parse versions using semantic versioning for proper comparison + controlPlaneCurrentK8sVersion, err := semver.ParseTolerant(controlPlaneVersion) + if err != nil { + return false, fmt.Errorf("failed to parse control plane k8s version %q: %w", controlPlaneVersion, err) + } + + workerDesiredK8sVersion, err := semver.ParseTolerant(workerVersion) + if err != nil { + return false, fmt.Errorf("failed to parse worker desired k8s version %q: %w", workerVersion, err) + } + + // Allow if control plane version >= desired worker version + return controlPlaneCurrentK8sVersion.GE(workerDesiredK8sVersion), nil +} diff --git a/pkg/versionskew/skew_test.go b/pkg/versionskew/skew_test.go new file mode 100644 index 00000000..65d2c21e --- /dev/null +++ b/pkg/versionskew/skew_test.go @@ -0,0 +1,99 @@ +package versionskew + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestVersionSkew(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "VersionSkew Suite") +} + +var _ = Describe("IsSkewAllowed", func() { + Context("when control plane version is higher than worker version", func() { + It("should allow the skew", func() { + allowed, err := IsSkewAllowed("1.28.0", "1.27.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + + It("should allow the skew with patch versions", func() { + allowed, err := IsSkewAllowed("1.28.5", "1.28.3") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + }) + + Context("when control plane version equals worker version", func() { + It("should allow the skew", func() { + allowed, err := IsSkewAllowed("1.28.0", "1.28.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + + It("should allow the skew with patch versions", func() { + allowed, err := IsSkewAllowed("1.28.5", "1.28.5") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + }) + + Context("when control plane version is lower than worker version", func() { + It("should not allow the skew", func() { + allowed, err := IsSkewAllowed("1.27.0", "1.28.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeFalse()) + }) + + It("should not allow the skew with patch versions", func() { + allowed, err := IsSkewAllowed("1.28.3", "1.28.5") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeFalse()) + }) + + It("should not allow the skew with minor version difference", func() { + allowed, err := IsSkewAllowed("1.27.10", "1.28.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeFalse()) + }) + }) + + Context("when parsing version strings with prefixes", func() { + It("should handle v prefixes correctly", func() { + allowed, err := IsSkewAllowed("v1.28.0", "v1.27.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + + It("should handle mixed prefixes correctly", func() { + allowed, err := IsSkewAllowed("v1.28.0", "1.27.0") + Expect(err).ToNot(HaveOccurred()) + Expect(allowed).To(BeTrue()) + }) + }) + + Context("when version strings are invalid", func() { + It("should return error for invalid control plane version", func() { + _, err := IsSkewAllowed("invalid-version", "1.28.0") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to parse control plane k8s version")) + }) + + It("should return error for invalid worker version", func() { + _, err := IsSkewAllowed("1.28.0", "invalid-version") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to parse worker desired k8s version")) + }) + + It("should return error for empty versions", func() { + _, err := IsSkewAllowed("", "1.28.0") + Expect(err).To(HaveOccurred()) + + _, err = IsSkewAllowed("1.28.0", "") + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml new file mode 100644 index 00000000..4504d1ca --- /dev/null +++ b/tests/testdata/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -0,0 +1,847 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: ec2nodeclasses.karpenter.k8s.aws +spec: + group: karpenter.k8s.aws + names: + categories: + - karpenter + kind: EC2NodeClass + listKind: EC2NodeClassList + plural: ec2nodeclasses + shortNames: + - ec2nc + - ec2ncs + singular: ec2nodeclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.role + name: Role + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: EC2NodeClass is the Schema for the EC2NodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. + properties: + amiFamily: + description: |- + AMIFamily dictates the UserData format and default BlockDeviceMappings used when generating launch templates. + This field is optional when using an alias amiSelectorTerm, and the value will be inferred from the alias' + family. When an alias is specified, this field may only be set to its corresponding family or 'Custom'. If no + alias is specified, this field is required. + NOTE: We ignore the AMIFamily for hashing here because we hash the AMIFamily dynamically by using the alias using + the AMIFamily() helper function + enum: + - AL2 + - AL2023 + - Bottlerocket + - Custom + - Windows2019 + - Windows2022 + type: string + amiSelectorTerms: + description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + items: + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + alias: + description: |- + Alias specifies which EKS optimized AMI to select. + Each alias consists of a family and an AMI version, specified as "family@version". + Valid families include: al2, al2023, bottlerocket, windows2019, and windows2022. + The version can either be pinned to a specific AMI release, with that AMIs version format (ex: "al2023@v20240625" or "bottlerocket@v1.10.0"). + The version can also be set to "latest" for any family. Setting the version to latest will result in drift when a new AMI is released. This is **not** recommended for production environments. + Note: The Windows families do **not** support version pinning, and only latest may be used. + maxLength: 30 + type: string + x-kubernetes-validations: + - message: '''alias'' is improperly formatted, must match the format ''family@version''' + rule: self.matches('^[a-zA-Z0-9]+@.+$') + - message: 'family is not supported, must be one of the following: ''al2'', ''al2023'', ''bottlerocket'', ''windows2019'', ''windows2022''' + rule: self.split('@')[0] in ['al2','al2023','bottlerocket','windows2019','windows2022'] + - message: windows families may only specify version 'latest' + rule: 'self.split(''@'')[0] in [''windows2019'',''windows2022''] ? self.split(''@'')[1] == ''latest'' : true' + id: + description: ID is the ami id in EC2 + pattern: ami-[0-9a-z]+ + type: string + name: + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. + type: string + owner: + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + type: string + ssmParameter: + description: SSMParameter is the name (or ARN) of the SSM parameter containing the Image ID. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select amis. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + minItems: 1 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name', 'alias', 'ssmParameter'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name) || has(x.alias) || has(x.ssmParameter)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.id) && (has(x.alias) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms' + rule: '!self.exists(x, has(x.alias) && (has(x.id) || has(x.tags) || has(x.name) || has(x.owner)))' + - message: '''alias'' is mutually exclusive, cannot be set with a combination of other amiSelectorTerms' + rule: '!(self.exists(x, has(x.alias)) && self.size() != 1)' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + type: boolean + blockDeviceMappings: + description: BlockDeviceMappings to be applied to provisioned nodes. + items: + properties: + deviceName: + description: The device name (for example, /dev/sdh or xvdh). + type: string + ebs: + description: EBS contains parameters used to automatically set up EBS volumes when an instance is launched. + properties: + deleteOnTermination: + description: DeleteOnTermination indicates whether the EBS volume is deleted on instance termination. + type: boolean + encrypted: + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. + type: boolean + iops: + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + The following are the supported values for each volume type: + + * gp3: 3,000-16,000 IOPS + + * io1: 100-64,000 IOPS + + * io2: 100-64,000 IOPS + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. + format: int64 + type: integer + kmsKeyID: + description: Identifier (key ID, key alias, key ARN, or alias ARN) of the customer managed KMS key to use for EBS encryption. + type: string + snapshotID: + description: SnapshotID is the ID of an EBS snapshot + type: string + throughput: + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. + format: int64 + type: integer + volumeInitializationRate: + description: |- + VolumeInitializationRate specifies the Amazon EBS Provisioned Rate for Volume Initialization, + in MiB/s, at which to download the snapshot blocks from Amazon S3 to the volume. This is also known as volume + initialization. Specifying a volume initialization rate ensures that the volume is initialized at a + predictable and consistent rate after creation. Only allowed if SnapshotID is set. + Valid Range: Minimum value of 100. Maximum value of 300. + format: int32 + maximum: 300 + minimum: 100 + type: integer + volumeSize: + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + * gp2 and gp3: 1-16,384 + + * io1 and io2: 4-16,384 + + * st1 and sc1: 125-16,384 + + * standard: 1-1,024 + pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ + type: string + volumeType: + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 + type: string + type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) + - message: snapshotID must be set when volumeInitializationRate is set + rule: '!has(self.volumeInitializationRate) || (has(self.snapshotID) && self.snapshotID != '''')' + rootVolume: + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. + type: boolean + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 + capacityReservationSelectorTerms: + description: |- + CapacityReservationSelectorTerms is a list of capacity reservation selector terms. Each term is ORed together to + determine the set of eligible capacity reservations. + items: + properties: + id: + description: ID is the capacity reservation id in EC2 + pattern: ^cr-[0-9a-z]+$ + type: string + ownerID: + description: Owner is the owner id for the ami. + pattern: ^[0-9]{12}$ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select capacity reservations. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set along with tags in a capacity reservation selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.ownerID)))' + context: + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + type: string + detailedMonitoring: + description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched + type: boolean + instanceProfile: + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. + type: string + x-kubernetes-validations: + - message: instanceProfile cannot be empty + rule: self != '' + instanceStorePolicy: + description: InstanceStorePolicy specifies how to handle instance-store disks. + enum: + - RAID0 + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + metadataOptions: + default: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 1 + httpTokens: required + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + for limiting exposure of Instance Metadata and User Data to pods. + If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 + disabled, with httpPutResponseLimit of 1, and with httpTokens + required. + properties: + httpEndpoint: + default: enabled + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. + enum: + - enabled + - disabled + type: string + httpProtocolIPv6: + default: disabled + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". + enum: + - enabled + - disabled + type: string + httpPutResponseHopLimit: + default: 1 + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 1. + format: int64 + maximum: 64 + minimum: 1 + type: integer + httpTokens: + default: required + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. + enum: + - required + - optional + type: string + type: object + role: + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. + type: string + x-kubernetes-validations: + - message: role cannot be empty + rule: self != '' + - message: immutable field changed + rule: self == oldSelf + securityGroupSelectorTerms: + description: SecurityGroupSelectorTerms is a list of security group selector terms. The terms are ORed. + items: + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the security group id in EC2 + pattern: sg-[0-9a-z]+ + type: string + name: + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select security groups. + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination of other fields in a security group selector term' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' + subnetSelectorTerms: + description: SubnetSelectorTerms is a list of subnet selector terms. The terms are ORed. + items: + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. + properties: + id: + description: ID is the subnet id in EC2 + pattern: subnet-[0-9a-z]+ + type: string + tags: + additionalProperties: + type: string + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 + type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') + type: object + maxItems: 30 + type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination of other fields in a subnet selector term' + rule: '!self.all(x, has(x.id) && has(x.tags))' + tags: + additionalProperties: + type: string + description: Tags to be applied on ec2 resources like instances and launch templates. + type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching eks:eks-cluster-name + rule: self.all(k, k !='eks:eks-cluster-name') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') + userData: + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. + type: string + required: + - amiSelectorTerms + - securityGroupSelectorTerms + - subnetSelectorTerms + type: object + x-kubernetes-validations: + - message: must specify exactly one of ['role', 'instanceProfile'] + rule: (has(self.role) && !has(self.instanceProfile)) || (!has(self.role) && has(self.instanceProfile)) + - message: changing from 'instanceProfile' to 'role' is not supported. You must delete and recreate this node class if you want to change this. + rule: (has(oldSelf.role) && has(self.role)) || (has(oldSelf.instanceProfile) && has(self.instanceProfile)) + - message: if set, amiFamily must be 'AL2' or 'Custom' when using an AL2 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2'') : true)' + - message: if set, amiFamily must be 'AL2023' or 'Custom' when using an AL2023 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''al2023'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''AL2023'') : true)' + - message: if set, amiFamily must be 'Bottlerocket' or 'Custom' when using a Bottlerocket alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''bottlerocket'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Bottlerocket'') : true)' + - message: if set, amiFamily must be 'Windows2019' or 'Custom' when using a Windows2019 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2019'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2019'') : true)' + - message: if set, amiFamily must be 'Windows2022' or 'Custom' when using a Windows2022 alias + rule: '!has(self.amiFamily) || (self.amiSelectorTerms.exists(x, has(x.alias) && x.alias.find(''^[^@]+'') == ''windows2022'') ? (self.amiFamily == ''Custom'' || self.amiFamily == ''Windows2022'') : true)' + - message: must specify amiFamily if amiSelectorTerms does not contain an alias + rule: 'self.amiSelectorTerms.exists(x, has(x.alias)) ? true : has(self.amiFamily)' + status: + description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass + properties: + amis: + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. + items: + description: AMI contains resolved AMI selector values utilized for node launch + properties: + deprecated: + description: Deprecation status of the AMI + type: boolean + id: + description: ID of the AMI + type: string + name: + description: Name of the AMI + type: string + requirements: + description: Requirements of the AMI to be utilized on an instance type + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + required: + - id + - requirements + type: object + type: array + capacityReservations: + description: |- + CapacityReservations contains the current capacity reservation values that are available to this NodeClass under the + CapacityReservation selectors. + items: + properties: + availabilityZone: + description: The availability zone the capacity reservation is available in. + type: string + endTime: + description: |- + The time at which the capacity reservation expires. Once expired, the reserved capacity is released and Karpenter + will no longer be able to launch instances into that reservation. + format: date-time + type: string + id: + description: The id for the capacity reservation. + pattern: ^cr-[0-9a-z]+$ + type: string + instanceMatchCriteria: + description: Indicates the type of instance launches the capacity reservation accepts. + enum: + - open + - targeted + type: string + instanceType: + description: The instance type for the capacity reservation. + type: string + ownerID: + description: The ID of the AWS account that owns the capacity reservation. + pattern: ^[0-9]{12}$ + type: string + reservationType: + default: default + description: The type of capacity reservation. + enum: + - default + - capacity-block + type: string + state: + default: active + description: |- + The state of the capacity reservation. A capacity reservation is considered to be expiring if it is within the EC2 + reclaimation window. Only capacity-block reservations may be in this state. + enum: + - active + - expiring + type: string + required: + - availabilityZone + - id + - instanceMatchCriteria + - instanceType + - ownerID + type: object + type: array + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + instanceProfile: + description: InstanceProfile contains the resolved instance profile for the role + type: string + securityGroups: + description: |- + SecurityGroups contains the current security group values that are available to the + cluster under the SecurityGroups selectors. + items: + description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch + properties: + id: + description: ID of the security group + type: string + name: + description: Name of the security group + type: string + required: + - id + type: object + type: array + subnets: + description: |- + Subnets contains the current subnet values that are available to the + cluster under the subnet selectors. + items: + description: Subnet contains resolved Subnet selector values utilized for node launch + properties: + id: + description: ID of the subnet + type: string + zone: + description: The associated availability zone + type: string + zoneID: + description: The associated availability zone ID + type: string + required: + - id + - zone + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/tests/testdata/crds/karpenter.sh_nodeclaims.yaml b/tests/testdata/crds/karpenter.sh_nodeclaims.yaml new file mode 100644 index 00000000..c989950e --- /dev/null +++ b/tests/testdata/crds/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,399 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.imageID + name: ImageID + priority: 1 + type: string + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + - jsonPath: .status.conditions[?(@.type=="Drifted")].status + name: Drifted + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-type", "karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} diff --git a/tests/testdata/crds/karpenter.sh_nodepools.yaml b/tests/testdata/crds/karpenter.sh_nodepools.yaml new file mode 100644 index 00000000..1f87a6fe --- /dev/null +++ b/tests/testdata/crds/karpenter.sh_nodepools.yaml @@ -0,0 +1,525 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.18.0 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.k8s.aws" is restricted + rule: self.all(x, x in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+|Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + x-kubernetes-validations: + - message: group may not be empty + rule: self != '' + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + x-kubernetes-validations: + - message: kind may not be empty + rule: self != '' + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + x-kubernetes-validations: + - message: name may not be empty + rule: self != '' + required: + - group + - kind + - name + type: object + x-kubernetes-validations: + - message: nodeClassRef.group is immutable + rule: self.group == oldSelf.group + - message: nodeClassRef.kind is immutable + rule: self.kind == oldSelf.kind + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.k8s.aws" is restricted + rule: self in ["karpenter.k8s.aws/capacity-reservation-id", "karpenter.k8s.aws/ec2nodeclass", "karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu", "karpenter.k8s.aws/instance-cpu-manufacturer", "karpenter.k8s.aws/instance-cpu-sustained-clock-speed-mhz", "karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-ebs-bandwidth", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + nodeClassObservedGeneration: + description: |- + NodeClassObservedGeneration represents the observed nodeClass generation for referenced nodeClass. If this does not match + the actual NodeClass Generation, NodeRegistrationHealthy status condition on the NodePool will be reset + format: int64 + type: integer + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {}