Create karpenter custom resources in workload clusters (#268)

* wip

* wip2

* wip3

* Use envtest k8s client

* Only use security groups and subnets if defined

* Use map for subnet and security group selectors

* Take types from karpenter project

* Add managed-by label to resources created by controller

* Check if NodePool field is nil

* Avoid setting defaults

* Run goimports in api files (including generated files)

* Refactor tests

* Fix user data s3 key

* Ignore 'no matches for kind' errors when deleting karpenter CRs

* Mimic upstream ec2nodeclass

* Add CAPA additional tags to karpenter resources

* Refactor

* Use conditions properly

* Fix condition message formatting

* Go mod tidy

* Use different copies for the different patch operations

* Use update instead of patch for conditions

* Rename policy skew condition

* Fix condition message formatting again

* Update conditions immediately

* Use patchHelper

* Improve condition handling

* Reconcile AWS metadata endpoint config

* Bring back status.ready field

* Reconcile taints and startuptaints

* Extract version skew to its own package

* Requeue instead of returning error when version skew policy does not allow workers upgrade

* Add comment explaining goimports on generated files

* linting fixes

* revert makegen

* revert makegen

* file origins

* not needed

* patch KMP

* Limit rbac permissions

* remove hold from build

---------

Co-authored-by: Pau Rosello <[email protected]>

Author: fiunchinho

.circleci/config.yml

@@ -74,8 +74,6 @@ workflows:
             only: /^v.*/
 
     - architect/go-build:
-        requires:
-        - hold
         context: architect
         name: go-build
         binary: aws-resolver-rules-operator

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Create karpenter custom resources in workload clusters.
+
 ## [0.20.0] - 2025-06-23
 
 ### Changed

Makefile.custom.mk

@@ -26,14 +26,8 @@ crds: controller-gen ## Generate CustomResourceDefinition.
 generate: controller-gen crds ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	go generate ./...
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
-
-.PHONY: fmt
-fmt: ## Run go fmt against code.
-	go fmt ./...
-
-.PHONY: vet
-vet: ## Run go vet against code.
-	go vet ./...
+# We need to run goimports after controller-gen to avoid CI complains about goimports in the generated files
+	@go run golang.org/x/tools/cmd/goimports -w ./api/v1alpha1
 
 .PHONY: create-acceptance-cluster
 create-acceptance-cluster: kind
@@ -127,7 +121,7 @@ coverage-html: test-unit
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 .PHONY: controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.16.5)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.18.0)
 
 ENVTEST = $(shell pwd)/bin/setup-envtest
 .PHONY: envtest

Makefile.gen.go.mk

@@ -12,6 +12,7 @@ MODULE         := $(shell go list -m)
 OS             := $(shell go env GOOS)
 SOURCES        := $(shell find . -name '*.go')
 VERSION        := $(shell architect project version)
+
 ifeq ($(OS), linux)
 EXTLDFLAGS := -static
 endif

api/v1alpha1/duration.go

@@ -0,0 +1,80 @@
+package v1alpha1
+
+import (
+	"encoding/json"
+	"fmt"
+	"slices"
+	"time"
+
+	"github.com/samber/lo"
+)
+
+const Never = "Never"
+
+// NillableDuration is a wrapper around time.Duration which supports correct
+// marshaling to YAML and JSON. It uses the value "Never" to signify
+// that the duration is disabled and sets the inner duration as nil
+type NillableDuration struct {
+	*time.Duration
+
+	// Raw is used to ensure we remarshal the NillableDuration in the same format it was specified.
+	// This ensures tools like Flux and ArgoCD don't mistakenly detect drift due to our conversion webhooks.
+	Raw []byte `hash:"ignore"`
+}
+
+func MustParseNillableDuration(val string) NillableDuration {
+	nd := NillableDuration{}
+	// Use %q instead of %s to ensure that we unmarshal the value as a string and not an int
+	lo.Must0(json.Unmarshal([]byte(fmt.Sprintf("%q", val)), &nd))
+	return nd
+}
+
+// UnmarshalJSON implements the json.Unmarshaller interface.
+func (d *NillableDuration) UnmarshalJSON(b []byte) error {
+	var str string
+	err := json.Unmarshal(b, &str)
+	if err != nil {
+		return err
+	}
+	if str == Never {
+		return nil
+	}
+	pd, err := time.ParseDuration(str)
+	if err != nil {
+		return err
+	}
+	d.Raw = slices.Clone(b)
+	d.Duration = &pd
+	return nil
+}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (d NillableDuration) MarshalJSON() ([]byte, error) {
+	if d.Raw != nil {
+		return d.Raw, nil
+	}
+	if d.Duration != nil {
+		return json.Marshal((*d.Duration).String())
+	}
+	return json.Marshal(Never)
+}
+
+// ToUnstructured implements the value.UnstructuredConverter interface.
+func (d NillableDuration) ToUnstructured() interface{} {
+	if d.Raw != nil {
+		// Decode the JSON bytes to get the actual string value
+		var str string
+		if err := json.Unmarshal(d.Raw, &str); err == nil {
+			return str
+		}
+		// Fallback to string conversion if unmarshal fails
+		if d.Duration != nil {
+			return (*d.Duration).String()
+		}
+		return Never
+	}
+	if d.Duration != nil {
+		return (*d.Duration).String()
+	}
+	return Never
+}

api/v1alpha1/ec2nodeclass.go

@@ -18,14 +18,19 @@ package v1alpha1
 
 import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	capi "sigs.k8s.io/cluster-api/api/v1beta1"
 )
 
 // KarpenterMachinePoolSpec defines the desired state of KarpenterMachinePool.
 type KarpenterMachinePoolSpec struct {
-	// The name or the Amazon Resource Name (ARN) of the instance profile associated
-	// with the IAM role for the instance. The instance profile contains the IAM
-	// role.
-	IamInstanceProfile string `json:"iamInstanceProfile,omitempty"`
+	// NodePool specifies the configuration for the Karpenter NodePool
+	// +optional
+	NodePool *NodePoolSpec `json:"nodePool,omitempty"`
+
+	// EC2NodeClass specifies the configuration for the Karpenter EC2NodeClass
+	// +optional
+	EC2NodeClass *EC2NodeClassSpec `json:"ec2NodeClass,omitempty"`
+
 	// ProviderIDList are the identification IDs of machine instances provided by the provider.
 	// This field must match the provider IDs as seen on the node objects corresponding to a machine pool's machine instances.
 	// +optional
@@ -34,13 +39,17 @@ type KarpenterMachinePoolSpec struct {
 
 // KarpenterMachinePoolStatus defines the observed state of KarpenterMachinePool.
 type KarpenterMachinePoolStatus struct {
-	// Ready is true when the provider resource is ready.
+	// Ready denotes that the KarpenterMachinePool is ready and fulfilling the infrastructure contract.
 	// +optional
 	Ready bool `json:"ready"`
 
 	// Replicas is the most recently observed number of replicas
 	// +optional
 	Replicas int32 `json:"replicas"`
+
+	// Conditions defines current service state of the KarpenterMachinePool.
+	// +optional
+	Conditions capi.Conditions `json:"conditions,omitempty"`
 }
 
 // +kubebuilder:object:root=true
@@ -72,3 +81,11 @@ type KarpenterMachinePoolList struct {
 func init() {
 	SchemeBuilder.Register(&KarpenterMachinePool{}, &KarpenterMachinePoolList{})
 }
+
+func (in *KarpenterMachinePool) GetConditions() capi.Conditions {
+	return in.Status.Conditions
+}
+
+func (in *KarpenterMachinePool) SetConditions(conditions capi.Conditions) {
+	in.Status.Conditions = conditions
+}