diff --git a/api/v1alpha1/aws_node_types.go b/api/v1alpha1/aws_node_types.go index 0fbfbef6f..a9fa54d75 100644 --- a/api/v1alpha1/aws_node_types.go +++ b/api/v1alpha1/aws_node_types.go @@ -25,7 +25,13 @@ type AWSControlPlaneNodeSpec struct { } type AWSWorkerNodeSpec struct { - // The IAM instance profile to use for the cluster Machines. + // The failureDomain the machine deployment will use. + // +kubebuilder:validation:Optional + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:MaxLength=128 + FailureDomain string `json:"failureDomain,omitempty"` + +// The IAM instance profile to use for the cluster Machines. // +kubebuilder:validation:Optional // +kubebuilder:default=nodes.cluster-api-provider-aws.sigs.k8s.io // +kubebuilder:validation:MinLength=1 diff --git a/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml index 9dc56f148..f3e4d57b1 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_awsworkernodeconfigs.yaml @@ -100,6 +100,11 @@ spec: type: string type: object type: object + failureDomain: + description: The failureDomain the machine deployment will use. + maxLength: 128 + minLength: 1 + type: string iamInstanceProfile: default: nodes.cluster-api-provider-aws.sigs.k8s.io description: The IAM instance profile to use for the cluster Machines. diff --git a/api/v1alpha1/crds/caren.nutanix.com_eksworkernodeconfigs.yaml b/api/v1alpha1/crds/caren.nutanix.com_eksworkernodeconfigs.yaml index 239da5f76..dbaeb60a8 100644 --- a/api/v1alpha1/crds/caren.nutanix.com_eksworkernodeconfigs.yaml +++ b/api/v1alpha1/crds/caren.nutanix.com_eksworkernodeconfigs.yaml @@ -100,6 +100,11 @@ spec: type: string type: object type: object + failureDomain: + description: The failureDomain the machine deployment will use. + maxLength: 128 + minLength: 1 + type: string iamInstanceProfile: default: nodes.cluster-api-provider-aws.sigs.k8s.io description: The IAM instance profile to use for the cluster Machines. diff --git a/common/pkg/testutils/capitest/request/items.go b/common/pkg/testutils/capitest/request/items.go index cde5bff4c..db4ab82ee 100644 --- a/common/pkg/testutils/capitest/request/items.go +++ b/common/pkg/testutils/capitest/request/items.go @@ -245,3 +245,33 @@ func NewWorkerDockerMachineTemplateRequestItem( uid, ) } + +func NewWorkerMachineDeploymentRequestItem( + uid types.UID, +) runtimehooksv1.GeneratePatchesRequestItem { + return NewRequestItem( + &clusterv1.MachineDeployment{ + TypeMeta: metav1.TypeMeta{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "MachineDeployment", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-machinedeployment", + Namespace: Namespace, + }, + Spec: clusterv1.MachineDeploymentSpec{ + Template: clusterv1.MachineTemplateSpec{ + Spec: clusterv1.MachineSpec{ + ClusterName: ClusterName, + }, + }, + }, + }, + &runtimehooksv1.HolderReference{ + APIVersion: clusterv1.GroupVersion.String(), + Kind: "MachineDeployment", + FieldPath: "", + }, + uid, + ) +} diff --git a/docs/content/customization/aws/failure-domain.md b/docs/content/customization/aws/failure-domain.md new file mode 100644 index 000000000..116827582 --- /dev/null +++ b/docs/content/customization/aws/failure-domain.md @@ -0,0 +1,73 @@ ++++ +title = "AWS Failure Domain" ++++ + +The AWS failure domain customization allows the user to specify the AWS availability zone (failure domain) for worker node deployments. +This customization can be applied to individual MachineDeployments to distribute worker nodes across different availability zones for high availability. +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To specify a failure domain for worker nodes, use the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: workerConfig + value: + aws: + failureDomain: us-west-2a +``` + +You can customize individual MachineDeployments by using the overrides field to deploy workers across multiple availability zones: + +```yaml +spec: + topology: + # ... + workers: + machineDeployments: + - class: default-worker + name: md-0 + variables: + overrides: + - name: workerConfig + value: + aws: + failureDomain: us-west-2a + - class: default-worker + name: md-1 + variables: + overrides: + - name: workerConfig + value: + aws: + failureDomain: us-west-2b + - class: default-worker + name: md-2 + variables: + overrides: + - name: workerConfig + value: + aws: + failureDomain: us-west-2c +``` + +## Resulting CAPA Configuration + +Applying this configuration will result in the following value being set: + +- worker `MachineDeployment`: + + - ```yaml + spec: + template: + spec: + failureDomain: us-west-2a + ``` diff --git a/docs/content/customization/eks/failure-domain.md b/docs/content/customization/eks/failure-domain.md new file mode 100644 index 000000000..116827582 --- /dev/null +++ b/docs/content/customization/eks/failure-domain.md @@ -0,0 +1,73 @@ ++++ +title = "AWS Failure Domain" ++++ + +The AWS failure domain customization allows the user to specify the AWS availability zone (failure domain) for worker node deployments. +This customization can be applied to individual MachineDeployments to distribute worker nodes across different availability zones for high availability. +This customization will be available when the +[provider-specific cluster configuration patch]({{< ref "..">}}) is included in the `ClusterClass`. + +## Example + +To specify a failure domain for worker nodes, use the following configuration: + +```yaml +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: +spec: + topology: + variables: + - name: workerConfig + value: + aws: + failureDomain: us-west-2a +``` + +You can customize individual MachineDeployments by using the overrides field to deploy workers across multiple availability zones: + +```yaml +spec: + topology: + # ... + workers: + machineDeployments: + - class: default-worker + name: md-0 + variables: + overrides: + - name: workerConfig + value: + aws: + failureDomain: us-west-2a + - class: default-worker + name: md-1 + variables: + overrides: + - name: workerConfig + value: + aws: + failureDomain: us-west-2b + - class: default-worker + name: md-2 + variables: + overrides: + - name: workerConfig + value: + aws: + failureDomain: us-west-2c +``` + +## Resulting CAPA Configuration + +Applying this configuration will result in the following value being set: + +- worker `MachineDeployment`: + + - ```yaml + spec: + template: + spec: + failureDomain: us-west-2a + ``` diff --git a/pkg/handlers/aws/mutation/failuredomain/inject_suite_test.go b/pkg/handlers/aws/mutation/failuredomain/inject_suite_test.go new file mode 100644 index 000000000..d4781e130 --- /dev/null +++ b/pkg/handlers/aws/mutation/failuredomain/inject_suite_test.go @@ -0,0 +1,16 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestFailureDomainPatch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "AWS failure domain mutator suite") +} diff --git a/pkg/handlers/aws/mutation/failuredomain/inject_worker.go b/pkg/handlers/aws/mutation/failuredomain/inject_worker.go new file mode 100644 index 000000000..491f7e7b2 --- /dev/null +++ b/pkg/handlers/aws/mutation/failuredomain/inject_worker.go @@ -0,0 +1,103 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + "context" + + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables" +) + +const ( + // VariableName is the external patch variable name. + VariableName = "failureDomain" +) + +type awsFailureDomainWorkerPatchHandler struct { + variableName string + variableFieldPath []string +} + +func NewWorkerPatch() *awsFailureDomainWorkerPatchHandler { + return NewAWSFailureDomainWorkerPatchHandler( + v1alpha1.WorkerConfigVariableName, + v1alpha1.AWSVariableName, + VariableName, + ) +} + +func NewAWSFailureDomainWorkerPatchHandler( + variableName string, + variableFieldPath ...string, +) *awsFailureDomainWorkerPatchHandler { + return &awsFailureDomainWorkerPatchHandler{ + variableName: variableName, + variableFieldPath: variableFieldPath, + } +} + +func (h *awsFailureDomainWorkerPatchHandler) Mutate( + ctx context.Context, + obj *unstructured.Unstructured, + vars map[string]apiextensionsv1.JSON, + holderRef runtimehooksv1.HolderReference, + _ client.ObjectKey, + _ mutation.ClusterGetter, +) error { + log := ctrl.LoggerFrom(ctx).WithValues( + "holderRef", holderRef, + ) + + failureDomainVar, err := variables.Get[string]( + vars, + h.variableName, + h.variableFieldPath..., + ) + if err != nil { + if variables.IsNotFoundError(err) { + log.V(5).Info("AWS failure domain variable for worker not defined") + return nil + } + return err + } + + log = log.WithValues( + "variableName", + h.variableName, + "variableFieldPath", + h.variableFieldPath, + "variableValue", + failureDomainVar, + ) + + // Check if this is a MachineDeployment + if obj.GetKind() != "MachineDeployment" || obj.GetAPIVersion() != clusterv1.GroupVersion.String() { + log.V(5).Info("not a MachineDeployment, skipping") + return nil + } + + log.WithValues( + "patchedObjectKind", obj.GetKind(), + "patchedObjectName", client.ObjectKeyFromObject(obj), + ).Info("setting failure domain in worker MachineDeployment spec") + + if err := unstructured.SetNestedField( + obj.Object, + failureDomainVar, + "spec", "template", "spec", "failureDomain", + ); err != nil { + return err + } + + return nil +} diff --git a/pkg/handlers/aws/mutation/failuredomain/inject_worker_test.go b/pkg/handlers/aws/mutation/failuredomain/inject_worker_test.go new file mode 100644 index 000000000..d771ab376 --- /dev/null +++ b/pkg/handlers/aws/mutation/failuredomain/inject_worker_test.go @@ -0,0 +1,60 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +var _ = Describe("Generate AMI patches for Worker", func() { + patchGenerator := func() mutation.GeneratePatches { + return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewWorkerPatch()).(mutation.GeneratePatches) + } + + testDefs := []capitest.PatchTestDef{ + { + Name: "failure domain for workers set", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.WorkerConfigVariableName, + "us-west-2a", + v1alpha1.AWSVariableName, + VariableName, + ), + capitest.VariableWithValue( + runtimehooksv1.BuiltinsName, + apiextensionsv1.JSON{ + Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`), + }, + ), + }, + RequestItem: request.NewWorkerMachineDeploymentRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ + Operation: "add", + Path: "/spec/template/spec/failureDomain", + ValueMatcher: gomega.Equal("us-west-2a"), + }}, + }, + } + + // create test node for each case + for _, tt := range testDefs { + It(tt.Name, func() { + capitest.AssertGeneratePatches( + GinkgoT(), + patchGenerator, + &tt, + ) + }) + } +}) diff --git a/pkg/handlers/aws/mutation/failuredomain/variables_test.go b/pkg/handlers/aws/mutation/failuredomain/variables_test.go new file mode 100644 index 000000000..c95b122c5 --- /dev/null +++ b/pkg/handlers/aws/mutation/failuredomain/variables_test.go @@ -0,0 +1,32 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + "testing" + + "k8s.io/utils/ptr" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + awsworkerconfig "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/workerconfig" +) + +func TestVariableValidation(t *testing.T) { + capitest.ValidateDiscoverVariables( + t, + v1alpha1.WorkerConfigVariableName, + ptr.To(v1alpha1.AWSWorkerNodeConfig{}.VariableSchema()), + false, + awsworkerconfig.NewVariable, + capitest.VariableTestDef{ + Name: "specified failure domain", + Vals: v1alpha1.AWSWorkerNodeConfigSpec{ + AWS: &v1alpha1.AWSWorkerNodeSpec{ + FailureDomain: "us-west-2a", + }, + }, + }, + ) +} diff --git a/pkg/handlers/aws/mutation/metapatch_handler.go b/pkg/handlers/aws/mutation/metapatch_handler.go index 6c3145ed0..e930d96e1 100644 --- a/pkg/handlers/aws/mutation/metapatch_handler.go +++ b/pkg/handlers/aws/mutation/metapatch_handler.go @@ -11,6 +11,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/ami" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/cni/calico" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/controlplaneloadbalancer" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/failuredomain" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/iaminstanceprofile" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/identityref" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/instancetype" @@ -54,6 +55,7 @@ func MetaPatchHandler(mgr manager.Manager) handlers.Named { func MetaWorkerPatchHandler(mgr manager.Manager) handlers.Named { patchHandlers := []mutation.MetaMutator{ tags.NewWorkerPatch(), + failuredomain.NewWorkerPatch(), iaminstanceprofile.NewWorkerPatch(), instancetype.NewWorkerPatch(), ami.NewWorkerPatch(), diff --git a/pkg/handlers/eks/mutation/failuredomain/inject_suite_test.go b/pkg/handlers/eks/mutation/failuredomain/inject_suite_test.go new file mode 100644 index 000000000..ee3d7de8a --- /dev/null +++ b/pkg/handlers/eks/mutation/failuredomain/inject_suite_test.go @@ -0,0 +1,16 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestFailureDomainPatch(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "EKS failure domain mutator suite") +} diff --git a/pkg/handlers/eks/mutation/failuredomain/inject_worker.go b/pkg/handlers/eks/mutation/failuredomain/inject_worker.go new file mode 100644 index 000000000..d40f70f4f --- /dev/null +++ b/pkg/handlers/eks/mutation/failuredomain/inject_worker.go @@ -0,0 +1,18 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + awsfailuredomain "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/failuredomain" +) + +func NewWorkerPatch() mutation.MetaMutator { + return awsfailuredomain.NewAWSFailureDomainWorkerPatchHandler( + v1alpha1.WorkerConfigVariableName, + v1alpha1.EKSVariableName, + awsfailuredomain.VariableName, + ) +} diff --git a/pkg/handlers/eks/mutation/failuredomain/inject_worker_test.go b/pkg/handlers/eks/mutation/failuredomain/inject_worker_test.go new file mode 100644 index 000000000..07ce2338e --- /dev/null +++ b/pkg/handlers/eks/mutation/failuredomain/inject_worker_test.go @@ -0,0 +1,61 @@ +// Copyright 2023 Nutanix. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package failuredomain + +import ( + . "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1" + + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/testutils/capitest/request" + awsfailuredomain "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/aws/mutation/failuredomain" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/test/helpers" +) + +var _ = Describe("Generate failure domain patches for Worker", func() { + patchGenerator := func() mutation.GeneratePatches { + return mutation.NewMetaGeneratePatchesHandler("", helpers.TestEnv.Client, NewWorkerPatch()).(mutation.GeneratePatches) + } + + testDefs := []capitest.PatchTestDef{ + { + Name: "failure domain for workers set", + Vars: []runtimehooksv1.Variable{ + capitest.VariableWithValue( + v1alpha1.WorkerConfigVariableName, + "us-west-2b", + v1alpha1.EKSVariableName, + awsfailuredomain.VariableName, + ), + capitest.VariableWithValue( + runtimehooksv1.BuiltinsName, + apiextensionsv1.JSON{ + Raw: []byte(`{"machineDeployment": {"class": "a-worker"}}`), + }, + ), + }, + RequestItem: request.NewWorkerMachineDeploymentRequestItem(""), + ExpectedPatchMatchers: []capitest.JSONPatchMatcher{{ + Operation: "add", + Path: "/spec/template/spec/failureDomain", + ValueMatcher: gomega.Equal("us-west-2b"), + }}, + }, + } + + // create test node for each case + for _, tt := range testDefs { + It(tt.Name, func() { + capitest.AssertGeneratePatches( + GinkgoT(), + patchGenerator, + &tt, + ) + }) + } +}) diff --git a/pkg/handlers/eks/mutation/metapatch_handler.go b/pkg/handlers/eks/mutation/metapatch_handler.go index 1461dcc62..b520ea0d9 100644 --- a/pkg/handlers/eks/mutation/metapatch_handler.go +++ b/pkg/handlers/eks/mutation/metapatch_handler.go @@ -9,6 +9,7 @@ import ( "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/eks/mutation/ami" + "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/eks/mutation/failuredomain" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/eks/mutation/iaminstanceprofile" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/eks/mutation/identityref" "github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/eks/mutation/instancetype" @@ -40,6 +41,7 @@ func MetaPatchHandler(mgr manager.Manager) handlers.Named { // MetaWorkerPatchHandler returns a meta patch handler for mutating CAPA workers. func MetaWorkerPatchHandler(mgr manager.Manager) handlers.Named { patchHandlers := []mutation.MetaMutator{ + failuredomain.NewWorkerPatch(), iaminstanceprofile.NewWorkerPatch(), instancetype.NewWorkerPatch(), ami.NewWorkerPatch(),