Add feature gate BootstrapFormatIgnition

This commit adds the feature gate BootstrapFormatIgnition that will
control the usage of field `ignition` in AWSMachine & AWSMachineTemplate
and `s3Bucket` in AWSCluster.

If user provides `ignition` field and/or `s3Bucket` without setting the
feature gate then the webhook rejects the request with a validation
error.

Signed-off-by: Suraj Deshmukh <[email protected]>

Author: surajssd

api/v1beta1/awscluster_webhook_test.go

@@ -25,8 +25,10 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	. "github.com/onsi/gomega"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	utilfeature "k8s.io/component-base/featuregate/testing"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
+	"sigs.k8s.io/cluster-api-provider-aws/feature"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	utildefaulting "sigs.k8s.io/cluster-api/util/defaulting"
 )
@@ -223,6 +225,8 @@ func TestAWSCluster_ValidateCreate(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			defer utilfeature.SetFeatureGateDuringTest(t, feature.Gates, feature.BootstrapFormatIgnition, true)()
+
 			cluster := tt.cluster.DeepCopy()
 			cluster.ObjectMeta = metav1.ObjectMeta{
 				GenerateName: "cluster-",

api/v1beta1/awsmachine_webhook.go

@@ -26,6 +26,8 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	"sigs.k8s.io/cluster-api-provider-aws/feature"
 )
 
 // log is for logging in this package.
@@ -159,6 +161,12 @@ func (r *AWSMachine) ignitionEnabled() bool {
 func (r *AWSMachine) validateIgnitionAndCloudInit() field.ErrorList {
 	var allErrs field.ErrorList
 
+	// Feature gate is not enabled but ignition is enabled then send a forbidden error.
+	if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) && r.ignitionEnabled() {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "ignition"),
+			"can be set only if the BootstrapFormatIgnition feature gate is enabled"))
+	}
+
 	if r.ignitionEnabled() && r.cloudInitConfigured() {
 		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "cloudInit"), "cannot be set if spec.ignition is set"))
 	}

api/v1beta1/awsmachinetemplate_webhook.go

@@ -24,6 +24,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
+
+	"sigs.k8s.io/cluster-api-provider-aws/feature"
 )
 
 func (r *AWSMachineTemplate) SetupWebhookWithManager(mgr ctrl.Manager) error {
@@ -113,6 +115,12 @@ func (r *AWSMachineTemplate) ValidateCreate() error {
 	allErrs = append(allErrs, r.validateRootVolume()...)
 	allErrs = append(allErrs, r.validateNonRootVolumes()...)
 
+	// Feature gate is not enabled but ignition is enabled then send a forbidden error.
+	if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) && spec.Ignition != nil {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "ignition"),
+			"can be set only if the BootstrapFormatIgnition feature gate is enabled"))
+	}
+
 	cloudInitConfigured := spec.CloudInit.SecureSecretsBackend != "" || spec.CloudInit.InsecureSkipSecretsManager
 	if cloudInitConfigured && spec.Ignition != nil {
 		allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "template", "spec", "cloudInit"),

api/v1beta1/s3bucket.go

@@ -21,6 +21,8 @@ import (
 	"net"
 
 	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"sigs.k8s.io/cluster-api-provider-aws/feature"
 )
 
 // Validate validates S3Bucket fields.
@@ -35,6 +37,12 @@ func (b *S3Bucket) Validate() []*field.Error {
 		errs = append(errs, field.Required(field.NewPath("spec", "s3Bucket", "name"), "can't be empty"))
 	}
 
+	// Feature gate is not enabled but ignition is enabled then send a forbidden error.
+	if !feature.Gates.Enabled(feature.BootstrapFormatIgnition) {
+		errs = append(errs, field.Forbidden(field.NewPath("spec", "s3Bucket"),
+			"can be set only if the BootstrapFormatIgnition feature gate is enabled"))
+	}
+
 	if b.ControlPlaneIAMInstanceProfile == "" {
 		errs = append(errs,
 			field.Required(field.NewPath("spec", "s3Bucket", "controlPlaneIAMInstanceProfiles"), "can't be empty"))

config/manager/manager.yaml

@@ -19,7 +19,7 @@ spec:
       containers:
       - args:
         - "--leader-elect"
-        - "--feature-gates=EKS=${CAPA_EKS:=true},EKSEnableIAM=${CAPA_EKS_IAM:=false},EKSAllowAddRoles=${CAPA_EKS_ADD_ROLES:=false},EKSFargate=${EXP_EKS_FARGATE:=false},MachinePool=${EXP_MACHINE_POOL:=false},EventBridgeInstanceState=${EVENT_BRIDGE_INSTANCE_STATE:=false},AutoControllerIdentityCreator=${AUTO_CONTROLLER_IDENTITY_CREATOR:=true}"
+        - "--feature-gates=EKS=${CAPA_EKS:=true},EKSEnableIAM=${CAPA_EKS_IAM:=false},EKSAllowAddRoles=${CAPA_EKS_ADD_ROLES:=false},EKSFargate=${EXP_EKS_FARGATE:=false},MachinePool=${EXP_MACHINE_POOL:=false},EventBridgeInstanceState=${EVENT_BRIDGE_INSTANCE_STATE:=false},AutoControllerIdentityCreator=${AUTO_CONTROLLER_IDENTITY_CREATOR:=true},BootstrapFormatIgnition=${EXP_BOOTSTRAP_FORMAT_IGNITION:=false}"
         - "--v=${CAPA_LOGLEVEL:=0}"
         - "--metrics-bind-addr=127.0.0.1:8080"
         image: controller:latest

docs/book/src/development/tilt-setup.md

@@ -56,7 +56,8 @@ Next, create a `tilt-settings.json` file and place it in your local copy of `clu
     "AWS_B64ENCODED_CREDENTIALS": "W2RlZmFZSZnRg==",
     "EXP_EKS_FARGATE": "false",
     "CAPA_EKS_IAM": "false",
-    "CAPA_EKS_ADD_ROLES": "false"
+    "CAPA_EKS_ADD_ROLES": "false",
+    "EXP_BOOTSTRAP_FORMAT_IGNITION": "true"
   },
   "extra_args": {
     "aws": ["--v=2"]
@@ -186,15 +187,15 @@ export EKS_KUBERNETES_VERSION=v1.15
 **Create CAPA managed workload cluster:**
 
 ```bash
-cat templates/cluster-template.yaml 
+cat templates/cluster-template.yaml
 cat templates/cluster-template.yaml | $HOME/go/bin/envsubst > test-cluster.yaml
 kubectl apply -f test-cluster.yaml
 ```
 
 **Create EKS workload cluster:**
 
 ```bash
-cat templates/cluster-template-eks.yaml 
+cat templates/cluster-template-eks.yaml
 cat templates/cluster-template-eks.yaml | $HOME/go/bin/envsubst > test-cluster.yaml
 kubectl apply -f test-cluster.yaml
 ```

feature/feature.go

@@ -62,6 +62,9 @@ const (
 	// owner: @sedefsavas
 	// alpha: v0.6
 	AutoControllerIdentityCreator featuregate.Feature = "AutoControllerIdentityCreator"
+
+	// BootstrapFormatIgnition will allow an user to enable alternate machine bootstrap format, viz. Ignition.
+	BootstrapFormatIgnition featuregate.Feature = "BootstrapFormatIgnition"
 )
 
 func init() {
@@ -79,4 +82,5 @@ var defaultCAPAFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
 	EventBridgeInstanceState:      {Default: false, PreRelease: featuregate.Alpha},
 	MachinePool:                   {Default: false, PreRelease: featuregate.Alpha},
 	AutoControllerIdentityCreator: {Default: true, PreRelease: featuregate.Alpha},
+	BootstrapFormatIgnition:       {Default: false, PreRelease: featuregate.Alpha},
 }

main.go

@@ -392,6 +392,10 @@ func enableGates(ctx context.Context, mgr ctrl.Manager, awsServiceEndpoints []sc
 			os.Exit(1)
 		}
 	}
+
+	if feature.Gates.Enabled(feature.BootstrapFormatIgnition) {
+		setupLog.Info("Enabling Ignition support for machine bootstrap data")
+	}
 }
 func initFlags(fs *pflag.FlagSet) {
 	fs.StringVar(

test/e2e/data/e2e_conf.yaml

@@ -283,6 +283,7 @@ variables:
   # INIT_WITH_KUBERNETES_VERSION are only used by the clusterctl upgrade test to initialize
   # the management cluster to be upgraded.
   INIT_WITH_KUBERNETES_VERSION: "v1.21.6"
+  EXP_BOOTSTRAP_FORMAT_IGNITION: "true"
 
 intervals:
   default/wait-cluster: ["30m", "10s"]