PCP-4952: EKS AL2023 Custom AMI with Dynamic Credentials(STS) Cluster Provisioning is failing

Makefile

@@ -92,7 +92,7 @@ endif
 
 # Release variables
 
-STAGING_REGISTRY ?= gcr.io/spectro-dev-public/cluster-api-aws
+STAGING_REGISTRY ?= us-east1-docker.pkg.dev/spectro-images/dev/$(USER)/cluster-api-aws
 STAGING_BUCKET ?= k8s-staging-cluster-api-aws
 BUCKET ?= $(STAGING_BUCKET)
 PROD_REGISTRY := registry.k8s.io/cluster-api-aws
@@ -126,7 +126,7 @@ ARCH ?= amd64
 # ALL_ARCH = amd64 arm arm64 ppc64le s390x
 ALL_ARCH = amd64 arm64
 
-REGISTRY ?= gcr.io/spectro-dev-public/$(USER)/${RELEASE_LOC}
+REGISTRY ?= us-east1-docker.pkg.dev/spectro-images/dev/$(USER)/${RELEASE_LOC}
 
 # main controller
 CORE_IMAGE_NAME ?= cluster-api-aws-controller

bootstrap/eks/controllers/eksconfig_controller.go

@@ -20,19 +20,18 @@ package controllers
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -45,14 +44,15 @@ import (
 	"sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/internal/userdata"
 	ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
 	expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/feature"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/util/paused"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
 	expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
-	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
+	kubeconfigutil "sigs.k8s.io/cluster-api/util/kubeconfig"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
 )
@@ -323,36 +323,22 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1
 			log.Info("Using mock CA certificate for test environment")
 			nodeInput.CACert = "mock-ca-certificate-for-testing"
 		} else {
-			// Fetch CA cert from EKS API
-			sess, err := session.NewSession(&aws.Config{Region: aws.String(controlPlane.Spec.Region)})
-			if err != nil {
-				log.Error(err, "Failed to create AWS session for EKS API")
-				conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition,
-					eksbootstrapv1.DataSecretGenerationFailedReason,
-					clusterv1.ConditionSeverityWarning,
-					"Failed to create AWS session: %v", err)
-				return ctrl.Result{}, err
+			// Fetch CA cert from KubeConfig secret
+			// We already have the cluster object passed to this function
+			obj := client.ObjectKey{
+				Namespace: cluster.Namespace,
+				Name:      cluster.Name,
 			}
-			eksClient := eks.New(sess)
-			describeInput := &eks.DescribeClusterInput{Name: aws.String(controlPlane.Spec.EKSClusterName)}
-			clusterOut, err := eksClient.DescribeCluster(describeInput)
+			ca, err := extractCAFromSecret(ctx, r.Client, obj)
 			if err != nil {
-				log.Error(err, "Failed to describe EKS cluster for CA cert fetch")
+				log.Error(err, "Failed to extract CA from kubeconfig secret")
 				conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition,
 					eksbootstrapv1.DataSecretGenerationFailedReason,
 					clusterv1.ConditionSeverityWarning,
-					"Failed to describe EKS cluster: %v", err)
+					"Failed to extract CA from kubeconfig secret: %v", err)
 				return ctrl.Result{}, err
-			} else if clusterOut.Cluster != nil && clusterOut.Cluster.CertificateAuthority != nil && clusterOut.Cluster.CertificateAuthority.Data != nil {
-				nodeInput.CACert = *clusterOut.Cluster.CertificateAuthority.Data
-			} else {
-				log.Error(nil, "CA certificate not found in EKS cluster response")
-				conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition,
-					eksbootstrapv1.DataSecretGenerationFailedReason,
-					clusterv1.ConditionSeverityWarning,
-					"CA certificate not found in EKS cluster response")
-				return ctrl.Result{}, fmt.Errorf("CA certificate not found in EKS cluster response")
 			}
+			nodeInput.CACert = ca
 		}
 
 		// Get AMI ID from AWSManagedMachinePool's launch template if specified
@@ -581,3 +567,23 @@ func (r *EKSConfigReconciler) updateBootstrapSecret(ctx context.Context, secret
 	}
 	return false, nil
 }
+
+func extractCAFromSecret(ctx context.Context, c client.Client, obj client.ObjectKey) (string, error) {
+	data, err := kubeconfigutil.FromSecret(ctx, c, obj)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to get kubeconfig secret %s", obj.Name)
+	}
+	config, err := clientcmd.Load(data)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse kubeconfig data from secret %s", obj.Name)
+	}
+
+	// Iterate through all clusters in the kubeconfig and use the first one with CA data
+	for _, cluster := range config.Clusters {
+		if len(cluster.CertificateAuthorityData) > 0 {
+			return base64.StdEncoding.EncodeToString(cluster.CertificateAuthorityData), nil
+		}
+	}
+
+	return "", fmt.Errorf("no cluster with CA data found in kubeconfig")
+}

bootstrap/eks/internal/userdata/node.go

@@ -25,6 +25,7 @@ import (
 	"github.com/alessio/shellescape"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
+
 	eksbootstrapv1 "sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/api/v1beta2"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
 )
@@ -51,13 +52,7 @@ runcmd:
 {{- template "mounts" .Mounts}}
 `
 
-	// Common MIME header and boundary template
-	mimeHeaderTemplate = `MIME-Version: 1.0
-Content-Type: multipart/mixed; boundary="{{.Boundary}}"
-
-`
-
-	// Shell script part template for AL2023
+	// Shell script part template for AL2023.
 	shellScriptPartTemplate = `--{{.Boundary}}
 Content-Type: text/x-shellscript; charset="us-ascii"
 
@@ -75,7 +70,7 @@ set -o nounset
 {{- end}}
 {{- end}}`
 
-	// Node config part template for AL2023
+	// Node config part template for AL2023.
 	nodeConfigPartTemplate = `
 --{{.Boundary}}
 Content-Type: application/node.eks.aws
@@ -98,111 +93,8 @@ spec:
     - "--node-labels={{if and .KubeletExtraArgs (index .KubeletExtraArgs "node-labels")}}{{index .KubeletExtraArgs "node-labels"}}{{else}}eks.amazonaws.com/nodegroup-image={{if .AMIImageID}}{{.AMIImageID}}{{end}},eks.amazonaws.com/capacityType={{if .CapacityType}}{{.CapacityType}}{{else}}ON_DEMAND{{end}},eks.amazonaws.com/nodegroup={{.NodeGroupName}}{{end}}"
 
 --{{.Boundary}}--`
-
-	// AL2023-specific templates.
-	al2023KubeletExtraArgsTemplate = `{{- define "al2023KubeletExtraArgs" -}}
-{{- if . }}
-    - "--node-labels={{range $k, $v := .}}{{$k}}={{$v}}{{end}}"
-{{- end -}}
-{{- end -}}`
-
-	al2023ContainerRuntimeTemplate = `{{- define "al2023ContainerRuntime" -}}
-{{- if . -}}
-  containerRuntime: {{.}}
-{{- end -}}
-{{- end -}}`
-
-	al2023DockerConfigTemplate = `{{- define "al2023DockerConfig" -}}
-{{- if and . (ne . "''") -}}
-  dockerConfig: {{.}}
-{{- end -}}
-{{- end -}}`
-
-	al2023APIRetryAttemptsTemplate = `{{- define "al2023APIRetryAttempts" -}}
-{{- if . -}}
-  apiRetryAttempts: {{.}}
-{{- end -}}
-{{- end -}}`
-
-	al2023PauseContainerTemplate = `{{- define "al2023PauseContainer" -}}
-{{- if and .AccountNumber .Version -}}
-  pauseContainer:
-    accountNumber: {{.AccountNumber}}
-    version: {{.Version}}
-{{- end -}}
-{{- end -}}`
-
-	al2023FilesTemplate = `{{- define "al2023Files" -}}
-{{- if . -}}
-  files:{{ range . }}
-    - path: {{.Path}}
-      content: |
-{{.Content | Indent 8}}{{ if ne .Owner "" }}
-      owner: {{.Owner}}{{ end }}{{ if ne .Permissions "" }}
-      permissions: '{{.Permissions}}'{{ end }}{{ end }}
-{{- end -}}
-{{- end -}}`
-
-	al2023DiskSetupTemplate = `{{- define "al2023DiskSetup" -}}
-{{- if . -}}
-  diskSetup:{{ if .Partitions }}
-    partitions:{{ range .Partitions }}
-      - device: {{.Device}}
-        layout: {{.Layout}}{{ if .Overwrite }}
-        overwrite: {{.Overwrite}}{{ end }}{{ if .TableType }}
-        tableType: {{.TableType}}{{ end }}{{ end }}{{ end }}{{ if .Filesystems }}
-    filesystems:{{ range .Filesystems }}
-      - device: {{.Device}}
-        filesystem: {{.Filesystem}}
-        label: {{.Label}}{{ if .Partition }}
-        partition: {{.Partition}}{{ end }}{{ if .Overwrite }}
-        overwrite: {{.Overwrite}}{{ end }}{{ if .ExtraOpts }}
-        extraOpts:{{ range .ExtraOpts }}
-          - {{.}}{{ end }}{{ end }}{{ end }}{{ end }}
-{{- end -}}
-{{- end -}}`
-
-	al2023MountsTemplate = `{{- define "al2023Mounts" -}}
-{{- if . -}}
-  mounts:{{ range . }}
-    -{{ range . }}
-      - {{.}}{{ end }}{{ end }}
-{{- end -}}
-{{- end -}}`
-
-	al2023UsersTemplate = `{{- define "al2023Users" -}}
-{{- if . -}}
-  users:{{ range . }}
-    - name: {{.Name}}{{ if .Gecos }}
-      gecos: {{.Gecos}}{{ end }}{{ if .Groups }}
-      groups: {{.Groups}}{{ end }}{{ if .HomeDir }}
-      homeDir: {{.HomeDir}}{{ end }}{{ if .Inactive }}
-      inactive: {{.Inactive}}{{ end }}{{ if .Shell }}
-      shell: {{.Shell}}{{ end }}{{ if .Passwd }}
-      passwd: {{.Passwd}}{{ end }}{{ if .PrimaryGroup }}
-      primaryGroup: {{.PrimaryGroup}}{{ end }}{{ if .LockPassword }}
-      lockPassword: {{.LockPassword}}{{ end }}{{ if .Sudo }}
-      sudo: {{.Sudo}}{{ end }}{{ if .SSHAuthorizedKeys }}
-      sshAuthorizedKeys:{{ range .SSHAuthorizedKeys }}
-        - {{.}}{{ end }}{{ end }}{{ end }}
-{{- end -}}
-{{- end -}}`
-
-	al2023NTPTemplate = `{{- define "al2023NTP" -}}
-{{- if . -}}
-  ntp:{{ if .Enabled }}
-    enabled: true{{ end }}{{ if .Servers }}
-    servers:{{ range .Servers }}
-      - {{.}}{{ end }}{{ end }}
-{{- end -}}
-{{- end -}}`
 )
 
-// NodeUserData is responsible for generating userdata for EKS nodes.
-type NodeUserData struct {
-	input *NodeInput
-}
-
 // NodeInput contains all the information required to generate user data for a node.
 type NodeInput struct {
 	ClusterName           string
@@ -339,7 +231,7 @@ func generateAL2023UserData(input *NodeInput) ([]byte, error) {
 	var buf bytes.Buffer
 
 	// Write MIME header
-	if _, err := buf.WriteString(fmt.Sprintf("MIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=\"%s\"\n\n", input.Boundary)); err != nil {
+	if _, err := buf.WriteString(fmt.Sprintf("MIME-Version: 1.0\nContent-Type: multipart/mixed; boundary=%q\n\n", input.Boundary)); err != nil {
 		return nil, fmt.Errorf("failed to write MIME header: %v", err)
 	}
 
@@ -363,7 +255,7 @@ func generateAL2023UserData(input *NodeInput) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-// getCapacityTypeString returns the string representation of the capacity type
+// getCapacityTypeString returns the string representation of the capacity type.
 func (ni *NodeInput) getCapacityTypeString() string {
 	if ni.CapacityType == nil {
 		return "ON_DEMAND"
@@ -378,7 +270,7 @@ func (ni *NodeInput) getCapacityTypeString() string {
 	}
 }
 
-// validateAL2023Input validates the input for AL2023 user data generation
+// validateAL2023Input validates the input for AL2023 user data generation.
 func validateAL2023Input(input *NodeInput) error {
 	if input.APIServerEndpoint == "" {
 		return fmt.Errorf("API server endpoint is required for AL2023")