PCP-4952: EKS AL2023 Custom AMI with Dynamic Credentials(STS) Cluster Provisioning is failing

Makefile

@@ -92,7 +92,7 @@ endif
 
 # Release variables
 
-STAGING_REGISTRY ?= gcr.io/spectro-dev-public/cluster-api-aws
+STAGING_REGISTRY ?= us-east1-docker.pkg.dev/spectro-images/dev/$(USER)/cluster-api-aws
 STAGING_BUCKET ?= k8s-staging-cluster-api-aws
 BUCKET ?= $(STAGING_BUCKET)
 PROD_REGISTRY := registry.k8s.io/cluster-api-aws
@@ -126,7 +126,7 @@ ARCH ?= amd64
 # ALL_ARCH = amd64 arm arm64 ppc64le s390x
 ALL_ARCH = amd64 arm64
 
-REGISTRY ?= gcr.io/spectro-dev-public/$(USER)/${RELEASE_LOC}
+REGISTRY ?= us-east1-docker.pkg.dev/spectro-images/dev/$(USER)/${RELEASE_LOC}
 
 # main controller
 CORE_IMAGE_NAME ?= cluster-api-aws-controller

bootstrap/eks/controllers/eksconfig_controller.go

@@ -20,19 +20,18 @@ package controllers
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"fmt"
 	"os"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/eks"
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -45,14 +44,15 @@ import (
 	"sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/internal/userdata"
 	ekscontrolplanev1 "sigs.k8s.io/cluster-api-provider-aws/v2/controlplane/eks/api/v1beta2"
 	expinfrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/feature"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/util/paused"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
 	expclusterv1 "sigs.k8s.io/cluster-api/exp/api/v1beta1"
-	"sigs.k8s.io/cluster-api/feature"
 	"sigs.k8s.io/cluster-api/util"
 	"sigs.k8s.io/cluster-api/util/conditions"
+	kubeconfigutil "sigs.k8s.io/cluster-api/util/kubeconfig"
 	"sigs.k8s.io/cluster-api/util/patch"
 	"sigs.k8s.io/cluster-api/util/predicates"
 )
@@ -323,36 +323,22 @@ func (r *EKSConfigReconciler) joinWorker(ctx context.Context, cluster *clusterv1
 			log.Info("Using mock CA certificate for test environment")
 			nodeInput.CACert = "mock-ca-certificate-for-testing"
 		} else {
-			// Fetch CA cert from EKS API
-			sess, err := session.NewSession(&aws.Config{Region: aws.String(controlPlane.Spec.Region)})
-			if err != nil {
-				log.Error(err, "Failed to create AWS session for EKS API")
-				conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition,
-					eksbootstrapv1.DataSecretGenerationFailedReason,
-					clusterv1.ConditionSeverityWarning,
-					"Failed to create AWS session: %v", err)
-				return ctrl.Result{}, err
+			// Fetch CA cert from KubeConfig secret
+			// We already have the cluster object passed to this function
+			obj := client.ObjectKey{
+				Namespace: cluster.Namespace,
+				Name:      cluster.Name,
 			}
-			eksClient := eks.New(sess)
-			describeInput := &eks.DescribeClusterInput{Name: aws.String(controlPlane.Spec.EKSClusterName)}
-			clusterOut, err := eksClient.DescribeCluster(describeInput)
+			ca, err := r.extractCAFromSecret(ctx, obj)
 			if err != nil {
-				log.Error(err, "Failed to describe EKS cluster for CA cert fetch")
+				log.Error(err, "Failed to extract CA from kubeconfig secret")
 				conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition,
 					eksbootstrapv1.DataSecretGenerationFailedReason,
 					clusterv1.ConditionSeverityWarning,
-					"Failed to describe EKS cluster: %v", err)
+					"Failed to extract CA from kubeconfig secret: %v", err)
 				return ctrl.Result{}, err
-			} else if clusterOut.Cluster != nil && clusterOut.Cluster.CertificateAuthority != nil && clusterOut.Cluster.CertificateAuthority.Data != nil {
-				nodeInput.CACert = *clusterOut.Cluster.CertificateAuthority.Data
-			} else {
-				log.Error(nil, "CA certificate not found in EKS cluster response")
-				conditions.MarkFalse(config, eksbootstrapv1.DataSecretAvailableCondition,
-					eksbootstrapv1.DataSecretGenerationFailedReason,
-					clusterv1.ConditionSeverityWarning,
-					"CA certificate not found in EKS cluster response")
-				return ctrl.Result{}, fmt.Errorf("CA certificate not found in EKS cluster response")
 			}
+			nodeInput.CACert = ca
 		}
 
 		// Get AMI ID from AWSManagedMachinePool's launch template if specified
@@ -581,3 +567,23 @@ func (r *EKSConfigReconciler) updateBootstrapSecret(ctx context.Context, secret
 	}
 	return false, nil
 }
+
+func (r *EKSConfigReconciler) extractCAFromSecret(ctx context.Context, obj client.ObjectKey) (string, error) {
+	data, err := kubeconfigutil.FromSecret(ctx, r.Client, obj)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to get kubeconfig secret %s", obj.Name)
+	}
+	config, err := clientcmd.Load(data)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to parse kubeconfig data from secret %s", obj.Name)
+	}
+
+	// Iterate through all clusters in the kubeconfig and use the first one with CA data
+	for _, cluster := range config.Clusters {
+		if len(cluster.CertificateAuthorityData) > 0 {
+			return base64.StdEncoding.EncodeToString(cluster.CertificateAuthorityData), nil
+		}
+	}
+
+	return "", fmt.Errorf("no cluster with CA data found in kubeconfig")
+}

bootstrap/eks/internal/userdata/node.go

@@ -25,6 +25,7 @@ import (
 	"github.com/alessio/shellescape"
 	"k8s.io/klog/v2"
 	"k8s.io/utils/ptr"
+
 	eksbootstrapv1 "sigs.k8s.io/cluster-api-provider-aws/v2/bootstrap/eks/api/v1beta2"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/exp/api/v1beta2"
 )
@@ -75,7 +76,7 @@ set -o nounset
 {{- end}}
 {{- end}}`
 
-	// Node config part template for AL2023
+	// Node config part template for AL2023.
 	nodeConfigPartTemplate = `
 --{{.Boundary}}
 Content-Type: application/node.eks.aws
@@ -363,7 +364,7 @@ func generateAL2023UserData(input *NodeInput) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
-// getCapacityTypeString returns the string representation of the capacity type
+// getCapacityTypeString returns the string representation of the capacity type.
 func (ni *NodeInput) getCapacityTypeString() string {
 	if ni.CapacityType == nil {
 		return "ON_DEMAND"
@@ -378,7 +379,7 @@ func (ni *NodeInput) getCapacityTypeString() string {
 	}
 }
 
-// validateAL2023Input validates the input for AL2023 user data generation
+// validateAL2023Input validates the input for AL2023 user data generation.
 func validateAL2023Input(input *NodeInput) error {
 	if input.APIServerEndpoint == "" {
 		return fmt.Errorf("API server endpoint is required for AL2023")