debug debug

Author: LiangquanLi930

test/e2e/shared/aws.go

@@ -45,6 +45,7 @@ import (
 	iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/client"
 	awscreds "github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
@@ -774,9 +775,36 @@ func deleteCloudFormationStack(prov client.ConfigProvider, t *cfn_bootstrap.Temp
 }
 
 func ensureTestImageUploaded(e2eCtx *E2EContext) error {
+	By("ensureTestImageUploaded: Creating AWS session for ECR Public")
 	sessionForRepo := NewAWSSessionRepoWithKey(e2eCtx.Environment.BootstrapAccessKey)
 
+	if sess, ok := sessionForRepo.(*session.Session); ok {
+		creds, err := sess.Config.Credentials.Get()
+		if err != nil {
+			By(fmt.Sprintf("ensureTestImageUploaded: Failed to get credentials from session: %v", err))
+			return fmt.Errorf("failed to get credentials: %w", err)
+		}
+		By(fmt.Sprintf("ensureTestImageUploaded: Session credentials - AccessKeyId: %s, ProviderName: %s",
+			creds.AccessKeyID, creds.ProviderName))
+	} else {
+		By("ensureTestImageUploaded: WARNING: Session is not *session.Session type")
+	}
+
+	By("ensureTestImageUploaded: Creating ECR Public client")
 	ecrSvc := ecrpublic.New(sessionForRepo)
+
+	By("ensureTestImageUploaded: Testing ECR Public connection")
+	_, err := ecrSvc.DescribeRegistries(&ecrpublic.DescribeRegistriesInput{})
+	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded: Failed to connect to ECR Public: %v", err))
+		if awsErr, ok := err.(awserr.Error); ok {
+			By(fmt.Sprintf("ensureTestImageUploaded: AWS Error - Code: %s, Message: %s",
+				awsErr.Code(), awsErr.Message()))
+		}
+		return fmt.Errorf("failed to connect to ECR Public: %w", err)
+	}
+	By("ensureTestImageUploaded: Successfully connected to ECR Public")
+
 	repoName := ""
 	if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
 		output, err := ecrSvc.CreateRepository(&ecrpublic.CreateRepositoryInput{
@@ -788,10 +816,12 @@ func ensureTestImageUploaded(e2eCtx *E2EContext) error {
 
 		if err != nil {
 			if !awserrors.IsRepositoryExists(err) {
+				By(fmt.Sprintf("ensureTestImageUploaded: Failed to create repository: %v", err))
 				return false, err
 			}
 			out, err := ecrSvc.DescribeRepositories(&ecrpublic.DescribeRepositoriesInput{RepositoryNames: []*string{aws.String("capa/update")}})
 			if err != nil || len(out.Repositories) == 0 {
+				By(fmt.Sprintf("ensureTestImageUploaded: Failed to describe existing repository: %v", err))
 				return false, err
 			}
 			repoName = aws.StringValue(out.Repositories[0].RepositoryUri)
@@ -804,25 +834,38 @@ func ensureTestImageUploaded(e2eCtx *E2EContext) error {
 		return err
 	}
 
+	By(fmt.Sprintf("ensureTestImageUploaded: Repository ready - %s", repoName))
+
+	By("ensureTestImageUploaded: Inspecting Docker image")
 	cmd := exec.Command("docker", "inspect", "--format='{{index .Id}}'", "gcr.io/k8s-staging-cluster-api/capa-manager:e2e")
 	var stdOut bytes.Buffer
 	cmd.Stdout = &stdOut
-	err := cmd.Run()
+	err = cmd.Run()
 	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded: Failed to inspect Docker image: %v", err))
 		return err
 	}
 
 	imageSha := strings.ReplaceAll(strings.TrimSuffix(stdOut.String(), "\n"), "'", "")
+	By(fmt.Sprintf("ensureTestImageUploaded: Docker image SHA: %s", imageSha))
 
 	ecrImageName := repoName + ":e2e"
+	By(fmt.Sprintf("ensureTestImageUploaded: Tagging image as %s", ecrImageName))
 	cmd = exec.Command("docker", "tag", imageSha, ecrImageName) //nolint:gosec
 	err = cmd.Run()
 	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded: Failed to tag Docker image: %v", err))
 		return err
 	}
 
+	By("ensureTestImageUploaded: Getting ECR authorization token")
 	outToken, err := ecrSvc.GetAuthorizationToken(&ecrpublic.GetAuthorizationTokenInput{})
 	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded: Failed to get authorization token: %v", err))
+		if awsErr, ok := err.(awserr.Error); ok {
+			By(fmt.Sprintf("ensureTestImageUploaded: AWS Error getting token - Code: %s, Message: %s",
+				awsErr.Code(), awsErr.Message()))
+		}
 		return err
 	}
 
@@ -831,22 +874,31 @@ func ensureTestImageUploaded(e2eCtx *E2EContext) error {
 
 	strList := strings.Split(string(decodedUsernamePassword), ":")
 	if len(strList) != 2 {
+		By("ensureTestImageUploaded: Failed to decode ECR authentication token")
 		return errors.New("failed to decode ECR authentication token")
 	}
 
+	By("ensureTestImageUploaded: Logging into ECR Public")
 	cmd = exec.Command("docker", "login", "--username", strList[0], "--password", strList[1], "public.ecr.aws") //nolint:gosec
 	err = cmd.Run()
 	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded: Failed to login to ECR Public: %v", err))
 		return err
 	}
 
+	By("ensureTestImageUploaded: Pushing image to ECR Public")
 	cmd = exec.Command("docker", "push", ecrImageName)
 	err = cmd.Run()
 	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded: Failed to push image: %v", err))
 		return err
 	}
+
+	By("ensureTestImageUploaded: Image pushed successfully")
 	e2eCtx.E2EConfig.Variables["CAPI_IMAGES_REGISTRY"] = repoName
 	e2eCtx.E2EConfig.Variables["E2E_IMAGE_TAG"] = "e2e"
+
+	By("ensureTestImageUploaded: Function completed successfully")
 	return nil
 }
 
@@ -915,8 +967,9 @@ func encodeCredentials(accessKey *iamtypes.AccessKey, region string) string {
 // newUserAccessKey generates a new AWS Access Key pair based off of the
 // bootstrap user. This tests that the CloudFormation policy is correct.
 func newUserAccessKey(ctx context.Context, cfg *awsv2.Config, userName string) *iamtypes.AccessKey {
-	iamSvc := iam.NewFromConfig(*cfg)
+	By(fmt.Sprintf("newUserAccessKey: Starting for user: %s", userName))
 
+	iamSvc := iam.NewFromConfig(*cfg)
 	keyOuts, _ := iamSvc.ListAccessKeys(ctx, &iam.ListAccessKeysInput{
 		UserName: aws.String(userName),
 	})
@@ -928,6 +981,7 @@ func newUserAccessKey(ctx context.Context, cfg *awsv2.Config, userName string) *
 		})
 		Expect(err).NotTo(HaveOccurred())
 	}
+
 	By(fmt.Sprintf("Creating an access key: user-name=%s", userName))
 	out, err := iamSvc.CreateAccessKey(ctx, &iam.CreateAccessKeyInput{UserName: aws.String(userName)})
 	Expect(err).NotTo(HaveOccurred())

test/e2e/shared/suite.go

@@ -32,6 +32,8 @@ import (
 	"time"
 
 	iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types"
+	"github.com/aws/aws-sdk-go/aws/awserr"
+	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/gofrs/flock"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -158,7 +160,74 @@ func Node1BeforeSuite(e2eCtx *E2EContext) []byte {
 	e2eCtx.Environment.BootstrapAccessKey = newUserAccessKey(context.TODO(), e2eCtx.AWSSessionV2, bootstrapTemplate.Spec.BootstrapUser.UserName)
 	e2eCtx.BootstrapUserAWSSession = NewAWSSessionWithKey(e2eCtx.Environment.BootstrapAccessKey)
 	e2eCtx.BootstrapUserAWSSessionV2 = NewAWSSessionWithKeyV2(e2eCtx.Environment.BootstrapAccessKey)
-	Expect(ensureTestImageUploaded(e2eCtx)).NotTo(HaveOccurred())
+
+	// 添加详细的日志来诊断AWS凭证问题
+	By(fmt.Sprintf("Bootstrap access key created - AccessKeyId: %s, Status: %s",
+		*e2eCtx.Environment.BootstrapAccessKey.AccessKeyId,
+		e2eCtx.Environment.BootstrapAccessKey.Status))
+
+	// 验证访问密钥是否有效
+	By("Verifying bootstrap access key credentials")
+	if e2eCtx.BootstrapUserAWSSession != nil {
+		By("BootstrapUserAWSSession created successfully")
+		// 尝试获取凭证信息
+		if sess, ok := e2eCtx.BootstrapUserAWSSession.(*session.Session); ok {
+			creds, err := sess.Config.Credentials.Get()
+			if err != nil {
+				By(fmt.Sprintf("Failed to get credentials from BootstrapUserAWSSession: %v", err))
+			} else {
+				By(fmt.Sprintf("BootstrapUserAWSSession credentials - AccessKeyId: %s, ProviderName: %s",
+					creds.AccessKeyID, creds.ProviderName))
+			}
+		}
+	} else {
+		By("WARNING: BootstrapUserAWSSession is nil")
+	}
+
+	if e2eCtx.BootstrapUserAWSSessionV2 != nil {
+		By("BootstrapUserAWSSessionV2 created successfully")
+		// 验证V2配置
+		creds, err := e2eCtx.BootstrapUserAWSSessionV2.Credentials.Retrieve(context.TODO())
+		if err != nil {
+			By(fmt.Sprintf("Failed to get credentials from BootstrapUserAWSSessionV2: %v", err))
+		} else {
+			By(fmt.Sprintf("BootstrapUserAWSSessionV2 credentials - AccessKeyId: %s",
+				creds.AccessKeyID))
+		}
+	} else {
+		By("WARNING: BootstrapUserAWSSessionV2 is nil")
+	}
+
+	// 等待一段时间让访问密钥传播
+	By("Waiting for access key to propagate...")
+	time.Sleep(10 * time.Second)
+
+	// 尽可能加更多的log，找到为什么会failed
+	By("Starting ensureTestImageUploaded with detailed logging")
+	err = ensureTestImageUploaded(e2eCtx)
+	if err != nil {
+		By(fmt.Sprintf("ensureTestImageUploaded failed with error: %v", err))
+		// 尝试获取更多错误信息
+		if awsErr, ok := err.(awserr.Error); ok {
+			By(fmt.Sprintf("AWS Error details - Code: %s, Message: %s",
+				awsErr.Code(), awsErr.Message()))
+		}
+		// 重新尝试获取凭证信息
+		if e2eCtx.BootstrapUserAWSSession != nil {
+			if sess, ok := e2eCtx.BootstrapUserAWSSession.(*session.Session); ok {
+				creds, err := sess.Config.Credentials.Get()
+				if err != nil {
+					By(fmt.Sprintf("Credentials retrieval failed after error: %v", err))
+				} else {
+					By(fmt.Sprintf("Current credentials - AccessKeyId: %s, ProviderName: %s",
+						creds.AccessKeyID, creds.ProviderName))
+				}
+			}
+		}
+		// 返回错误而不是[]byte
+		return []byte{}
+	}
+	By("ensureTestImageUploaded completed successfully")
 
 	// Image ID is needed when using a CI Kubernetes version. This is used in conformance test and upgrade to main test.
 	if !e2eCtx.IsManaged {