From d91ce3817a8a008eb4b4ea88422152562ef70448 Mon Sep 17 00:00:00 2001 From: gangwgr Date: Thu, 21 Aug 2025 18:06:50 +0530 Subject: [PATCH 1/2] OCPBUGS-59626: operator: don't react to events from all namespaces An informer for an empty namespace will list resources from all namespaces and cluster level. This may be undesirable, as this makes the controller react to events from different namespaces, while we're only interested in cluster-level events for clusterInformers. This prevents kas-operator from becoming degraded when unrelated namespace secrets are corrupted, which was causing false positive degradations. The fix removes the empty string namespace parameter that was causing the operator to watch ALL namespaces, and instead uses cluster-level only informers as intended by library-go#1985. Fixes: OCPBUGS-59626 Related: library-go#1985 --- pkg/operator/starter.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 1e0551cff..f0f01a0e9 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -129,7 +129,9 @@ func RunOperator(ctx context.Context, controllerContext *controllercmd.Controlle "openshift-etcd", "openshift-apiserver", ) - clusterInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, "") + // OCPBUGS-59626: Use cluster-level informers only, don't watch all namespaces + // Remove empty namespace ("") parameter to prevent watching ALL namespaces + clusterInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient) configInformers := configv1informers.NewSharedInformerFactory(configClient, 10*time.Minute) operatorClient, dynamicInformersForAllNamespaces, err := genericoperatorclient.NewStaticPodOperatorClient( From 0576352c4213c7ad1b7bbc663c06e3e705724101 Mon Sep 17 00:00:00 2001 From: gangwgr Date: Fri, 7 Nov 2025 22:49:16 +0530 Subject: [PATCH 2/2] KMS TESTING --- ...yload_cluster-kube-apiserver-operator.json | 57 ++ test/extended/tests-extension/compute_kms.go | 113 +++ .../tests-extension/compute_kms_aws.go | 524 ++++++++++++ test/extended/tests-extension/go.mod | 24 +- test/extended/tests-extension/go.sum | 44 +- test/extended/tests-extension/kms_tests.go | 560 ++++++++++++ .../testdata/kms_tests_aws.yaml | 119 +++ test/extended/tests-extension/util.go | 802 ++++++++++++++++++ 8 files changed, 2229 insertions(+), 14 deletions(-) create mode 100644 test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json create mode 100644 test/extended/tests-extension/compute_kms.go create mode 100644 test/extended/tests-extension/compute_kms_aws.go create mode 100644 test/extended/tests-extension/kms_tests.go create mode 100644 test/extended/tests-extension/testdata/kms_tests_aws.yaml create mode 100644 test/extended/tests-extension/util.go diff --git a/test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json b/test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json new file mode 100644 index 000000000..1cd1d8bc7 --- /dev/null +++ b/test/extended/tests-extension/.openshift-tests-extension/openshift_payload_cluster-kube-apiserver-operator.json @@ -0,0 +1,57 @@ +[ + { + "name": "[Jira:kube-apiserver][sig-api-machinery][FeatureGate:EventTTL] Event TTL Configuration should configure and validate eventTTLMinutes=5m [Timeout:90m][Serial][Disruptive][Slow][Suite:openshift/cluster-kube-apiserver-operator/conformance/serial]", + "labels": { + "Lifecycle:informing": {} + }, + "tags": { + "timeout": "90m" + }, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "informing", + "environmentSelector": {} + }, + { + "name": "[Jira:kube-apiserver][sig-api-machinery][FeatureGate:EventTTL] Event TTL Configuration should configure and validate eventTTLMinutes=10m [Timeout:90m][Serial][Disruptive][Slow][Suite:openshift/cluster-kube-apiserver-operator/conformance/serial]", + "labels": { + "Lifecycle:informing": {} + }, + "tags": { + "timeout": "90m" + }, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "informing", + "environmentSelector": {} + }, + { + "name": "[Jira:kube-apiserver][sig-api-machinery][FeatureGate:EventTTL] Event TTL Configuration should configure and validate eventTTLMinutes=15m [Timeout:90m][Serial][Disruptive][Slow][Suite:openshift/cluster-kube-apiserver-operator/conformance/serial]", + "labels": { + "Lifecycle:informing": {} + }, + "tags": { + "timeout": "90m" + }, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "informing", + "environmentSelector": {} + }, + { + "name": "[Jira:kube-apiserver][sig-api-machinery] sanity test should always pass [Suite:openshift/cluster-kube-apiserver-operator/conformance/parallel]", + "labels": {}, + "resources": { + "isolation": {} + }, + "source": "openshift:payload:cluster-kube-apiserver-operator", + "lifecycle": "blocking", + "environmentSelector": {} + } +] diff --git a/test/extended/tests-extension/compute_kms.go b/test/extended/tests-extension/compute_kms.go new file mode 100644 index 000000000..11b7b9c22 --- /dev/null +++ b/test/extended/tests-extension/compute_kms.go @@ -0,0 +1,113 @@ +package extended + +import ( + "context" + "fmt" + "os" + "strings" + + g "github.com/onsi/ginkgo/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// YamlKmsTestCase represents a KMS test case from YAML +type YamlKmsTestCase struct { + Name string `yaml:"name"` + Initial string `yaml:"initial"` + Expected string `yaml:"expected,omitempty"` + ExpectedError string `yaml:"expectedError,omitempty"` +} + +// ComputeNode interface to handle compute nodes across different cloud platforms +type ComputeNode interface { + GetName() string + GetInstanceID() (string, error) + CreateKMSKey() string + DeleteKMSKey(keyArn string) + LoadKMSTestCasesFromYAML() ([]YamlKmsTestCase, error) + GetIamRoleNameFromId() string + RenderKmsKeyPolicy() string + UpdateKmsPolicy(keyID string) + GetRegionFromARN(arn string) string + VerifyEncryptionType(ctx context.Context, client dynamic.Interface) (string, bool) + VerifySecretEncryption(ctx context.Context, namespace, secretName string) (bool, string) + VerifyOAuthTokenEncryption(ctx context.Context, tokenType, tokenName string) (bool, string) + ExecuteCommand(command string) (string, error) +} + +// instance is the base struct for all compute node implementations +type instance struct { + nodeName string + kubeClient *kubernetes.Clientset + dynamicClient dynamic.Interface + ctx context.Context +} + +func (i *instance) GetName() string { + return i.nodeName +} + +// ExecuteCommand executes a command on the node via oc debug +func (i *instance) ExecuteCommand(command string) (string, error) { + // Use the executeNodeCommand wrapper from util.go + return executeNodeCommand(i.nodeName, command) +} + +// ComputeNodes handles a collection of ComputeNode interfaces +type ComputeNodes []ComputeNode + +// GetNodes gets master nodes according to platform with the specified label +func GetNodes(ctx context.Context, kubeClient *kubernetes.Clientset, dynamicClient dynamic.Interface, label string) (ComputeNodes, func()) { + platform := checkPlatform(kubeClient) + + switch platform { + case "aws": + return GetAwsNodes(ctx, kubeClient, dynamicClient, label) + case "gcp": + g.Skip("GCP platform KMS support not yet implemented") + return nil, nil + case "azure": + g.Skip("Azure platform KMS support not yet implemented") + return nil, nil + default: + g.Skip(fmt.Sprintf("Platform %s is not supported for KMS tests. Expected AWS, GCP, or Azure.", platform)) + return nil, nil + } +} + +// checkPlatform determines the cloud platform of the cluster +func checkPlatform(kubeClient *kubernetes.Clientset) string { + // Check for AWS-specific labels or annotations + nodes, err := kubeClient.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{Limit: 1}) + if err != nil || len(nodes.Items) == 0 { + return "unknown" + } + + node := nodes.Items[0] + + // Check provider ID format + if providerID := node.Spec.ProviderID; providerID != "" { + if strings.HasPrefix(providerID, "aws://") { + return "aws" + } + if strings.HasPrefix(providerID, "gce://") { + return "gcp" + } + if strings.HasPrefix(providerID, "azure://") { + return "azure" + } + } + + return "unknown" +} + +// getAWSRegion gets the AWS region from environment or config +func getAWSRegion() string { + if region := os.Getenv("AWS_REGION"); region != "" { + return region + } + // Default to us-east-1 if not specified + return "us-east-1" +} diff --git a/test/extended/tests-extension/compute_kms_aws.go b/test/extended/tests-extension/compute_kms_aws.go new file mode 100644 index 000000000..46d2b77cc --- /dev/null +++ b/test/extended/tests-extension/compute_kms_aws.go @@ -0,0 +1,524 @@ +package extended + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "text/template" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/aws/aws-sdk-go-v2/service/iam" + "github.com/aws/aws-sdk-go-v2/service/kms" + kmsTypes "github.com/aws/aws-sdk-go-v2/service/kms/types" + "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi" + rgttypes "github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi/types" + "github.com/aws/aws-sdk-go-v2/service/sts" + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "gopkg.in/yaml.v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// awsInstance implements ComputeNode interface for AWS platform +type awsInstance struct { + instance + awsConfig aws.Config + region string +} + +// GetAwsNodes gets AWS nodes and loads cloud credentials with the specified label +func GetAwsNodes(ctx context.Context, kubeClient *kubernetes.Clientset, dynamicClient dynamic.Interface, label string) ([]ComputeNode, func()) { + // Get AWS credentials from cluster + err := getAwsCredentialFromCluster() + o.Expect(err).NotTo(o.HaveOccurred()) + + region := getAWSRegion() + + // Load AWS config + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(region)) + o.Expect(err).NotTo(o.HaveOccurred()) + + // Get node names + nodeList, err := kubeClient.CoreV1().Nodes().List(ctx, metav1.ListOptions{ + LabelSelector: fmt.Sprintf("node-role.kubernetes.io/%s", label), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(nodeList.Items)).To(o.BeNumerically(">", 0), "No nodes found with label %s", label) + + var results []ComputeNode + for _, node := range nodeList.Items { + results = append(results, newAwsInstance(ctx, kubeClient, dynamicClient, node.Name, cfg, region)) + } + + return results, nil +} + +func newAwsInstance(ctx context.Context, kubeClient *kubernetes.Clientset, dynamicClient dynamic.Interface, nodeName string, awsConfig aws.Config, region string) *awsInstance { + return &awsInstance{ + instance: instance{ + nodeName: nodeName, + kubeClient: kubeClient, + dynamicClient: dynamicClient, + ctx: ctx, + }, + awsConfig: awsConfig, + region: region, + } +} + +// GetInstanceID retrieves the EC2 instance ID from the node's provider ID +func (a *awsInstance) GetInstanceID() (string, error) { + node, err := a.kubeClient.CoreV1().Nodes().Get(a.ctx, a.nodeName, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get node %s: %w", a.nodeName, err) + } + + // Provider ID format: aws://// + // Example: aws:///us-east-1a/i-1234567890abcdef0 + providerID := node.Spec.ProviderID + if providerID == "" { + return "", fmt.Errorf("node %s has no provider ID", a.nodeName) + } + + parts := strings.Split(providerID, "/") + if len(parts) < 2 { + return "", fmt.Errorf("invalid provider ID format: %s", providerID) + } + + instanceID := parts[len(parts)-1] + return instanceID, nil +} + +// CreateKMSKey creates or retrieves an AWS KMS key for testing +func (a *awsInstance) CreateKMSKey() string { + Logf("[AWS-KMS] Initializing AWS KMS client in region: %s", a.region) + kmsClient := kms.NewFromConfig(a.awsConfig) + rgtClient := resourcegroupstaggingapi.NewFromConfig(a.awsConfig) + + // Check for existing test keys with the specific tag + Logf("[AWS-KMS] Searching for existing KMS keys with tag Purpose=ocp-kms-qe-ci-test") + getResourcesInput := &resourcegroupstaggingapi.GetResourcesInput{ + ResourceTypeFilters: []string{"kms"}, + TagFilters: []rgttypes.TagFilter{ + { + Key: aws.String("Purpose"), + Values: []string{"ocp-kms-qe-ci-test"}, + }, + }, + } + + existingKeys, err := rgtClient.GetResources(a.ctx, getResourcesInput) + o.Expect(err).NotTo(o.HaveOccurred()) + + var myKmsKeyArn string + + if len(existingKeys.ResourceTagMappingList) > 0 { + myKmsKeyArn = *existingKeys.ResourceTagMappingList[0].ResourceARN + Logf("[AWS-KMS] Found existing KMS key: %s", myKmsKeyArn) + g.By(fmt.Sprintf("Found existing KMS key: %s", myKmsKeyArn)) + + // Check if key is scheduled for deletion and cancel if needed + Logf("[AWS-KMS] Checking key status for: %s", myKmsKeyArn) + describeInput := &kms.DescribeKeyInput{ + KeyId: aws.String(myKmsKeyArn), + } + keyMetadata, err := kmsClient.DescribeKey(a.ctx, describeInput) + o.Expect(err).NotTo(o.HaveOccurred()) + + Logf("[AWS-KMS] Key state: %s", keyMetadata.KeyMetadata.KeyState) + if keyMetadata.KeyMetadata.DeletionDate != nil { + Logf("[AWS-KMS] Key is scheduled for deletion on: %v", keyMetadata.KeyMetadata.DeletionDate) + g.By("Canceling scheduled deletion and enabling key") + + Logf("[AWS-KMS] Canceling key deletion...") + _, err = kmsClient.CancelKeyDeletion(a.ctx, &kms.CancelKeyDeletionInput{ + KeyId: aws.String(myKmsKeyArn), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[AWS-KMS] ✓ Deletion canceled") + + Logf("[AWS-KMS] Enabling key...") + _, err = kmsClient.EnableKey(a.ctx, &kms.EnableKeyInput{ + KeyId: aws.String(myKmsKeyArn), + }) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[AWS-KMS] ✓ Key enabled") + } else { + Logf("[AWS-KMS] Key is active and ready to use") + } + } else { + Logf("[AWS-KMS] No existing key found, creating new KMS key") + g.By("Creating new KMS key") + createKeyInput := &kms.CreateKeyInput{ + Description: aws.String("OCP KMS QE CI Test Key"), + KeySpec: kmsTypes.KeySpecSymmetricDefault, + KeyUsage: kmsTypes.KeyUsageTypeEncryptDecrypt, + Tags: []kmsTypes.Tag{ + { + TagKey: aws.String("Purpose"), + TagValue: aws.String("ocp-kms-qe-ci-test"), + }, + }, + } + + Logf("[AWS-KMS] Creating KMS key with spec: SYMMETRIC_DEFAULT, usage: ENCRYPT_DECRYPT") + createResult, err := kmsClient.CreateKey(a.ctx, createKeyInput) + if err != nil { + if strings.Contains(err.Error(), "AccessDeniedException") { + Logf("[AWS-KMS] ✗ Access denied - insufficient permissions") + g.Skip("AWS credentials don't have permission to create KMS keys") + } + Logf("[AWS-KMS] ✗ Failed to create key: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + + myKmsKeyArn = *createResult.KeyMetadata.Arn + Logf("[AWS-KMS] ✓ Created new KMS key: %s", myKmsKeyArn) + Logf("[AWS-KMS] Key ID: %s", *createResult.KeyMetadata.KeyId) + g.By(fmt.Sprintf("Created KMS key: %s", myKmsKeyArn)) + } + + return myKmsKeyArn +} + +// DeleteKMSKey schedules a KMS key for deletion +func (a *awsInstance) DeleteKMSKey(keyArn string) { + Logf("[AWS-KMS] Scheduling KMS key for deletion: %s", keyArn) + kmsClient := kms.NewFromConfig(a.awsConfig) + + // Schedule key deletion with minimum waiting period (7 days) + input := &kms.ScheduleKeyDeletionInput{ + KeyId: aws.String(keyArn), + PendingWindowInDays: aws.Int32(7), // Minimum allowed by AWS + } + + result, err := kmsClient.ScheduleKeyDeletion(a.ctx, input) + if err != nil { + // Don't fail the test if key deletion fails + Logf("[AWS-KMS] Warning: Failed to schedule key deletion: %v", err) + return + } + + if result.DeletionDate != nil { + Logf("[AWS-KMS] ✓ Key scheduled for deletion on: %v", *result.DeletionDate) + } else { + Logf("[AWS-KMS] ✓ Key deletion scheduled") + } + g.By(fmt.Sprintf("Scheduled KMS key deletion: %s", keyArn)) +} + +// LoadKMSTestCasesFromYAML loads test cases from the YAML file +func (a *awsInstance) LoadKMSTestCasesFromYAML() ([]YamlKmsTestCase, error) { + testDataFile := filepath.Join("testdata", "kms_tests_aws.yaml") + + data, err := os.ReadFile(testDataFile) + if err != nil { + return nil, fmt.Errorf("failed to read test data file: %w", err) + } + + var testCases []YamlKmsTestCase + err = yaml.Unmarshal(data, &testCases) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal test cases: %w", err) + } + + return testCases, nil +} + +// GetIamRoleNameFromId retrieves the IAM role name attached to the EC2 instance +func (a *awsInstance) GetIamRoleNameFromId() string { + Logf("[AWS-IAM] Retrieving IAM role for instance: %s", a.nodeName) + instanceID, err := a.GetInstanceID() + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[AWS-IAM] Instance ID: %s", instanceID) + + ec2Client := ec2.NewFromConfig(a.awsConfig) + iamClient := iam.NewFromConfig(a.awsConfig) + + // Describe the instance to get IAM instance profile + Logf("[AWS-IAM] Describing EC2 instance...") + describeInput := &ec2.DescribeInstancesInput{ + InstanceIds: []string{instanceID}, + } + + result, err := ec2Client.DescribeInstances(a.ctx, describeInput) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(result.Reservations)).To(o.BeNumerically(">", 0)) + o.Expect(len(result.Reservations[0].Instances)).To(o.BeNumerically(">", 0)) + + instance := result.Reservations[0].Instances[0] + o.Expect(instance.IamInstanceProfile).NotTo(o.BeNil()) + o.Expect(instance.IamInstanceProfile.Arn).NotTo(o.BeNil()) + + Logf("[AWS-IAM] Instance profile ARN: %s", *instance.IamInstanceProfile.Arn) + + // Extract profile name from ARN + arnParts := strings.Split(*instance.IamInstanceProfile.Arn, "/") + o.Expect(len(arnParts)).To(o.BeNumerically(">=", 2)) + profileName := arnParts[1] + Logf("[AWS-IAM] Instance profile name: %s", profileName) + + // Get instance profile to retrieve role + Logf("[AWS-IAM] Fetching instance profile details...") + profileInput := &iam.GetInstanceProfileInput{ + InstanceProfileName: aws.String(profileName), + } + + profileOutput, err := iamClient.GetInstanceProfile(a.ctx, profileInput) + o.Expect(err).NotTo(o.HaveOccurred()) + o.Expect(len(profileOutput.InstanceProfile.Roles)).To(o.BeNumerically(">", 0)) + + roleName := *profileOutput.InstanceProfile.Roles[0].RoleName + Logf("[AWS-IAM] ✓ IAM role name: %s", roleName) + g.By(fmt.Sprintf("IAM Role Name for instance %s: %s", instanceID, roleName)) + + return roleName +} + +const keyAWSPolicyTemplate = ` +{ + "Id": "key-policy-01", + "Statement": [ + { + "Sid": "Enable IAM User Permissions", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{.AccountID}}:root" + }, + "Action": "kms:*", + "Resource": "*" + }, + { + "Sid": "Allow use of the key", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{.AccountID}}:role/{{.MasterRoleName}}" + }, + "Action": [ + "kms:Encrypt", + "kms:Decrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*", + "kms:DescribeKey" + ], + "Resource": "*" + }, + { + "Sid": "Allow attachment of persistent resources", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::{{.AccountID}}:role/{{.MasterRoleName}}" + }, + "Action": [ + "kms:CreateGrant", + "kms:ListGrants", + "kms:RevokeGrant" + ], + "Resource": "*", + "Condition": { + "Bool": { + "kms:GrantIsForAWSResource": "true" + } + } + } + ] +} +` + +type KeyAWSPolicyData struct { + AccountID string + MasterRoleName string +} + +// RenderKmsKeyPolicy renders the KMS key policy template +func (a *awsInstance) RenderKmsKeyPolicy() string { + Logf("[AWS-Policy] Rendering KMS key policy template") + stsClient := sts.NewFromConfig(a.awsConfig) + + // Get AWS account ID + Logf("[AWS-Policy] Retrieving AWS account ID via STS...") + callerIdentity, err := stsClient.GetCallerIdentity(a.ctx, &sts.GetCallerIdentityInput{}) + o.Expect(err).NotTo(o.HaveOccurred()) + + accountID := *callerIdentity.Account + Logf("[AWS-Policy] AWS Account ID: %s", accountID) + + masterRoleName := a.GetIamRoleNameFromId() + + Logf("[AWS-Policy] Parsing policy template...") + tmpl, err := template.New("keyPolicy").Parse(keyAWSPolicyTemplate) + o.Expect(err).NotTo(o.HaveOccurred()) + + var rendered bytes.Buffer + err = tmpl.Execute(&rendered, KeyAWSPolicyData{ + AccountID: accountID, + MasterRoleName: masterRoleName, + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + Logf("[AWS-Policy] ✓ Policy rendered for account %s and role %s", accountID, masterRoleName) + g.By(fmt.Sprintf("Rendered KMS Policy for account %s and role %s", accountID, masterRoleName)) + return rendered.String() +} + +// UpdateKmsPolicy updates the KMS key policy +func (a *awsInstance) UpdateKmsPolicy(keyID string) { + Logf("[AWS-KMS] Updating KMS key policy for: %s", keyID) + kmsClient := kms.NewFromConfig(a.awsConfig) + kmsPolicy := a.RenderKmsKeyPolicy() + + Logf("[AWS-KMS] Applying policy to key...") + putPolicyInput := &kms.PutKeyPolicyInput{ + KeyId: aws.String(keyID), + PolicyName: aws.String("default"), + Policy: aws.String(kmsPolicy), + } + + _, err := kmsClient.PutKeyPolicy(a.ctx, putPolicyInput) + if err != nil { + Logf("[AWS-KMS] ✗ Failed to update policy: %v", err) + } + o.Expect(err).NotTo(o.HaveOccurred()) + + Logf("[AWS-KMS] ✓ Policy updated successfully") + g.By(fmt.Sprintf("Updated KMS key policy for key: %s", keyID)) +} + +// GetRegionFromARN extracts the region from an AWS KMS ARN +// ARN format: arn:aws:kms:region:account:key/key-id +func (a *awsInstance) GetRegionFromARN(arn string) string { + parts := strings.Split(arn, ":") + if len(parts) < 4 { + Logf("[AWS] Warning: Invalid ARN format: %s", arn) + return a.region // fallback to instance region + } + return parts[3] +} + +// VerifyEncryptionType calls the generic utility function +func (a *awsInstance) VerifyEncryptionType(ctx context.Context, client dynamic.Interface) (string, bool) { + return verifyEncryptionType(ctx, client) +} + +// VerifySecretEncryption verifies that a secret is encrypted in etcd with the expected format +// Returns: (isEncrypted, encryptionFormat) +func (a *awsInstance) VerifySecretEncryption(ctx context.Context, namespace, secretName string) (bool, string) { + Logf("[Verify-Secret] Checking encryption for secret %s/%s", namespace, secretName) + + // Execute etcdctl command to get the secret from etcd + etcdKey := fmt.Sprintf("/kubernetes.io/secrets/%s/%s", namespace, secretName) + + // Use single quotes around the etcd key to prevent shell expansion + command := fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --prefix --keys-only", + etcdKey, + ) + + output, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-Secret] Failed to query etcd: %v", err) + return false, "" + } + + // Check if key exists + if !strings.Contains(output, etcdKey) { + Logf("[Verify-Secret] Secret not found in etcd") + return false, "" + } + + // Get the actual value to check encryption format + command = fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --print-value-only | head -c 20", + etcdKey, + ) + + value, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-Secret] Failed to get secret value from etcd: %v", err) + return false, "" + } + + // Check for KMSv2 encryption prefix + if strings.HasPrefix(value, "k8s:enc:kms:v2:") { + Logf("[Verify-Secret] ✓ Secret is encrypted with KMSv2") + return true, "k8s:enc:kms:v2:" + } else if strings.HasPrefix(value, "k8s:enc:kms:v1:") { + Logf("[Verify-Secret] Secret is encrypted with KMSv1") + return true, "k8s:enc:kms:v1:" + } else if strings.HasPrefix(value, "k8s:enc:") { + Logf("[Verify-Secret] Secret is encrypted with format: %s", value[:15]) + return true, value[:15] + } + + Logf("[Verify-Secret] Secret is not encrypted (no k8s:enc: prefix)") + return false, "" +} + +// VerifyOAuthTokenEncryption verifies that an OAuth token is encrypted in etcd +// Returns: (isEncrypted, encryptionFormat) +func (a *awsInstance) VerifyOAuthTokenEncryption(ctx context.Context, tokenType, tokenName string) (bool, string) { + Logf("[Verify-OAuth] Checking encryption for %s: %s", tokenType, tokenName) + + // etcd key format for OAuth tokens + var etcdKey string + if tokenType == "oauthaccesstokens" { + etcdKey = fmt.Sprintf("/kubernetes.io/oauth.openshift.io/oauthaccesstokens/%s", tokenName) + } else if tokenType == "oauthauthorizetokens" { + etcdKey = fmt.Sprintf("/kubernetes.io/oauth.openshift.io/oauthauthorizetokens/%s", tokenName) + } else { + Logf("[Verify-OAuth] Unknown token type: %s", tokenType) + return false, "" + } + + // Use single quotes around the etcd key to prevent shell expansion of special chars like ~ + command := fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --prefix --keys-only", + etcdKey, + ) + + output, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-OAuth] Failed to query etcd: %v", err) + return false, "" + } + + // Check if key exists + if !strings.Contains(output, etcdKey) { + Logf("[Verify-OAuth] Token not found in etcd") + return false, "" + } + + // Get the actual value to check encryption format + command = fmt.Sprintf( + "ETCD_POD=$(sudo crictl ps --name=etcd-member -q) && sudo crictl exec $ETCD_POD etcdctl get '%s' --print-value-only | head -c 20", + etcdKey, + ) + + value, err := a.ExecuteCommand(command) + if err != nil { + Logf("[Verify-OAuth] Failed to get token value from etcd: %v", err) + return false, "" + } + + // Check for KMSv2 encryption prefix + if strings.HasPrefix(value, "k8s:enc:kms:v2:") { + Logf("[Verify-OAuth] ✓ OAuth token is encrypted with KMSv2") + return true, "k8s:enc:kms:v2:" + } else if strings.HasPrefix(value, "k8s:enc:kms:v1:") { + Logf("[Verify-OAuth] OAuth token is encrypted with KMSv1") + return true, "k8s:enc:kms:v1:" + } else if strings.HasPrefix(value, "k8s:enc:") { + Logf("[Verify-OAuth] OAuth token is encrypted with format: %s", value[:15]) + return true, value[:15] + } + + Logf("[Verify-OAuth] OAuth token is not encrypted (no k8s:enc: prefix)") + return false, "" +} diff --git a/test/extended/tests-extension/go.mod b/test/extended/tests-extension/go.mod index 966ae55dd..b576e3812 100644 --- a/test/extended/tests-extension/go.mod +++ b/test/extended/tests-extension/go.mod @@ -3,12 +3,18 @@ module github.com/openshift/cluster-kube-apiserver-operator/test/extended/tests- go 1.24.0 require ( + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.28.0 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0 + github.com/aws/aws-sdk-go-v2/service/iam v1.38.2 + github.com/aws/aws-sdk-go-v2/service/kms v1.37.2 + github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.26.2 + github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 github.com/onsi/ginkgo/v2 v2.22.0 github.com/onsi/gomega v1.36.1 github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 - github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 - github.com/openshift/cluster-kube-apiserver-operator v0.0.0-00010101000000-000000000000 github.com/spf13/cobra v1.9.1 + gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.34.1 k8s.io/apimachinery v0.34.1 k8s.io/client-go v0.34.1 @@ -17,6 +23,16 @@ require ( require ( cel.dev/expr v0.24.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.17.41 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 // indirect + github.com/aws/smithy-go v1.22.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect @@ -38,13 +54,9 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 // indirect - github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/stoewer/go-strcase v1.3.0 // indirect - github.com/stretchr/testify v1.10.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect diff --git a/test/extended/tests-extension/go.sum b/test/extended/tests-extension/go.sum index e5f650f53..30f8478e5 100644 --- a/test/extended/tests-extension/go.sum +++ b/test/extended/tests-extension/go.sum @@ -2,6 +2,40 @@ cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/config v1.28.0 h1:FosVYWcqEtWNxHn8gB/Vs6jOlNwSoyOCA/g/sxyySOQ= +github.com/aws/aws-sdk-go-v2/config v1.28.0/go.mod h1:pYhbtvg1siOOg8h5an77rXle9tVG8T+BWLWAo7cOukc= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41 h1:7gXo+Axmp+R4Z+AK8YFQO0ZV3L0gizGINCOWxSLY9W8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.41/go.mod h1:u4Eb8d3394YLubphT4jLEwN1rLNq2wFOlT6OuxFwPzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17 h1:TMH3f/SCAWdNtXXVPPu5D6wrr4G5hI1rAxbcocKfC7Q= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.17/go.mod h1:1ZRXLdTpzdJb9fwTMXiLipENRxkGMTn1sfKexGllQCw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0 h1:cA4hWo269CN5RY7Arqt8BfzXF0KIN8DSNo/KcqHKkWk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.187.0/go.mod h1:ossaD9Z1ugYb6sq9QIqQLEOorCGcqUoxlhud9M9yE70= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.2 h1:8iFKuRj/FJipy/aDZ2lbq0DYuEHdrxp0qVsdi+ZEwnE= +github.com/aws/aws-sdk-go-v2/service/iam v1.38.2/go.mod h1:UBe4z0VZnbXGp6xaCW1ulE9pndjfpsnrU206rWZcR0Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3 h1:qcxX0JYlgWH3hpPUnd6U0ikcl6LLA9sLkXE2w1fpMvY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.3/go.mod h1:cLSNEmI45soc+Ef8K/L+8sEA3A3pYFEYf5B5UI+6bH4= +github.com/aws/aws-sdk-go-v2/service/kms v1.37.2 h1:tfBABi5R6aSZlhgTWHxL+opYUDOnIGoNcJLwVYv0jLM= +github.com/aws/aws-sdk-go-v2/service/kms v1.37.2/go.mod h1:dZYFcQwuoh+cLOlFnZItijZptmyDhRIkOKWFO1CfzV8= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.26.2 h1:SW+bplzotcNwVKph3FWsE4Zfk728edeFUCM5VmjbFy0= +github.com/aws/aws-sdk-go-v2/service/resourcegroupstaggingapi v1.26.2/go.mod h1:cgPfPTC/V3JqwCKed7Q6d0FrgarV7ltz4Bz6S4Q+Dqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2 h1:bSYXVyUzoTHoKalBmwaZxs97HU9DWWI3ehHSAMa7xOk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.2/go.mod h1:skMqY7JElusiOUjMJMOv1jJsP7YUg7DrhgqZZWuzu1U= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2 h1:AhmO1fHINP9vFYUE0LHzCWg/LfUWUF+zFPEcY9QXb7o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.2/go.mod h1:o8aQygT2+MVP0NaV6kbdE1YnnIM8RRVQzoeUH45GOdI= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2 h1:CiS7i0+FUe+/YY1GvIBLLrR/XNGZ4CtM1Ll0XavNuVo= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.2/go.mod h1:HtaiBI8CjYoNVde8arShXb94UbQQi9L4EMr6D+xGBwo= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,12 +100,6 @@ github.com/onsi/gomega v1.36.1 h1:bJDPBO7ibjxcbHMgSCoo4Yj18UWbKDlLwX1x9sybDcw= github.com/onsi/gomega v1.36.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292 h1:3athg6KQ+TaNfW4BWZDlGFt1ImSZEJWgzXtPC1VPITI= github.com/openshift-eng/openshift-tests-extension v0.0.0-20250804142706-7b3ab438a292/go.mod h1:6gkP5f2HL0meusT0Aim8icAspcD1cG055xxBZ9yC68M= -github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7 h1:Ot2fbEEPmF3WlPQkyEW/bUCV38GMugH/UmZvxpWceNc= -github.com/openshift/api v0.0.0-20251015095338-264e80a2b6e7/go.mod h1:d5uzF0YN2nQQFA0jIEWzzOZ+edmo6wzlGLvx5Fhz4uY= -github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235 h1:9JBeIXmnHlpXTQPi7LPmu1jdxznBhAE7bb1K+3D8gxY= -github.com/openshift/client-go v0.0.0-20251015124057-db0dee36e235/go.mod h1:L49W6pfrZkfOE5iC1PqEkuLkXG4W0BX4w8b+L2Bv7fM= -github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5 h1:bANtDc8SgetSK4nQehf59x3+H9FqVJCprgjs49/OTg0= -github.com/openshift/library-go v0.0.0-20251015151611-6fc7a74b67c5/go.mod h1:OlFFws1AO51uzfc48MsStGE4SFMWlMZD0+f5a/zCtKI= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20250416174521-4eb003743b54 h1:ehXndVZfIk/fo18YJCMJ+6b8HL8tzqjP7yWgchMnfCc= github.com/openshift/onsi-ginkgo/v2 v2.6.1-0.20250416174521-4eb003743b54/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -103,8 +131,6 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -163,6 +189,8 @@ gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSP gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/test/extended/tests-extension/kms_tests.go b/test/extended/tests-extension/kms_tests.go new file mode 100644 index 000000000..b4c0fc688 --- /dev/null +++ b/test/extended/tests-extension/kms_tests.go @@ -0,0 +1,560 @@ +package extended + +import ( + "context" + "fmt" + "os" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" +) + +// findExistingKMSKey attempts to find an existing KMS key configured in the APIServer +func findExistingKMSKey(ctx context.Context, node ComputeNode) (string, error) { + // This is a placeholder that would check if KMS is already configured + // In a real implementation, this would check the apiserver CR for existing KMS config + // For now, we'll return empty to indicate no existing key found + return "", fmt.Errorf("no existing KMS key found") +} + +var _ = g.Describe("[Jira:kube-apiserver][sig-api-machinery] API Server KMS", func() { + var ( + kubeClient *kubernetes.Clientset + dynamicClient dynamic.Interface + ctx context.Context + tmpdir string + + // Suite-level shared resources + kmsKeyArn string + kmsRegion string + masterNode ComputeNode + featureGateWasEnabled bool + ) + + // BeforeSuite runs once before all tests in this suite + g.BeforeEach(func() { + if kubeClient != nil { + return // Already initialized + } + + ctx = context.Background() + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE INITIALIZATION ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + // Create temporary directory for test files + var err error + tmpdir, err = os.MkdirTemp("", "kms-test-*") + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Suite-Setup] Created temporary directory: %s", tmpdir) + + // Get kubeconfig and create clients + kubeconfig := GetKubeConfig() + kubeClient, dynamicClient = CreateKubernetesClients(kubeconfig) + + // Check cluster health + g.By("Checking cluster health before KMS test suite") + Logf("[Suite-Setup] Performing cluster health check...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Suite-Setup] Cluster health check failed: %v", err) + g.Skip(fmt.Sprintf("Cluster health check failed: %s", err)) + } + Logf("[Suite-Setup] ✓ Cluster is healthy") + + // Step 1: Check and enable KMS feature gate + Logf("\n--- Step 1: Feature Gate Configuration ---") + g.By("Checking if KMSEncryptionProvider feature gate is enabled") + + isEnabled, err := isFeatureGateEnabled(ctx, dynamicClient, "KMSEncryptionProvider") + o.Expect(err).NotTo(o.HaveOccurred()) + + if isEnabled { + Logf("[Suite-Setup] ✓ KMSEncryptionProvider is already enabled") + featureGateWasEnabled = true + } else { + Logf("[Suite-Setup] KMSEncryptionProvider is not enabled, enabling now...") + featureGateWasEnabled = false + + err = patchFeatureGate(ctx, dynamicClient, `{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":["KMSEncryptionProvider"],"disabled":[]}}}`) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Suite-Setup] ✓ Feature gate patch applied") + + // Wait for kube-apiserver to rollout + g.By("Waiting for kube-apiserver operator to rollout after enabling KMS") + expectedStatus := map[string]string{"Progressing": "True"} + kubeApiserverCoStatus := map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + + Logf("[Suite-Setup] Waiting for operator to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Suite-Setup] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Suite-Setup] Waiting for operator to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Suite-Setup] ✓ kube-apiserver operator is stable after enabling feature gate") + + // Verify all cluster operators are still stable after feature gate change + g.By("Verifying all cluster operators are stable after feature gate change") + Logf("[Suite-Setup] Checking cluster stability after feature gate change...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Suite-Setup] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Suite-Setup] ✓ All cluster operators are stable after feature gate change") + } + + // Step 2: Get master node + Logf("\n--- Step 2: Master Node Discovery ---") + g.By("Getting master nodes") + nodes, cleanup := GetNodes(ctx, kubeClient, dynamicClient, "master") + if cleanup != nil { + g.DeferCleanup(cleanup) + } + o.Expect(len(nodes)).To(o.BeNumerically(">", 0), "No master nodes found") + masterNode = nodes[0] + Logf("[Suite-Setup] ✓ Using master node: %s", masterNode.GetName()) + + // Step 3: Check and create KMS key + Logf("\n--- Step 3: KMS Key Configuration ---") + g.By("Checking if KMS key already exists") + + existingKeyArn, err := findExistingKMSKey(ctx, masterNode) + if err == nil && existingKeyArn != "" { + Logf("[Suite-Setup] ✓ Found existing KMS key: %s", existingKeyArn) + kmsKeyArn = existingKeyArn + } else { + Logf("[Suite-Setup] No existing KMS key found, creating new one...") + kmsKeyArn = masterNode.CreateKMSKey() + o.Expect(kmsKeyArn).NotTo(o.BeEmpty()) + Logf("[Suite-Setup] ✓ Created KMS key: %s", kmsKeyArn) + + Logf("[Suite-Setup] Updating KMS key policy...") + masterNode.UpdateKmsPolicy(kmsKeyArn) + Logf("[Suite-Setup] ✓ KMS key policy updated") + } + + // Extract region from ARN + kmsRegion = masterNode.GetRegionFromARN(kmsKeyArn) + Logf("[Suite-Setup] ✓ KMS region: %s", kmsRegion) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE INITIALIZATION COMPLETE ║") + Logf("║ Feature Gate: %s ║", getStatusString(isEnabled)) + Logf("║ KMS Key ARN: %s", truncateString(kmsKeyArn, 45)) + Logf("║ KMS Region: %-47s║", kmsRegion) + Logf("╚════════════════════════════════════════════════════════════╝\n") + + // Register cleanup to run after all tests complete + g.DeferCleanup(func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE CLEANUP ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + if tmpdir != "" { + os.RemoveAll(tmpdir) + Logf("[Suite-Cleanup] Cleaned up temporary directory: %s", tmpdir) + } + + // Delete KMS key if it was created + if kmsKeyArn != "" { + Logf("[Suite-Cleanup] Deleting KMS key: %s", kmsKeyArn) + masterNode.DeleteKMSKey(kmsKeyArn) + } + + // Only disable feature gate if we enabled it + if !featureGateWasEnabled && dynamicClient != nil { + Logf("[Suite-Cleanup] Disabling KMSEncryptionProvider feature gate...") + err := patchFeatureGate(ctx, dynamicClient, `{"spec":{"featureSet":"CustomNoUpgrade","customNoUpgrade":{"enabled":[],"disabled":["KMSEncryptionProvider"]}}}`) + if err != nil { + Logf("[Suite-Cleanup] Warning: Failed to disable feature gate: %v", err) + } else { + Logf("[Suite-Cleanup] ✓ Feature gate disabled") + + // Wait for rollout + expectedStatus := map[string]string{"Progressing": "True"} + kubeApiserverCoStatus := map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + + Logf("[Suite-Cleanup] Waiting for operator to stabilize (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Suite-Cleanup] Warning: Operator did not start progressing: %v", err) + } + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + if err != nil { + Logf("[Suite-Cleanup] Warning: Operator did not stabilize: %v", err) + } else { + Logf("[Suite-Cleanup] ✓ Operator is stable") + + // Verify all cluster operators are stable after disabling feature gate + Logf("[Suite-Cleanup] Checking cluster stability after disabling feature gate...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Suite-Cleanup] Warning: Cluster stability check failed: %v", err) + } else { + Logf("[Suite-Cleanup] ✓ All cluster operators are stable after cleanup") + } + } + } + } + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS TEST SUITE CLEANUP COMPLETE ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) + }) + + g.It("should validate KMS encryption configuration [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:30m]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS ENCRYPTION CONFIGURATION VALIDATION TEST ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + Logf("\n--- Phase 1: Validate KMS Configuration Errors ---") + g.By("Loading KMS test cases from YAML") + Logf("[Phase 1] Loading test cases from YAML file") + + testCases, err := masterNode.LoadKMSTestCasesFromYAML() + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 1] Loaded %d test case(s)", len(testCases)) + + g.By("Running KMS validation test cases") + for i, tc := range testCases { + Logf("\n[Phase 1] Test Case %d/%d: %s", i+1, len(testCases), tc.Name) + g.By(fmt.Sprintf("Testing: %s", tc.Name)) + Logf("[Phase 1] Expected error: %s", tc.ExpectedError) + + // Try to apply the config - should fail with expected error + err = applyAPIServerConfig(ctx, dynamicClient, []byte(tc.Initial)) + if err == nil { + Logf("[Phase 1] ✗ FAILED: Expected validation error but got success") + } else { + Logf("[Phase 1] Actual error: %s", err.Error()) + } + + o.Expect(err).To(o.HaveOccurred(), "Expected validation error for test case: %s", tc.Name) + o.Expect(err.Error()).To(o.ContainSubstring(tc.ExpectedError), + "Error message should contain expected validation error") + + Logf("[Phase 1] ✓ Validation passed") + g.By(fmt.Sprintf("✓ Validation passed for: %s", tc.Name)) + } + Logf("[Phase 1] ✓ All %d validation test cases passed", len(testCases)) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ KMS ENCRYPTION VALIDATION COMPLETED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + g.By("KMS encryption configuration validation completed successfully") + }) + + g.It("should encrypt secrets using KMS [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:60m][Serial][Disruptive]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS SECRET ENCRYPTION VERIFICATION TEST ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + var ( + expectedStatus = map[string]string{"Progressing": "True"} + kubeApiserverCoStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + ) + + Logf("\n--- Phase 1: Apply KMS Encryption Configuration ---") + g.By("Configuring KMS encryption for API server") + Logf("[Phase 1] Using shared KMS key: %s", kmsKeyArn) + Logf("[Phase 1] Using region: %s", kmsRegion) + + // Check and apply KMS config if needed + needsRollout, err := checkAndApplyKMSConfig(ctx, dynamicClient, kmsKeyArn, kmsRegion) + o.Expect(err).NotTo(o.HaveOccurred()) + + if needsRollout { + g.By("Waiting for kube-apiserver operator to rollout") + Logf("[Phase 1] Waiting for kube-apiserver to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Phase 1] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Phase 1] Waiting for kube-apiserver to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not become stable after KMS configuration") + Logf("[Phase 1] ✓ kube-apiserver operator is stable") + + // Check KMS plugin health + Logf("[Phase 1] Checking KMS plugin health...") + isHealthy, healthMsg := checkKMSPluginHealth(ctx, dynamicClient) + if !isHealthy { + Logf("[Phase 1] Warning: KMS plugin health check: %s", healthMsg) + Logf("[Phase 1] Note: KMS socket errors during initial rollout are expected and should resolve") + } else { + Logf("[Phase 1] ✓ KMS plugin health check: %s", healthMsg) + } + } else { + Logf("[Phase 1] ✓ KMS already configured, no rollout needed") + } + + // Verify all cluster operators are still stable after KMS configuration + if needsRollout { + g.By("Verifying all cluster operators are stable after KMS configuration") + Logf("[Phase 1] Checking cluster stability after KMS configuration...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Phase 1] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Phase 1] ✓ All cluster operators are stable after KMS configuration") + } + + Logf("\n--- Phase 2: Verify Encryption Type ---") + g.By("Verifying encryption type is KMS") + encType, encCompleted := masterNode.VerifyEncryptionType(ctx, dynamicClient) + o.Expect(encType).To(o.Equal("KMS"), "Encryption type should be KMS") + Logf("[Phase 2] ✓ Encryption type: %s", encType) + Logf("[Phase 2] ✓ Encryption completed: %v", encCompleted) + + Logf("\n--- Phase 3: Create Test Secret ---") + g.By("Creating test namespace and secret") + testNamespace := "kms-secret-test" + Logf("[Phase 3] Creating namespace: %s", testNamespace) + + err = createNamespace(ctx, kubeClient, testNamespace) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 3] ✓ Namespace created") + + defer func() { + Logf("[Cleanup] Deleting test namespace: %s", testNamespace) + deleteNamespace(ctx, kubeClient, testNamespace) + }() + + secretName := "mysecret1" + secretData := map[string]string{"password": "SuperSecure123"} + Logf("[Phase 3] Creating secret: %s", secretName) + err = createSecret(ctx, kubeClient, testNamespace, secretName, secretData) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 3] ✓ Secret created successfully") + + Logf("\n--- Phase 4: Verify Secret Encryption in etcd ---") + g.By("Verifying secret is encrypted with KMSv2 in etcd") + Logf("[Phase 4] Checking etcd encryption format for secret") + + isEncrypted, encryptionFormat := masterNode.VerifySecretEncryption(ctx, testNamespace, secretName) + o.Expect(isEncrypted).To(o.BeTrue(), "Secret should be encrypted in etcd") + o.Expect(encryptionFormat).To(o.Equal("k8s:enc:kms:v2:"), "Secret should use KMSv2 encryption format") + Logf("[Phase 4] ✓ Secret is encrypted with format: %s", encryptionFormat) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ KMS SECRET ENCRYPTION VERIFIED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) + + g.It("should encrypt OAuthAccessTokens using KMS [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:120m][Serial][Disruptive]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS OAUTH ACCESS TOKEN ENCRYPTION VERIFICATION ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + var ( + expectedStatus = map[string]string{"Progressing": "True"} + kubeApiserverCoStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + userUID string + ) + + Logf("\n--- Phase 1: Ensure KMS Encryption Configuration ---") + g.By("Configuring KMS encryption for API server") + Logf("[Phase 1] Using shared KMS key: %s", kmsKeyArn) + Logf("[Phase 1] Using region: %s", kmsRegion) + + // Check and apply KMS config if needed + needsRollout, err := checkAndApplyKMSConfig(ctx, dynamicClient, kmsKeyArn, kmsRegion) + o.Expect(err).NotTo(o.HaveOccurred()) + + if needsRollout { + g.By("Waiting for kube-apiserver operator to rollout") + Logf("[Phase 1] Waiting for kube-apiserver to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Phase 1] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Phase 1] Waiting for kube-apiserver to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not become stable after KMS configuration") + Logf("[Phase 1] ✓ kube-apiserver operator is stable") + + // Check KMS plugin health + Logf("[Phase 1] Checking KMS plugin health...") + isHealthy, healthMsg := checkKMSPluginHealth(ctx, dynamicClient) + if !isHealthy { + Logf("[Phase 1] Warning: KMS plugin health check: %s", healthMsg) + Logf("[Phase 1] Note: KMS socket errors during initial rollout are expected and should resolve") + } else { + Logf("[Phase 1] ✓ KMS plugin health check: %s", healthMsg) + } + + // Verify all cluster operators are still stable after KMS configuration + g.By("Verifying all cluster operators are stable after KMS configuration") + Logf("[Phase 1] Checking cluster stability after KMS configuration...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Phase 1] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Phase 1] ✓ All cluster operators are stable after KMS configuration") + } else { + Logf("[Phase 1] ✓ KMS already configured, no rollout needed") + } + + Logf("\n--- Phase 2: Generate Token and Resource Name ---") + g.By("Generating secure token and resource name") + + tokenValue, tokenResourceName := generateSecureToken() + Logf("[Phase 2] ✓ Generated token value: %s", maskToken(tokenValue)) + Logf("[Phase 2] ✓ Token resource name: %s", tokenResourceName) + + Logf("\n--- Phase 3: Get Test User Information ---") + g.By("Getting test user UID") + testUser := "test-user-01" + userUID, err = getUserUID(ctx, dynamicClient, testUser) + if err != nil { + Logf("[Phase 3] Test user not found, creating user") + userUID = createTestUser(ctx, dynamicClient, testUser) + } + o.Expect(userUID).NotTo(o.BeEmpty()) + Logf("[Phase 3] ✓ Test user UID: %s", userUID) + + Logf("\n--- Phase 4: Create OAuthAccessToken ---") + g.By("Creating OAuthAccessToken") + + accessToken := createOAuthAccessToken(tokenResourceName, tokenValue, testUser, userUID) + Logf("[Phase 4] Creating access token: %s", tokenResourceName) + + err = applyOAuthToken(ctx, dynamicClient, "oauthaccesstokens", accessToken) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 4] ✓ OAuthAccessToken created successfully") + + defer func() { + Logf("[Cleanup] Deleting OAuthAccessToken: %s", tokenResourceName) + deleteOAuthToken(ctx, dynamicClient, "oauthaccesstokens", tokenResourceName) + }() + + Logf("\n--- Phase 5: Verify Token Encryption in etcd ---") + g.By("Verifying OAuthAccessToken is encrypted with KMSv2 in etcd") + + isEncrypted, encryptionFormat := masterNode.VerifyOAuthTokenEncryption(ctx, "oauthaccesstokens", tokenResourceName) + o.Expect(isEncrypted).To(o.BeTrue(), "OAuthAccessToken should be encrypted in etcd") + o.Expect(encryptionFormat).To(o.Equal("k8s:enc:kms:v2:"), "OAuthAccessToken should use KMSv2 encryption format") + Logf("[Phase 5] ✓ OAuthAccessToken is encrypted with format: %s", encryptionFormat) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ OAUTH ACCESS TOKEN ENCRYPTION VERIFIED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) + + g.It("should encrypt OAuthAuthorizeTokens using KMS [Suite:openshift/cluster-kube-apiserver-operator/kms] [Timeout:120m][Serial][Disruptive]", + func() { + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ KMS OAUTH AUTHORIZE TOKEN ENCRYPTION VERIFICATION ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + + var ( + expectedStatus = map[string]string{"Progressing": "True"} + kubeApiserverCoStatus = map[string]string{"Available": "True", "Progressing": "False", "Degraded": "False"} + userUID string + ) + + Logf("\n--- Phase 1: Ensure KMS Encryption Configuration ---") + g.By("Configuring KMS encryption for API server") + Logf("[Phase 1] Using shared KMS key: %s", kmsKeyArn) + Logf("[Phase 1] Using region: %s", kmsRegion) + + // Check and apply KMS config if needed + needsRollout, err := checkAndApplyKMSConfig(ctx, dynamicClient, kmsKeyArn, kmsRegion) + o.Expect(err).NotTo(o.HaveOccurred()) + + if needsRollout { + g.By("Waiting for kube-apiserver operator to rollout") + Logf("[Phase 1] Waiting for kube-apiserver to start progressing (timeout: 300s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 300, expectedStatus) + if err != nil { + Logf("[Phase 1] Warning: Operator did not start progressing: %v", err) + } + + Logf("[Phase 1] Waiting for kube-apiserver to become stable (timeout: 1800s)") + err = waitForOperatorStatus(ctx, dynamicClient, "kube-apiserver", 1800, kubeApiserverCoStatus) + o.Expect(err).NotTo(o.HaveOccurred(), "kube-apiserver operator did not become stable after KMS configuration") + Logf("[Phase 1] ✓ kube-apiserver operator is stable") + + // Check KMS plugin health + Logf("[Phase 1] Checking KMS plugin health...") + isHealthy, healthMsg := checkKMSPluginHealth(ctx, dynamicClient) + if !isHealthy { + Logf("[Phase 1] Warning: KMS plugin health check: %s", healthMsg) + Logf("[Phase 1] Note: KMS socket errors during initial rollout are expected and should resolve") + } else { + Logf("[Phase 1] ✓ KMS plugin health check: %s", healthMsg) + } + + // Verify all cluster operators are still stable after KMS configuration + g.By("Verifying all cluster operators are stable after KMS configuration") + Logf("[Phase 1] Checking cluster stability after KMS configuration...") + err = waitForClusterStable(ctx, dynamicClient) + if err != nil { + Logf("[Phase 1] Cluster stability check failed: %v", err) + o.Expect(err).NotTo(o.HaveOccurred()) + } + Logf("[Phase 1] ✓ All cluster operators are stable after KMS configuration") + } else { + Logf("[Phase 1] ✓ KMS already configured, no rollout needed") + } + + Logf("\n--- Phase 2: Generate Auth Code and Resource Name ---") + g.By("Generating secure authorization code and resource name") + + authCode, authResourceName := generateSecureToken() + Logf("[Phase 2] ✓ Generated auth code: %s", maskToken(authCode)) + Logf("[Phase 2] ✓ Auth token resource name: %s", authResourceName) + + Logf("\n--- Phase 3: Get Test User Information ---") + g.By("Getting test user UID") + testUser := "test-user-01" + userUID, err = getUserUID(ctx, dynamicClient, testUser) + if err != nil { + Logf("[Phase 3] Test user not found, creating user") + userUID = createTestUser(ctx, dynamicClient, testUser) + } + o.Expect(userUID).NotTo(o.BeEmpty()) + Logf("[Phase 3] ✓ Test user UID: %s", userUID) + + Logf("\n--- Phase 4: Create OAuthAuthorizeToken ---") + g.By("Creating OAuthAuthorizeToken") + + authorizeToken := createOAuthAuthorizeToken(authResourceName, authCode, testUser, userUID) + Logf("[Phase 4] Creating authorize token: %s", authResourceName) + + err = applyOAuthToken(ctx, dynamicClient, "oauthauthorizetokens", authorizeToken) + o.Expect(err).NotTo(o.HaveOccurred()) + Logf("[Phase 4] ✓ OAuthAuthorizeToken created successfully") + + defer func() { + Logf("[Cleanup] Deleting OAuthAuthorizeToken: %s", authResourceName) + deleteOAuthToken(ctx, dynamicClient, "oauthauthorizetokens", authResourceName) + }() + + Logf("\n--- Phase 5: Verify Token Encryption in etcd ---") + g.By("Verifying OAuthAuthorizeToken is encrypted with KMSv2 in etcd") + + isEncrypted, encryptionFormat := masterNode.VerifyOAuthTokenEncryption(ctx, "oauthauthorizetokens", authResourceName) + o.Expect(isEncrypted).To(o.BeTrue(), "OAuthAuthorizeToken should be encrypted in etcd") + o.Expect(encryptionFormat).To(o.Equal("k8s:enc:kms:v2:"), "OAuthAuthorizeToken should use KMSv2 encryption format") + Logf("[Phase 5] ✓ OAuthAuthorizeToken is encrypted with format: %s", encryptionFormat) + + Logf("\n╔════════════════════════════════════════════════════════════╗") + Logf("║ ✓ OAUTH AUTHORIZE TOKEN ENCRYPTION VERIFIED SUCCESSFULLY ║") + Logf("╚════════════════════════════════════════════════════════════╝\n") + }) +}) diff --git a/test/extended/tests-extension/testdata/kms_tests_aws.yaml b/test/extended/tests-extension/testdata/kms_tests_aws.yaml new file mode 100644 index 000000000..d1aced61c --- /dev/null +++ b/test/extended/tests-extension/testdata/kms_tests_aws.yaml @@ -0,0 +1,119 @@ +- name: Should fail to create encrypt with KMS for AWS without region + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + expectedError: "spec.encryption.kms.aws.region: Required value" + +- name: Should not allow kms config with encrypt aescbc + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: aescbc + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + region: us-east-1 + expectedError: "kms config is required when encryption type is KMS, and forbidden otherwise" + +- name: Should fail to create with an empty KMS config + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: {} + expectedError: "spec.encryption.kms.type: Required value" + +- name: Should fail to create with kms type AWS but without aws config + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + expectedError: "aws config is required when kms provider type is AWS, and forbidden otherwise" + +- name: Should fail to create AWS KMS without a keyARN + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + region: us-east-1 + expectedError: "spec.encryption.kms.aws.keyARN: Required value" + +- name: Should fail to create AWS KMS with invalid keyARN format + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: not-a-kms-arn + region: us-east-1 + expectedError: "keyARN must follow the format `arn:aws:kms:::key/`. The account ID must be a 12 digit number and the region and key ID should consist only of lowercase hexadecimal characters and hyphens (-)." + +- name: Should fail to create AWS KMS with empty region + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + region: "" + expectedError: "spec.encryption.kms.aws.region in body should be at least 1 chars long" + +- name: Should fail to create AWS KMS with invalid region format + initial: | + apiVersion: config.openshift.io/v1 + kind: APIServer + metadata: + name: cluster + spec: + encryption: + type: KMS + kms: + type: AWS + aws: + keyARN: arn:aws:kms:us-east-1:101010101010:key/9a512e29-0d9c-4cf5-8174-fc1a5b22cd6a + region: "INVALID-REGION" + expectedError: "region must be a valid AWS region, consisting of lowercase characters, digits and hyphens (-) only." + diff --git a/test/extended/tests-extension/util.go b/test/extended/tests-extension/util.go new file mode 100644 index 000000000..0033edc1f --- /dev/null +++ b/test/extended/tests-extension/util.go @@ -0,0 +1,802 @@ +package extended + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "os" + "os/exec" + "strings" + "time" + + g "github.com/onsi/ginkgo/v2" + "gopkg.in/yaml.v2" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// Logf logs formatted output to Ginkgo writer +func Logf(format string, args ...interface{}) { + fmt.Fprintf(g.GinkgoWriter, format+"\n", args...) +} + +// Failf fails the test with formatted message +func Failf(format string, args ...interface{}) { + g.Fail(fmt.Sprintf(format, args...)) +} + +// GetKubeConfig gets KUBECONFIG from environment and validates it exists +func GetKubeConfig() string { + kubeconfig := os.Getenv("KUBECONFIG") + if kubeconfig == "" { + Logf("[Setup] KUBECONFIG not set, skipping test") + g.Skip("KUBECONFIG environment variable not set") + } + Logf("[Setup] Using KUBECONFIG: %s", kubeconfig) + return kubeconfig +} + +// CreateKubernetesClients creates Kubernetes and dynamic clients from kubeconfig +func CreateKubernetesClients(kubeconfig string) (*kubernetes.Clientset, dynamic.Interface) { + config, err := clientcmd.BuildConfigFromFlags("", kubeconfig) + if err != nil { + Failf("Failed to load Kubernetes config: %v", err) + } + Logf("[Setup] Kubernetes config loaded successfully") + + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + Failf("Failed to create Kubernetes client: %v", err) + } + Logf("[Setup] Kubernetes client created") + + dynamicClient, err := dynamic.NewForConfig(config) + if err != nil { + Failf("Failed to create dynamic client: %v", err) + } + Logf("[Setup] Dynamic client created") + + return kubeClient, dynamicClient +} + +// patchFeatureGate patches the cluster featuregate +func patchFeatureGate(ctx context.Context, client dynamic.Interface, patchData string) error { + featureGateGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "featuregates", + } + + _, err := client.Resource(featureGateGVR).Patch(ctx, "cluster", "application/merge-patch+json", + []byte(patchData), v1.PatchOptions{}) + + return err +} + +// applyAPIServerConfig attempts to apply an APIServer configuration +func applyAPIServerConfig(ctx context.Context, client dynamic.Interface, yamlData []byte) error { + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + // Parse YAML to get the desired spec + var yamlObj interface{} + err := yaml.Unmarshal(yamlData, &yamlObj) + if err != nil { + return fmt.Errorf("failed to unmarshal YAML: %w", err) + } + + // Convert to unstructured + yamlMap, err := convertToStringMap(yamlObj) + if err != nil { + return fmt.Errorf("failed to convert YAML to unstructured: %w", err) + } + + // Get the existing APIServer resource to preserve metadata including resourceVersion + existing, err := client.Resource(apiServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return fmt.Errorf("failed to get existing apiserver: %w", err) + } + + // Extract spec from YAML and set it on the existing resource + if spec, found := yamlMap["spec"]; found { + existing.Object["spec"] = spec + } + + // Try to update the APIServer resource - this will trigger server-side validation + _, err = client.Resource(apiServerGVR).Update(ctx, existing, v1.UpdateOptions{}) + return err +} + +// convertToStringMap converts interface{} to map[string]interface{} recursively +func convertToStringMap(i interface{}) (map[string]interface{}, error) { + switch x := i.(type) { + case map[interface{}]interface{}: + m := map[string]interface{}{} + for k, v := range x { + strKey, ok := k.(string) + if !ok { + return nil, fmt.Errorf("non-string key found: %v", k) + } + switch val := v.(type) { + case map[interface{}]interface{}: + converted, err := convertToStringMap(val) + if err != nil { + return nil, err + } + m[strKey] = converted + case []interface{}: + m[strKey] = convertSlice(val) + default: + m[strKey] = val + } + } + return m, nil + case map[string]interface{}: + return x, nil + default: + return nil, fmt.Errorf("expected map, got %T", i) + } +} + +// convertSlice converts []interface{} recursively +func convertSlice(s []interface{}) []interface{} { + result := make([]interface{}, len(s)) + for i, v := range s { + switch val := v.(type) { + case map[interface{}]interface{}: + converted, _ := convertToStringMap(val) + result[i] = converted + case []interface{}: + result[i] = convertSlice(val) + default: + result[i] = val + } + } + return result +} + +// waitForClusterStable waits for the cluster to be stable +// Checks that all cluster operators are Available=True, Progressing=False, Degraded=False +func waitForClusterStable(ctx context.Context, client dynamic.Interface) error { + coGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "clusteroperators", + } + + // Wait for all COs to be stable + return wait.PollUntilContextTimeout(ctx, 10*time.Second, 18*time.Minute, true, + func(ctx context.Context) (bool, error) { + coList, err := client.Resource(coGVR).List(ctx, v1.ListOptions{}) + if err != nil { + return false, nil + } + + unstableCOs := []string{} + for _, item := range coList.Items { + coName := item.GetName() + conditions, found, err := unstructured.NestedSlice(item.Object, "status", "conditions") + if !found || err != nil { + unstableCOs = append(unstableCOs, coName) + continue + } + + currentStatus := make(map[string]string) + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + condType := condition["type"].(string) + status := condition["status"].(string) + currentStatus[condType] = status + } + + // Check if CO is stable (Available=True, Progressing=False, Degraded=False) + if currentStatus["Available"] != "True" || + currentStatus["Progressing"] != "False" || + currentStatus["Degraded"] != "False" { + unstableCOs = append(unstableCOs, coName) + } + } + + if len(unstableCOs) > 0 { + return false, nil + } + + return true, nil + }) +} + +// waitForOperatorStatus waits for an operator to reach the expected status +func waitForOperatorStatus(ctx context.Context, client dynamic.Interface, operatorName string, timeoutSeconds int, expectedStatus map[string]string) error { + coGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "clusteroperators", + } + + return wait.PollUntilContextTimeout(ctx, 10*time.Second, time.Duration(timeoutSeconds)*time.Second, true, + func(ctx context.Context) (bool, error) { + co, err := client.Resource(coGVR).Get(ctx, operatorName, v1.GetOptions{}) + if err != nil { + return false, nil + } + + conditions, found, err := unstructured.NestedSlice(co.Object, "status", "conditions") + if !found || err != nil { + return false, nil + } + + currentStatus := make(map[string]string) + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + condType := condition["type"].(string) + status := condition["status"].(string) + currentStatus[condType] = status + } + + // Check if current status matches expected status + for expectedType, expectedValue := range expectedStatus { + if currentStatus[expectedType] != expectedValue { + return false, nil + } + } + + return true, nil + }) +} + +// checkAndApplyKMSConfig checks if KMS config is already applied and correct, applies if needed +// Returns: (needsRollout bool, error) +func checkAndApplyKMSConfig(ctx context.Context, client dynamic.Interface, expectedKeyARN, expectedRegion string) (bool, error) { + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + // Get current apiserver config + apiServer, err := client.Resource(apiServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get apiserver config: %w", err) + } + + // Check current encryption config + encType, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "type") + if encType == "KMS" { + // KMS is already configured, verify it's correct + kmsType, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "kms", "type") + if kmsType == "AWS" { + currentKeyARN, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "kms", "aws", "keyARN") + currentRegion, _, _ := unstructured.NestedString(apiServer.Object, "spec", "encryption", "kms", "aws", "region") + + if currentKeyARN == expectedKeyARN && currentRegion == expectedRegion { + Logf("[KMS-Config] ✓ KMS is already configured correctly") + Logf("[KMS-Config] Key ARN: %s", currentKeyARN) + Logf("[KMS-Config] Region: %s", currentRegion) + return false, nil // No need to apply, already correct + } + + Logf("[KMS-Config] KMS is configured but with different values") + Logf("[KMS-Config] Current Key ARN: %s", currentKeyARN) + Logf("[KMS-Config] Expected Key ARN: %s", expectedKeyARN) + Logf("[KMS-Config] Current Region: %s", currentRegion) + Logf("[KMS-Config] Expected Region: %s", expectedRegion) + } + } else if encType != "" { + Logf("[KMS-Config] Current encryption type: %s (not KMS)", encType) + } else { + Logf("[KMS-Config] No encryption currently configured") + } + + // Apply KMS configuration + Logf("[KMS-Config] Applying KMS encryption configuration...") + kmsConfig := fmt.Sprintf(`{ + "spec": { + "encryption": { + "type": "KMS", + "kms": { + "type": "AWS", + "aws": { + "keyARN": "%s", + "region": "%s" + } + } + } + } + }`, expectedKeyARN, expectedRegion) + + err = patchAPIServerConfig(ctx, client, kmsConfig) + if err != nil { + return false, fmt.Errorf("failed to apply KMS config: %w", err) + } + + Logf("[KMS-Config] ✓ KMS configuration applied") + return true, nil // Config was applied, rollout needed +} + +// patchAPIServerConfig patches the API server configuration +func patchAPIServerConfig(ctx context.Context, client dynamic.Interface, patchData string) error { + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + _, err := client.Resource(apiServerGVR).Patch(ctx, "cluster", "application/merge-patch+json", + []byte(patchData), v1.PatchOptions{}) + + return err +} + +// createNamespace creates a namespace +func createNamespace(ctx context.Context, kubeClient *kubernetes.Clientset, namespace string) error { + ns := &corev1.Namespace{ + ObjectMeta: v1.ObjectMeta{ + Name: namespace, + }, + } + _, err := kubeClient.CoreV1().Namespaces().Create(ctx, ns, v1.CreateOptions{}) + return err +} + +// deleteNamespace deletes a namespace +func deleteNamespace(ctx context.Context, kubeClient *kubernetes.Clientset, namespace string) error { + return kubeClient.CoreV1().Namespaces().Delete(ctx, namespace, v1.DeleteOptions{}) +} + +// createSecret creates a secret in the specified namespace +func createSecret(ctx context.Context, kubeClient *kubernetes.Clientset, namespace, name string, data map[string]string) error { + secret := &corev1.Secret{ + ObjectMeta: v1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + StringData: data, + Type: corev1.SecretTypeOpaque, + } + _, err := kubeClient.CoreV1().Secrets(namespace).Create(ctx, secret, v1.CreateOptions{}) + return err +} + +// generateSecureToken generates a secure token and its resource name +// Returns: (tokenValue, tokenResourceName) +func generateSecureToken() (string, string) { + // Generate 32-byte random token + rawToken := make([]byte, 32) + _, err := rand.Read(rawToken) + if err != nil { + Failf("Failed to generate random token: %v", err) + } + + // Base64-URL encode the token + tokenValue := base64.URLEncoding.EncodeToString(rawToken) + tokenValue = strings.TrimRight(tokenValue, "=") + + // Calculate SHA256 hash of token + hash := sha256.Sum256([]byte(tokenValue)) + + // Base64-URL encode the hash + tokenHash := base64.URLEncoding.EncodeToString(hash[:]) + tokenHash = strings.TrimRight(tokenHash, "=") + + // Create resource name with sha256~ prefix + resourceName := "sha256~" + tokenHash + + return tokenValue, resourceName +} + +// maskToken masks a token for logging (shows first and last 4 characters) +func maskToken(token string) string { + if len(token) <= 8 { + return "****" + } + return token[:4] + "..." + token[len(token)-4:] +} + +// getUserUID gets the UID of a user +func getUserUID(ctx context.Context, client dynamic.Interface, userName string) (string, error) { + userGVR := schema.GroupVersionResource{ + Group: "user.openshift.io", + Version: "v1", + Resource: "users", + } + + user, err := client.Resource(userGVR).Get(ctx, userName, v1.GetOptions{}) + if err != nil { + return "", err + } + + uid, found, err := unstructured.NestedString(user.Object, "metadata", "uid") + if !found || err != nil { + return "", fmt.Errorf("uid not found in user object") + } + + return uid, nil +} + +// createTestUser creates a test user (simplified - in real cluster this would be done via IDP) +func createTestUser(ctx context.Context, client dynamic.Interface, userName string) string { + // In a real cluster, users come from IDP + // For testing, we'll use a well-known test user UID + // This is a placeholder - actual implementation depends on cluster setup + Logf("Using placeholder UID for test user: %s", userName) + return "00000000-0000-0000-0000-000000000001" +} + +// createOAuthAccessToken creates an OAuthAccessToken object +func createOAuthAccessToken(resourceName, tokenValue, userName, userUID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "oauth.openshift.io/v1", + "kind": "OAuthAccessToken", + "metadata": map[string]interface{}{ + "name": resourceName, + }, + "clientName": "openshift-challenging-client", + "userName": userName, + "userUID": userUID, + "scopes": []interface{}{ + "user:full", + }, + "expiresIn": 86400, // 24 hours + "redirectURI": "https://oauth-openshift.apps.example.com/oauth/token/implicit", + "token": tokenValue, + }, + } +} + +// createOAuthAuthorizeToken creates an OAuthAuthorizeToken object +func createOAuthAuthorizeToken(resourceName, authCode, userName, userUID string) *unstructured.Unstructured { + return &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "oauth.openshift.io/v1", + "kind": "OAuthAuthorizeToken", + "metadata": map[string]interface{}{ + "name": resourceName, + }, + "clientName": "openshift-challenging-client", + "userName": userName, + "userUID": userUID, + "scopes": []interface{}{ + "user:full", + }, + "expiresIn": 300, // 5 minutes + "redirectURI": "https://oauth-openshift.apps.example.com/oauth/token/implicit", + "code": authCode, + }, + } +} + +// applyOAuthToken applies an OAuth token (access or authorize) +func applyOAuthToken(ctx context.Context, client dynamic.Interface, resource string, token *unstructured.Unstructured) error { + oauthGVR := schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: resource, + } + + _, err := client.Resource(oauthGVR).Create(ctx, token, v1.CreateOptions{}) + return err +} + +// deleteOAuthToken deletes an OAuth token +func deleteOAuthToken(ctx context.Context, client dynamic.Interface, resource, name string) error { + oauthGVR := schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: resource, + } + + return client.Resource(oauthGVR).Delete(ctx, name, v1.DeleteOptions{}) +} + +// checkKMSPluginHealth checks if KMS plugin pods are healthy in kube-apiserver +// Returns: (isHealthy, message) +func checkKMSPluginHealth(ctx context.Context, client dynamic.Interface) (bool, string) { + kubeAPIServerGVR := schema.GroupVersionResource{ + Group: "operator.openshift.io", + Version: "v1", + Resource: "kubeapiservers", + } + + kubeAPIServer, err := client.Resource(kubeAPIServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return false, fmt.Sprintf("Failed to get kubeapiserver: %v", err) + } + + // Check conditions for KMS health + conditions, found, err := unstructured.NestedSlice(kubeAPIServer.Object, "status", "conditions") + if !found || err != nil { + return false, "KMS conditions not found in kubeapiserver status" + } + + // Look for KMS-related error conditions + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + condType, _ := condition["type"].(string) + status, _ := condition["status"].(string) + message, _ := condition["message"].(string) + reason, _ := condition["reason"].(string) + + // Check for Degraded condition related to KMS + if condType == "Degraded" && status == "True" { + if strings.Contains(message, "kms-provider") || strings.Contains(message, "kms") { + return false, fmt.Sprintf("KMS degraded: %s - %s", reason, message) + } + } + + // Check for KMSConnectionDegraded or similar conditions + if strings.Contains(condType, "KMS") && status == "True" { + return false, fmt.Sprintf("KMS condition %s: %s - %s", condType, reason, message) + } + } + + return true, "KMS plugin appears healthy" +} + +// verifyEncryptionType verifies the encryption type configured in the APIServer +// This is a generic function that works across all cloud platforms +// Returns: (encryptionType, encryptionCompleted) +func verifyEncryptionType(ctx context.Context, client dynamic.Interface) (string, bool) { + Logf("[Verify] Checking encryption configuration") + + apiServerGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "apiservers", + } + + apiServer, err := client.Resource(apiServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + Logf("[Verify] Failed to get apiserver: %v", err) + return "", false + } + + // Get encryption type + encType, found, err := unstructured.NestedString(apiServer.Object, "spec", "encryption", "type") + if !found || err != nil { + Logf("[Verify] Encryption type not found in spec") + return "", false + } + + // Check kubeapiserver for encryption status + kubeAPIServerGVR := schema.GroupVersionResource{ + Group: "operator.openshift.io", + Version: "v1", + Resource: "kubeapiservers", + } + + kubeAPIServer, err := client.Resource(kubeAPIServerGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + Logf("[Verify] Failed to get kubeapiserver: %v", err) + return encType, false + } + + // Get encryption conditions + conditions, found, err := unstructured.NestedSlice(kubeAPIServer.Object, "status", "conditions") + if !found || err != nil { + Logf("[Verify] Conditions not found in kubeapiserver status") + return encType, false + } + + // Check for Encrypted condition + encryptionCompleted := false + for _, cond := range conditions { + condition := cond.(map[string]interface{}) + if condition["type"] == "Encrypted" { + reason, _ := condition["reason"].(string) + if reason == "EncryptionCompleted" { + encryptionCompleted = true + message, _ := condition["message"].(string) + Logf("[Verify] Encryption status: %s - %s", reason, message) + } + } + } + + return encType, encryptionCompleted +} + +// isFeatureGateEnabled checks if a specific feature gate is enabled +func isFeatureGateEnabled(ctx context.Context, client dynamic.Interface, featureName string) (bool, error) { + featureGateGVR := schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "featuregates", + } + + featureGate, err := client.Resource(featureGateGVR).Get(ctx, "cluster", v1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("failed to get feature gate: %w", err) + } + + // Check status for enabled features + featureGates, found, err := unstructured.NestedSlice(featureGate.Object, "status", "featureGates") + if !found || err != nil { + return false, nil + } + + for _, fg := range featureGates { + fgDetails := fg.(map[string]interface{}) + enabled, found, err := unstructured.NestedSlice(fgDetails, "enabled") + if !found || err != nil { + continue + } + + for _, feature := range enabled { + featureMap := feature.(map[string]interface{}) + if name, found := featureMap["name"]; found && name == featureName { + return true, nil + } + } + } + + return false, nil +} + +// getStatusString returns a formatted status string +func getStatusString(enabled bool) string { + if enabled { + return "Already Enabled" + } + return "Newly Enabled " +} + +// truncateString truncates a string to the specified length +func truncateString(s string, maxLen int) string { + if len(s) <= maxLen { + // Pad with spaces to maintain box alignment + return fmt.Sprintf("%-"+fmt.Sprint(maxLen)+"s║", s) + } + return s[:maxLen-3] + "...║" +} + +// debugNode executes a command on a node using oc debug with chroot +// Returns stdout, stderr, and error +func debugNode(nodeName string, cmd ...string) (string, string, error) { + // Create context with timeout + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Build the oc debug command arguments + // oc debug node/ -- chroot /host + args := []string{"debug", fmt.Sprintf("node/%s", nodeName), "--", "chroot", "/host"} + args = append(args, cmd...) + + Logf("[DebugNode] Executing: oc %s", strings.Join(args, " ")) + + command := exec.CommandContext(ctx, "oc", args...) + + var stdout, stderr bytes.Buffer + command.Stdout = &stdout + command.Stderr = &stderr + + err := command.Run() + + return stdout.String(), stderr.String(), err +} + +// debugNodeRetryWithChroot executes a command on a node with retry logic +// Similar to compat_otp.DebugNodeRetryWithOptionsAndChroot +func debugNodeRetryWithChroot(nodeName string, cmd ...string) (string, error) { + var stdErr string + var stdOut string + var err error + + // Retry logic with polling + errWait := wait.Poll(3*time.Second, 30*time.Second, func() (bool, error) { + stdOut, stdErr, err = debugNode(nodeName, cmd...) + if err != nil { + Logf("[DebugNode] Retry attempt failed: %v", err) + return false, nil // Retry + } + return true, nil // Success + }) + + if errWait != nil { + return "", fmt.Errorf("failed to debug node after retries: %w", errWait) + } + + // Combine stdout and stderr + return strings.Join([]string{stdOut, stdErr}, "\n"), err +} + +// executeNodeCommand executes a command on a node using oc debug with chroot +func executeNodeCommand(nodeName, command string) (string, error) { + Logf("[Exec] Running command on node %s", nodeName) + Logf("[Exec] Command: %s", command) + + // Execute the command with retry + output, err := debugNodeRetryWithChroot(nodeName, "/bin/bash", "-c", command) + if err != nil { + Logf("[Exec] Command failed: %v", err) + return output, fmt.Errorf("failed to execute command on node %s: %w", nodeName, err) + } + + Logf("[Exec] Command completed successfully") + return output, nil +} + +// getAwsCredentialFromCluster retrieves AWS credentials from the cluster's kube-system namespace +// and sets them as environment variables +func getAwsCredentialFromCluster() error { + Logf("[AWS-Creds] Retrieving AWS credentials from cluster") + + // Get the aws-creds secret from kube-system namespace + cmd := exec.Command("oc", "get", "secret/aws-creds", "-n", "kube-system", "-o", "json") + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err := cmd.Run() + if err != nil { + // Skip for STS and C2S clusters + Logf("[AWS-Creds] Did not get credential to access AWS: %v", err) + g.Skip("Did not get credential to access AWS, skip the testing.") + return fmt.Errorf("failed to get AWS credentials from cluster: %w", err) + } + + // Parse the JSON output + var secret map[string]interface{} + if err := json.Unmarshal(stdout.Bytes(), &secret); err != nil { + return fmt.Errorf("failed to parse secret JSON: %w", err) + } + + // Extract base64-encoded credentials + data, ok := secret["data"].(map[string]interface{}) + if !ok { + return fmt.Errorf("secret data not found") + } + + accessKeyIDBase64, ok1 := data["aws_access_key_id"].(string) + secureKeyBase64, ok2 := data["aws_secret_access_key"].(string) + if !ok1 || !ok2 { + return fmt.Errorf("AWS credentials not found in secret") + } + + // Decode base64 credentials + accessKeyID, err := base64.StdEncoding.DecodeString(accessKeyIDBase64) + if err != nil { + return fmt.Errorf("failed to decode access key ID: %w", err) + } + + secureKey, err := base64.StdEncoding.DecodeString(secureKeyBase64) + if err != nil { + return fmt.Errorf("failed to decode secret access key: %w", err) + } + + // Get AWS region from infrastructure resource + cmd = exec.Command("oc", "get", "infrastructure", "cluster", "-o=jsonpath={.status.platformStatus.aws.region}") + stdout.Reset() + stderr.Reset() + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + err = cmd.Run() + if err != nil { + return fmt.Errorf("failed to get AWS region: %w", err) + } + + clusterRegion := strings.TrimSpace(stdout.String()) + + // Set environment variables + os.Setenv("AWS_ACCESS_KEY_ID", string(accessKeyID)) + os.Setenv("AWS_SECRET_ACCESS_KEY", string(secureKey)) + os.Setenv("AWS_REGION", clusterRegion) + + Logf("[AWS-Creds] ✓ AWS credentials set successfully") + Logf("[AWS-Creds] Region: %s", clusterRegion) + + return nil +}