|
| 1 | +/* |
| 2 | +Copyright 2024 The Kubernetes Authors. |
| 3 | +Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | +you may not use this file except in compliance with the License. |
| 5 | +You may obtain a copy of the License at |
| 6 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 7 | +Unless required by applicable law or agreed to in writing, software |
| 8 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 9 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 10 | +See the License for the specific language governing permissions and |
| 11 | +limitations under the License. |
| 12 | +*/ |
| 13 | + |
| 14 | +package e2e |
| 15 | + |
| 16 | +import ( |
| 17 | + "context" |
| 18 | + "fmt" |
| 19 | + "strings" |
| 20 | + "time" |
| 21 | + |
| 22 | + "github.com/aws/aws-sdk-go-v2/aws" |
| 23 | + "github.com/aws/aws-sdk-go-v2/config" |
| 24 | + "github.com/aws/aws-sdk-go-v2/service/ec2" |
| 25 | + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" |
| 26 | + "github.com/aws/aws-sdk-go-v2/service/elasticloadbalancingv2" |
| 27 | + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| 28 | + "k8s.io/apimachinery/pkg/util/wait" |
| 29 | + clientset "k8s.io/client-go/kubernetes" |
| 30 | + "k8s.io/kubernetes/test/e2e/framework" |
| 31 | +) |
| 32 | + |
| 33 | +// awsHelper provides AWS-specific operations for E2E tests |
| 34 | +type awsHelper struct { |
| 35 | + ec2Client *ec2.Client |
| 36 | + elbClient *elasticloadbalancingv2.Client |
| 37 | + ctx context.Context |
| 38 | + clusterName string |
| 39 | + vpcID string |
| 40 | + awsRegion string |
| 41 | +} |
| 42 | + |
| 43 | +// newAWSHelper creates a new AWS helper with automatic cluster discovery |
| 44 | +func newAWSHelper(ctx context.Context, cs clientset.Interface) (*awsHelper, error) { |
| 45 | + helper := &awsHelper{ |
| 46 | + ctx: ctx, |
| 47 | + } |
| 48 | + |
| 49 | + // Discover cluster configuration from nodes |
| 50 | + if err := helper.discoverClusterTag(cs); err != nil { |
| 51 | + return nil, fmt.Errorf("failed to discover cluster configuration: %w", err) |
| 52 | + } |
| 53 | + |
| 54 | + // Load AWS SDK configuration |
| 55 | + cfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(helper.awsRegion)) |
| 56 | + if err != nil { |
| 57 | + return nil, fmt.Errorf("unable to load AWS config: %w", err) |
| 58 | + } |
| 59 | + |
| 60 | + helper.ec2Client = ec2.NewFromConfig(cfg) |
| 61 | + helper.elbClient = elasticloadbalancingv2.NewFromConfig(cfg) |
| 62 | + |
| 63 | + return helper, nil |
| 64 | +} |
| 65 | + |
| 66 | +// discoverClusterTag discovers cluster configuration from node metadata |
| 67 | +func (h *awsHelper) discoverClusterTag(cs clientset.Interface) error { |
| 68 | + // List nodes to get provider ID |
| 69 | + nodes, err := cs.CoreV1().Nodes().List(h.ctx, metav1.ListOptions{}) |
| 70 | + if err != nil { |
| 71 | + return fmt.Errorf("failed to list nodes: %w", err) |
| 72 | + } |
| 73 | + |
| 74 | + if len(nodes.Items) == 0 { |
| 75 | + return fmt.Errorf("no nodes found in cluster") |
| 76 | + } |
| 77 | + |
| 78 | + // Extract region and instance ID from provider ID |
| 79 | + // Format: aws:///zone/instance-id |
| 80 | + providerID := nodes.Items[0].Spec.ProviderID |
| 81 | + parts := strings.Split(providerID, "/") |
| 82 | + if len(parts) < 2 { |
| 83 | + return fmt.Errorf("invalid provider ID format: %s", providerID) |
| 84 | + } |
| 85 | + instanceID := parts[len(parts)-1] |
| 86 | + |
| 87 | + // Extract region from zone in provider ID |
| 88 | + // Format: aws:///us-east-1a/i-xxxxx |
| 89 | + if len(parts) >= 4 { |
| 90 | + zone := parts[len(parts)-2] |
| 91 | + // Remove last character (availability zone letter) to get region |
| 92 | + if len(zone) > 0 { |
| 93 | + h.awsRegion = zone[:len(zone)-1] |
| 94 | + } |
| 95 | + } |
| 96 | + |
| 97 | + // Create temporary EC2 client to discover VPC |
| 98 | + cfg, err := config.LoadDefaultConfig(h.ctx, config.WithRegion(h.awsRegion)) |
| 99 | + if err != nil { |
| 100 | + return fmt.Errorf("unable to load AWS config for region %s: %w", h.awsRegion, err) |
| 101 | + } |
| 102 | + |
| 103 | + tmpEC2Client := ec2.NewFromConfig(cfg) |
| 104 | + |
| 105 | + // Describe instance to get VPC ID and cluster tag |
| 106 | + instancesOutput, err := tmpEC2Client.DescribeInstances(h.ctx, &ec2.DescribeInstancesInput{ |
| 107 | + InstanceIds: []string{instanceID}, |
| 108 | + }) |
| 109 | + if err != nil { |
| 110 | + return fmt.Errorf("failed to describe instance %s: %w", instanceID, err) |
| 111 | + } |
| 112 | + |
| 113 | + if len(instancesOutput.Reservations) == 0 || len(instancesOutput.Reservations[0].Instances) == 0 { |
| 114 | + return fmt.Errorf("instance %s not found", instanceID) |
| 115 | + } |
| 116 | + |
| 117 | + instance := instancesOutput.Reservations[0].Instances[0] |
| 118 | + h.vpcID = aws.ToString(instance.VpcId) |
| 119 | + |
| 120 | + // Extract cluster name from instance tags |
| 121 | + for _, tag := range instance.Tags { |
| 122 | + if strings.HasPrefix(aws.ToString(tag.Key), "kubernetes.io/cluster/") { |
| 123 | + h.clusterName = strings.TrimPrefix(aws.ToString(tag.Key), "kubernetes.io/cluster/") |
| 124 | + break |
| 125 | + } |
| 126 | + } |
| 127 | + |
| 128 | + if h.clusterName == "" { |
| 129 | + return fmt.Errorf("cluster tag not found on instance %s", instanceID) |
| 130 | + } |
| 131 | + |
| 132 | + framework.Logf("Discovered cluster configuration: region=%s, vpcID=%s, clusterName=%s", |
| 133 | + h.awsRegion, h.vpcID, h.clusterName) |
| 134 | + |
| 135 | + return nil |
| 136 | +} |
| 137 | + |
| 138 | +// createSecurityGroup creates a new security group with proper tagging for BYO SG tests |
| 139 | +func (h *awsHelper) createSecurityGroup(name, description string) (string, error) { |
| 140 | + result, err := h.ec2Client.CreateSecurityGroup(h.ctx, &ec2.CreateSecurityGroupInput{ |
| 141 | + GroupName: aws.String(name), |
| 142 | + Description: aws.String(description), |
| 143 | + TagSpecifications: []ec2types.TagSpecification{ |
| 144 | + { |
| 145 | + ResourceType: ec2types.ResourceTypeSecurityGroup, |
| 146 | + Tags: []ec2types.Tag{ |
| 147 | + { |
| 148 | + Key: aws.String("Name"), |
| 149 | + Value: aws.String(name), |
| 150 | + }, |
| 151 | + { |
| 152 | + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", h.clusterName)), |
| 153 | + Value: aws.String("shared"), // "shared" tag = user-managed, controller must not delete |
| 154 | + }, |
| 155 | + }, |
| 156 | + }, |
| 157 | + }, |
| 158 | + VpcId: aws.String(h.vpcID), |
| 159 | + }) |
| 160 | + if err != nil { |
| 161 | + return "", fmt.Errorf("failed to create security group: %w", err) |
| 162 | + } |
| 163 | + |
| 164 | + sgID := aws.ToString(result.GroupId) |
| 165 | + framework.Logf("Created security group %s (ID: %s) with 'shared' tag", name, sgID) |
| 166 | + |
| 167 | + return sgID, nil |
| 168 | +} |
| 169 | + |
| 170 | +// getSecurityGroup retrieves a security group by ID |
| 171 | +func (h *awsHelper) getSecurityGroup(sgID string) (*ec2types.SecurityGroup, error) { |
| 172 | + result, err := h.ec2Client.DescribeSecurityGroups(h.ctx, &ec2.DescribeSecurityGroupsInput{ |
| 173 | + GroupIds: []string{sgID}, |
| 174 | + }) |
| 175 | + if err != nil { |
| 176 | + return nil, err |
| 177 | + } |
| 178 | + |
| 179 | + if len(result.SecurityGroups) == 0 { |
| 180 | + return nil, fmt.Errorf("security group %s not found", sgID) |
| 181 | + } |
| 182 | + |
| 183 | + return &result.SecurityGroups[0], nil |
| 184 | +} |
| 185 | + |
| 186 | +// deleteSecurityGroup deletes a security group |
| 187 | +func (h *awsHelper) deleteSecurityGroup(sgID string) error { |
| 188 | + _, err := h.ec2Client.DeleteSecurityGroup(h.ctx, &ec2.DeleteSecurityGroupInput{ |
| 189 | + GroupId: aws.String(sgID), |
| 190 | + }) |
| 191 | + return err |
| 192 | +} |
| 193 | + |
| 194 | +// waitForSecurityGroupDeletion waits for a security group to be deleted, handling dependencies |
| 195 | +func (h *awsHelper) waitForSecurityGroupDeletion(sgID string, timeout time.Duration) error { |
| 196 | + framework.Logf("Waiting for security group %s deletion (timeout: %v)", sgID, timeout) |
| 197 | + |
| 198 | + return wait.PollImmediate(1*time.Second, timeout, func() (bool, error) { |
| 199 | + // Try to get the security group |
| 200 | + _, err := h.getSecurityGroup(sgID) |
| 201 | + if err != nil { |
| 202 | + // Security group not found = successfully deleted |
| 203 | + if strings.Contains(err.Error(), "InvalidGroup.NotFound") || |
| 204 | + strings.Contains(err.Error(), "does not exist") { |
| 205 | + framework.Logf("Security group %s successfully deleted", sgID) |
| 206 | + return true, nil |
| 207 | + } |
| 208 | + // Other errors are unexpected |
| 209 | + return false, fmt.Errorf("error checking security group %s: %w", sgID, err) |
| 210 | + } |
| 211 | + |
| 212 | + // Security group still exists, try to delete it |
| 213 | + err = h.deleteSecurityGroup(sgID) |
| 214 | + if err != nil { |
| 215 | + // Handle dependency violations - keep waiting |
| 216 | + if strings.Contains(err.Error(), "DependencyViolation") || |
| 217 | + strings.Contains(err.Error(), "InvalidGroup.InUse") { |
| 218 | + framework.Logf("Security group %s still in use, retrying...", sgID) |
| 219 | + return false, nil |
| 220 | + } |
| 221 | + |
| 222 | + // Already deleted (race condition) |
| 223 | + if strings.Contains(err.Error(), "InvalidGroup.NotFound") { |
| 224 | + framework.Logf("Security group %s successfully deleted", sgID) |
| 225 | + return true, nil |
| 226 | + } |
| 227 | + |
| 228 | + // Other errors are failures |
| 229 | + return false, fmt.Errorf("error deleting security group %s: %w", sgID, err) |
| 230 | + } |
| 231 | + |
| 232 | + // Deletion succeeded |
| 233 | + framework.Logf("Security group %s successfully deleted", sgID) |
| 234 | + return true, nil |
| 235 | + }) |
| 236 | +} |
0 commit comments