✨ edge subnets: support Local Zones provisioning networks

Introducing the mechanism to query the zone information
from the subnet's AvailabilityZone, saving the ZoneType and
the ParentZoneName in the SubnetSpec, both for managed and unmanaged.

The ZoneType is used to group the zones from regular and the edge zones.
Regular zones are with type 'availability-zone', and the edge zones are
types 'local-zone' and 'wavelength-zone'.

The following statements are valid for edge subnets:
- private subnets supports egress traffic only using NAT Gateway in the
  region.
- IPv6 subnets is not supported in edge zones
- subnet tags (kubernetes.io/role/*) for load balancer are not set in
  edge subnets. Edge subnets should not be elected by CCM to create
  service load balancers. Use ALB ingress instead.

✨ edge subnets/test: unit for subnets in Local Zones

Added unit tests to validate scenarios suing managed and unmanaged
subnets in AWS Local Zones, alongside new describe availability zones
API calls introduced in the subnet reconciliation loop.

✨ edge subnets/unit: fixes unit tests to describe zone calls

The edge subnets feature introduce a new AWS API call to describe zones,
DescribeAvailabilityZonesWithContext, to lookup zone attributes based in
the zone names in the reconciliator, and the create subnets.

The two new calls is required to support unmanaged subnets (BYO VPC),
where the method createSubnet() is not called.

There are some unit tests calling the create subnet flow, this change
add the mock calls for those calls.

Author: mtulio

controllers/awscluster_controller_test.go

@@ -76,6 +76,8 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) {
 			mockedVPCCallsForExistingVPCAndSubnets(m)
 			mockedCreateSGCalls(false, "vpc-exists", m)
 			mockedDescribeInstanceCall(m)
+			mockedDescribeAvailabilityZones(m, []string{"us-east-1c", "us-east-1a"})
+
 			// Second iteration: the AWS Cluster object has been patched,
 			// thus a valid Control Plane Endpoint has been provided
 			mockedVPCCallsForExistingVPCAndSubnets(m)
@@ -189,7 +191,9 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) {
 			mockedCreateSGCalls(false, "vpc-exists", m)
 			mockedCreateLBCalls(t, e)
 			mockedDescribeInstanceCall(m)
+			mockedDescribeAvailabilityZones(m, []string{"us-east-1c", "us-east-1a"})
 		}
+
 		expect(ec2Mock.EXPECT(), elbMock.EXPECT())
 
 		setup(t)
@@ -298,7 +302,9 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) {
 			mockedCreateSGCalls(true, "vpc-exists", m)
 			mockedCreateLBV2Calls(t, e)
 			mockedDescribeInstanceCall(m)
+			mockedDescribeAvailabilityZones(m, []string{"us-east-1c", "us-east-1a"})
 		}
+
 		expect(ec2Mock.EXPECT(), elbv2Mock.EXPECT())
 
 		g.Expect(testEnv.Create(ctx, &awsCluster)).To(Succeed())
@@ -384,7 +390,9 @@ func TestAWSClusterReconcilerIntegrationTests(t *testing.T) {
 			mockedCallsForMissingEverything(m, e, "my-managed-subnet-priv", "my-managed-subnet-pub")
 			mockedCreateSGCalls(false, "vpc-new", m)
 			mockedDescribeInstanceCall(m)
+			mockedDescribeAvailabilityZones(m, []string{"us-east-1a"})
 		}
+
 		expect(ec2Mock.EXPECT(), elbMock.EXPECT())
 
 		setup(t)
@@ -651,6 +659,26 @@ func mockedDeleteSGCalls(m *mocks.MockEC2APIMockRecorder) {
 	m.DescribeSecurityGroupsPagesWithContext(context.TODO(), gomock.Any(), gomock.Any()).Return(nil)
 }
 
+func mockedDescribeAvailabilityZones(m *mocks.MockEC2APIMockRecorder, zones []string) {
+	output := &ec2.DescribeAvailabilityZonesOutput{}
+	matcher := gomock.Any()
+
+	if len(zones) > 0 {
+		input := &ec2.DescribeAvailabilityZonesInput{}
+		for _, zone := range zones {
+			input.ZoneNames = append(input.ZoneNames, aws.String(zone))
+			output.AvailabilityZones = append(output.AvailabilityZones, &ec2.AvailabilityZone{
+				ZoneName: aws.String(zone),
+				ZoneType: aws.String("availability-zone"),
+			})
+		}
+
+		matcher = gomock.Eq(input)
+	}
+	m.DescribeAvailabilityZonesWithContext(context.TODO(), matcher).AnyTimes().
+		Return(output, nil)
+}
+
 func createControllerIdentity(g *WithT) *infrav1.AWSClusterControllerIdentity {
 	controllerIdentity := &infrav1.AWSClusterControllerIdentity{
 		TypeMeta: metav1.TypeMeta{

controlplane/eks/controllers/awsmanagedcontrolplane_controller_test.go

@@ -309,6 +309,18 @@ func mockedCallsForMissingEverything(ec2Rec *mocks.MockEC2APIMockRecorder, subne
 		Subnets: []*ec2.Subnet{},
 	}, nil)
 
+	zones := []*ec2.AvailabilityZone{}
+	for _, subnet := range subnets {
+		zones = append(zones, &ec2.AvailabilityZone{
+			ZoneName: aws.String(subnet.AvailabilityZone),
+			ZoneType: aws.String("availability-zone"),
+		})
+	}
+	ec2Rec.DescribeAvailabilityZonesWithContext(context.TODO(), gomock.Any()).
+		Return(&ec2.DescribeAvailabilityZonesOutput{
+			AvailabilityZones: zones,
+		}, nil).MaxTimes(2)
+
 	for subnetIndex, subnet := range subnets {
 		subnetID := fmt.Sprintf("subnet-%d", subnetIndex+1)
 		var kubernetesRoleTagKey string
@@ -320,6 +332,17 @@ func mockedCallsForMissingEverything(ec2Rec *mocks.MockEC2APIMockRecorder, subne
 			kubernetesRoleTagKey = "kubernetes.io/role/internal-elb"
 			capaRoleTagValue = "private"
 		}
+		ec2Rec.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{
+			ZoneNames: aws.StringSlice([]string{subnet.AvailabilityZone}),
+		}).
+			Return(&ec2.DescribeAvailabilityZonesOutput{
+				AvailabilityZones: []*ec2.AvailabilityZone{
+					{
+						ZoneName: aws.String(subnet.AvailabilityZone),
+						ZoneType: aws.String("availability-zone"),
+					},
+				},
+			}, nil).MaxTimes(1)
 		ec2Rec.CreateSubnetWithContext(context.TODO(), gomock.Eq(&ec2.CreateSubnetInput{
 			VpcId:            aws.String("vpc-new"),
 			CidrBlock:        aws.String(subnet.CidrBlock),

pkg/cloud/services/network/subnets.go

@@ -53,7 +53,6 @@ func (s *Service) reconcileSubnets() error {
 	defer func() {
 		s.scope.SetSubnets(subnets)
 	}()
-
 	var (
 		err      error
 		existing infrav1.Subnets
@@ -145,7 +144,7 @@ func (s *Service) reconcileSubnets() error {
 			// Make sure tags are up-to-date.
 			subnetTags := sub.Tags
 			if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
-				buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.GetResourceID(), existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags)
+				buildParams := s.getSubnetTagParams(unmanagedVPC, existingSubnet.GetResourceID(), existingSubnet.IsPublic, existingSubnet.AvailabilityZone, subnetTags, existingSubnet.IsEdge())
 				tagsBuilder := tags.New(&buildParams, tags.WithEC2(s.EC2Client))
 				if err := tagsBuilder.Ensure(existingSubnet.Tags); err != nil {
 					return false, err
@@ -158,7 +157,7 @@ func (s *Service) reconcileSubnets() error {
 				}
 
 				// We may not have a permission to tag unmanaged subnets.
-				// When tagging unmanaged subnet fails, record an event and proceed.
+				// When tagging unmanaged subnet fails, record an event and continue checking subnets.
 				record.Warnf(s.scope.InfraCluster(), "FailedTagSubnet", "Failed tagging unmanaged Subnet %q: %v", existingSubnet.GetResourceID(), err)
 				continue
 			}
@@ -175,6 +174,14 @@ func (s *Service) reconcileSubnets() error {
 		return errors.New("expected at least 1 subnet but got 0")
 	}
 
+	// Reconciling the zone information for the subnets. Subnets are grouped
+	// by regular zones (availability zones) or edge zones (local zones or wavelength zones)
+	// based in the zone-type attribute for zone.
+	if err := s.reconcileZoneInfo(subnets); err != nil {
+		record.Warnf(s.scope.InfraCluster(), "FailedNoZoneInfo", "Expected the zone attributes to be populated to subnet")
+		return errors.Wrapf(err, "expected the zone attributes to be populated to subnet")
+	}
+
 	// When the VPC is managed by CAPA, we need to create the subnets.
 	if !unmanagedVPC {
 		// Check that we need at least 1 private and 1 public subnet after we have updated the metadata
@@ -209,6 +216,35 @@ func (s *Service) reconcileSubnets() error {
 	return nil
 }
 
+func (s *Service) retrieveZoneInfo(zoneNames []string) ([]*ec2.AvailabilityZone, error) {
+	zones, err := s.EC2Client.DescribeAvailabilityZonesWithContext(context.TODO(), &ec2.DescribeAvailabilityZonesInput{
+		ZoneNames: aws.StringSlice(zoneNames),
+	})
+	if err != nil {
+		record.Eventf(s.scope.InfraCluster(), "FailedDescribeAvailableZones", "Failed getting available zones: %v", err)
+		return nil, errors.Wrap(err, "failed to describe availability zones")
+	}
+
+	return zones.AvailabilityZones, nil
+}
+
+// reconcileZoneInfo discover the zones for all subnets, and retrieve
+// persist the zone information from resource API, such as Type and
+// Parent Zone.
+func (s *Service) reconcileZoneInfo(subnets infrav1.Subnets) error {
+	if len(subnets) > 0 {
+		zones, err := s.retrieveZoneInfo(subnets.GetUniqueZones())
+		if err != nil {
+			return err
+		}
+		// Extract zone attributes from resource API for each subnet.
+		if err := subnets.SetZoneInfo(zones); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (s *Service) getDefaultSubnets() (infrav1.Subnets, error) {
 	zones, err := s.getAvailableZones()
 	if err != nil {
@@ -418,6 +454,26 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err
 		sn.Tags["Name"] = sn.ID
 	}
 
+	// Retrieve zone information used later to change the zone attributes.
+	if len(sn.AvailabilityZone) > 0 {
+		zones, err := s.retrieveZoneInfo([]string{sn.AvailabilityZone})
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to discover zone information for subnet's zone %q", sn.AvailabilityZone)
+		}
+		if err = sn.SetZoneInfo(zones); err != nil {
+			return nil, errors.Wrapf(err, "failed to update zone information for subnet's zone %q", sn.AvailabilityZone)
+		}
+	}
+
+	// IPv6 subnets are not generally supported by AWS Local Zones and Wavelength Zones.
+	// Local Zones have limited zone support for IPv6 subnets:
+	// https://docs.aws.amazon.com/local-zones/latest/ug/how-local-zones-work.html#considerations
+	if sn.IsIPv6 && sn.IsEdge() {
+		err := fmt.Errorf("failed to create subnet: IPv6 is not supported with zone type %q", sn.ZoneType)
+		record.Warnf(s.scope.InfraCluster(), "FailedCreateSubnet", "Failed creating managed Subnet for edge zones: %v", err)
+		return nil, err
+	}
+
 	// Build the subnet creation request.
 	input := &ec2.CreateSubnetInput{
 		VpcId:            aws.String(s.scope.VPC().ID),
@@ -426,7 +482,7 @@ func (s *Service) createSubnet(sn *infrav1.SubnetSpec) (*infrav1.SubnetSpec, err
 		TagSpecifications: []*ec2.TagSpecification{
 			tags.BuildParamsToTagSpecification(
 				ec2.ResourceTypeSubnet,
-				s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags),
+				s.getSubnetTagParams(false, services.TemporaryResourceID, sn.IsPublic, sn.AvailabilityZone, sn.Tags, sn.IsEdge()),
 			),
 		},
 	}
@@ -544,7 +600,7 @@ func (s *Service) deleteSubnet(id string) error {
 	return nil
 }
 
-func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, zone string, manualTags infrav1.Tags) infrav1.BuildParams {
+func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool, zone string, manualTags infrav1.Tags, isEdge bool) infrav1.BuildParams {
 	var role string
 	additionalTags := make(map[string]string)
 
@@ -553,12 +609,16 @@ func (s *Service) getSubnetTagParams(unmanagedVPC bool, id string, public bool,
 
 		if public {
 			role = infrav1.PublicRoleTagValue
-			additionalTags[externalLoadBalancerTag] = "1"
+			// Edge subnets should not have ELB tags to be selected by CCM to create load balancers.
+			if !isEdge {
+				additionalTags[externalLoadBalancerTag] = "1"
+			}
 		} else {
 			role = infrav1.PrivateRoleTagValue
-			additionalTags[internalLoadBalancerTag] = "1"
+			if !isEdge {
+				additionalTags[internalLoadBalancerTag] = "1"
+			}
 		}
-
 		// Add tag needed for Service type=LoadBalancer
 		additionalTags[infrav1.ClusterAWSCloudProviderTagKey(s.scope.KubernetesClusterName())] = string(infrav1.ResourceLifecycleShared)
 	}