✨ edge subnets/API: support edge subnets for Local Zones

This change introduce support of required network components to deploy
subnets on AWS Local Zones infrastructure.

The SubnetSpec API is introducing the field ZoneType and ParentZoneName
to handle the zone information for the subnet, discovered when
reconciling the subnet.

✨ edge subnets/API/gen: introduce edge subnets for Local Zones

Generate API changes to suppoer edge subnets for Local Zones.

✨ edge subnets/API/test: added unit to Local Zones

Testing new methods and workflow added to the API to
SubnetSpec (zone information).

✨ edge subnets/docs: added guide subnets on Local and Wavelength zones

Create a dedicated document, "topic", with instructions to deploy
network infrastructure (subnets, gateways and route tables) in "edge
zones" - Local Zone and Wavelength Zone infrastructure.

Author: mtulio

api/v1beta1/awscluster_conversion.go

@@ -104,14 +104,19 @@ func (src *AWSCluster) ConvertTo(dstRaw conversion.Hub) error {
 	dst.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup = restored.Spec.NetworkSpec.VPC.EmptyRoutesDefaultVPCSecurityGroup
 	dst.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch = restored.Spec.NetworkSpec.VPC.PrivateDNSHostnameTypeOnLaunch
 
-	// Restore SubnetSpec.ResourceID field, if any.
+	// Restore SubnetSpec.ResourceID, SubnetSpec.ParentZoneName, and SubnetSpec.ZoneType fields, if any.
 	for _, subnet := range restored.Spec.NetworkSpec.Subnets {
-		if len(subnet.ResourceID) == 0 {
-			continue
-		}
 		for i, dstSubnet := range dst.Spec.NetworkSpec.Subnets {
 			if dstSubnet.ID == subnet.ID {
-				dstSubnet.ResourceID = subnet.ResourceID
+				if len(subnet.ResourceID) > 0 {
+					dstSubnet.ResourceID = subnet.ResourceID
+				}
+				if subnet.ParentZoneName != nil {
+					dstSubnet.ParentZoneName = subnet.ParentZoneName
+				}
+				if subnet.ZoneType != nil {
+					dstSubnet.ZoneType = subnet.ZoneType
+				}
 				dstSubnet.DeepCopyInto(&dst.Spec.NetworkSpec.Subnets[i])
 			}
 		}

api/v1beta1/zz_generated.conversion.go

@@ -248,6 +248,11 @@ func (r *AWSCluster) validateNetwork() field.ErrorList {
 		if subnet.IsIPv6 || subnet.IPv6CidrBlock != "" {
 			allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "IPv6 cannot be used with unmanaged clusters at this time."))
 		}
+		if subnet.ZoneType != nil && subnet.IsEdge() {
+			if subnet.ParentZoneName == nil {
+				allErrs = append(allErrs, field.Invalid(field.NewPath("subnets"), r.Spec.NetworkSpec.Subnets, "ParentZoneName must be set when ZoneType is 'local-zone'."))
+			}
+		}
 	}
 
 	if r.Spec.NetworkSpec.VPC.CidrBlock != "" && r.Spec.NetworkSpec.VPC.IPAMPool != nil {

api/v1beta2/awscluster_webhook.go

@@ -20,6 +20,10 @@ import (
 	"fmt"
 	"sort"
 	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"k8s.io/utils/ptr"
 )
 
 const (
@@ -37,6 +41,11 @@ const (
 	DefaultAPIServerHealthThresholdCount = 5
 	// DefaultAPIServerUnhealthThresholdCount the API server unhealthy check threshold count.
 	DefaultAPIServerUnhealthThresholdCount = 3
+
+	// ZoneTypeAvailabilityZone defines the regular AWS zones in the Region.
+	ZoneTypeAvailabilityZone ZoneType = "availability-zone"
+	// ZoneTypeLocalZone defines the AWS zone type in Local Zone infrastructure.
+	ZoneTypeLocalZone ZoneType = "local-zone"
 )
 
 // NetworkStatus encapsulates AWS networking resources.
@@ -508,6 +517,39 @@ type SubnetSpec struct {
 
 	// Tags is a collection of tags describing the resource.
 	Tags Tags `json:"tags,omitempty"`
+
+	// ZoneType defines the type of the zone where the subnet is created.
+	//
+	// The valid values are availability-zone, and local-zone.
+	//
+	// Subnet with zone type availability-zone (regular) is always selected to create cluster
+	// resources, like Load Balancers, NAT Gateways, Contol Plane nodes, etc.
+	//
+	// Subnet with zone type local-zone is not eligible to automatically create
+	// regular cluster resources.
+	//
+	// The public subnet in availability-zone or local-zone is associated with regular public
+	// route table with default route entry to a Internet Gateway.
+	//
+	// The private subnet in the availability-zone is associated with a private route table with
+	// the default route entry to a NAT Gateway created in that zone.
+	//
+	// The private subnet in the local-zone is associated with a private route table with
+	// the default route entry re-using the NAT Gateway in the Region (preferred from the
+	// parent zone, the zone type availability-zone in the region, or first table available).
+	//
+	// +kubebuilder:validation:Enum=availability-zone;local-zone
+	// +optional
+	ZoneType *ZoneType `json:"zoneType,omitempty"`
+
+	// ParentZoneName is the zone name where the current subnet's zone is tied when
+	// the zone is a Local Zone.
+	//
+	// The subnets in Local Zone locations consume the ParentZoneName to determine the correct
+	// private route table to egress traffic to the internet.
+	//
+	// +optional
+	ParentZoneName *string `json:"parentZoneName,omitempty"`
 }
 
 // GetResourceID returns the identifier for this subnet,
@@ -524,6 +566,39 @@ func (s *SubnetSpec) String() string {
 	return fmt.Sprintf("id=%s/az=%s/public=%v", s.GetResourceID(), s.AvailabilityZone, s.IsPublic)
 }
 
+// IsEdge returns the true when the subnet is created in the edge zone,
+// Local Zones.
+func (s *SubnetSpec) IsEdge() bool {
+	return s.ZoneType != nil && *s.ZoneType == ZoneTypeLocalZone
+}
+
+// SetZoneInfo updates the subnets with zone information.
+func (s *SubnetSpec) SetZoneInfo(zones []*ec2.AvailabilityZone) error {
+	zoneInfo := func(zoneName string) *ec2.AvailabilityZone {
+		for _, zone := range zones {
+			if aws.StringValue(zone.ZoneName) == zoneName {
+				return zone
+			}
+		}
+		return nil
+	}
+
+	zone := zoneInfo(s.AvailabilityZone)
+	if zone == nil {
+		if len(s.AvailabilityZone) > 0 {
+			return fmt.Errorf("unable to update zone information for subnet '%v' and zone '%v'", s.ID, s.AvailabilityZone)
+		}
+		return fmt.Errorf("unable to update zone information for subnet '%v'", s.ID)
+	}
+	if zone.ZoneType != nil {
+		s.ZoneType = ptr.To(ZoneType(*zone.ZoneType))
+	}
+	if zone.ParentZoneName != nil {
+		s.ParentZoneName = zone.ParentZoneName
+	}
+	return nil
+}
+
 // Subnets is a slice of Subnet.
 // +listType=map
 // +listMapKey=id
@@ -541,6 +616,22 @@ func (s Subnets) ToMap() map[string]*SubnetSpec {
 
 // IDs returns a slice of the subnet ids.
 func (s Subnets) IDs() []string {
+	res := []string{}
+	for _, subnet := range s {
+		// Prevent returning edge zones (Local Zone) to regular Subnet IDs.
+		// Edge zones should not deploy control plane nodes, and does not support Nat Gateway and
+		// Network Load Balancers. Any resource for the core infrastructure should not consume edge
+		// zones.
+		if subnet.IsEdge() {
+			continue
+		}
+		res = append(res, subnet.GetResourceID())
+	}
+	return res
+}
+
+// IDsWithEdge returns a slice of the subnet ids.
+func (s Subnets) IDsWithEdge() []string {
 	res := []string{}
 	for _, subnet := range s {
 		res = append(res, subnet.GetResourceID())
@@ -581,21 +672,29 @@ func (s Subnets) FindEqual(spec *SubnetSpec) *SubnetSpec {
 // FilterPrivate returns a slice containing all subnets marked as private.
 func (s Subnets) FilterPrivate() (res Subnets) {
 	for _, x := range s {
+		// Subnets in AWS Local Zones or Wavelength should not be used by core infrastructure.
+		if x.IsEdge() {
+			continue
+		}
 		if !x.IsPublic {
 			res = append(res, x)
 		}
 	}
-	return
+	return res
 }
 
 // FilterPublic returns a slice containing all subnets marked as public.
 func (s Subnets) FilterPublic() (res Subnets) {
 	for _, x := range s {
+		// Subnets in AWS Local Zones or Wavelength should not be used by core infrastructure.
+		if x.IsEdge() {
+			continue
+		}
 		if x.IsPublic {
 			res = append(res, x)
 		}
 	}
-	return
+	return res
 }
 
 // FilterByZone returns a slice containing all subnets that live in the availability zone specified.
@@ -605,22 +704,32 @@ func (s Subnets) FilterByZone(zone string) (res Subnets) {
 			res = append(res, x)
 		}
 	}
-	return
+	return res
 }
 
 // GetUniqueZones returns a slice containing the unique zones of the subnets.
 func (s Subnets) GetUniqueZones() []string {
 	keys := make(map[string]bool)
 	zones := []string{}
 	for _, x := range s {
-		if _, value := keys[x.AvailabilityZone]; !value {
+		if _, value := keys[x.AvailabilityZone]; len(x.AvailabilityZone) > 0 && !value {
 			keys[x.AvailabilityZone] = true
 			zones = append(zones, x.AvailabilityZone)
 		}
 	}
 	return zones
 }
 
+// SetZoneInfo updates the subnets with zone information.
+func (s Subnets) SetZoneInfo(zones []*ec2.AvailabilityZone) error {
+	for i := range s {
+		if err := s[i].SetZoneInfo(zones); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 // CNISpec defines configuration for CNI.
 type CNISpec struct {
 	// CNIIngressRules specify rules to apply to control plane and worker node security groups.
@@ -835,3 +944,11 @@ func (i *IngressRule) Equals(o *IngressRule) bool {
 
 	return true
 }
+
+// ZoneType defines listener AWS Availability Zone type.
+type ZoneType string
+
+// String returns the string representation for the zone type.
+func (z ZoneType) String() string {
+	return string(z)
+}