[WIP] Keith/v6 prefix

cns/NetworkContainerContract.go

@@ -129,6 +129,7 @@ type CreateNetworkContainerRequest struct {
 	EndpointPolicies           []NetworkContainerRequestPolicies
 	NCStatus                   v1alpha.NCStatus
 	NetworkInterfaceInfo       NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code
+	IPFamilies                 map[IPFamily]struct{}
 }
 
 func (req *CreateNetworkContainerRequest) Validate() error {
@@ -746,3 +747,10 @@ type NodeRegisterRequest struct {
 	NumCores             int
 	NmAgentSupportedApis []string
 }
+
+type IPFamily string
+
+const (
+	IPv4Family IPFamily = "ipv4"
+	IPv6Family IPFamily = "ipv6"
+)

cns/kubecontroller/nodenetworkconfig/conversion_linux.go

@@ -1,6 +1,7 @@
 package nodenetworkconfig
 
 import (
+	"fmt"
 	"net/netip"
 	"strconv"
 
@@ -15,6 +16,7 @@ import (
 //nolint:gocritic //ignore hugeparam
 func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) {
 	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
+	ipFamilies := map[cns.IPFamily]struct{}{}
 
 	// iterate through all IP addresses in the subnet described by primaryPrefix and
 	// add them to the request as secondary IPConfigs.
@@ -23,12 +25,20 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPAddress: addr.String(),
 			NCVersion: int(nc.Version),
 		}
+
+	}
+	// adds the IPFamily of the primary CIDR to the set
+	if primaryIPPrefix.Addr().Is4() {
+		ipFamilies[cns.IPv4Family] = struct{}{}
+	} else {
+		ipFamilies[cns.IPv6Family] = struct{}{}
 	}
 
 	// Add IPs from CIDR block to the secondary IPConfigs
 	if nc.Type == v1alpha.VNETBlock {
 
 		for _, ipAssignment := range nc.IPAssignments {
+			// Here we would need to check all other assigned CIDR Blocks that aren't the primary.
 			cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP)
 			if err != nil {
 				return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP)
@@ -42,9 +52,17 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 					NCVersion: int(nc.Version),
 				}
 			}
+			// adds the IPFamily of the secondary CIDR to the set
+			if cidrPrefix.Addr().Is4() {
+				ipFamilies[cns.IPv4Family] = struct{}{}
+			} else {
+				ipFamilies[cns.IPv6Family] = struct{}{}
+			}
 		}
 	}
 
+	fmt.Printf("IPFamilies found on NC %+v are %+v", nc.ID, ipFamilies)
+
 	return &cns.CreateNetworkContainerRequest{
 		HostPrimaryIP:        nc.NodeIP,
 		SecondaryIPConfigs:   secondaryIPConfigs,
@@ -55,6 +73,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
-		NCStatus: nc.Status,
+		NCStatus:   nc.Status,
+		IPFamilies: ipFamilies,
 	}, nil
 }

cns/restserver/ipam.go

@@ -25,6 +25,7 @@ var (
 	ErrStoreEmpty             = errors.New("empty endpoint state store")
 	ErrParsePodIPFailed       = errors.New("failed to parse pod's ip")
 	ErrNoNCs                  = errors.New("no NCs found in the CNS internal state")
+	ErrNoIPFamilies           = errors.New("No IP Families found on NCs")
 	ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state")
 	ErrEndpointStateNotFound  = errors.New("endpoint state could not be found in the statefile")
 	ErrGetAllNCResponseEmpty  = errors.New("failed to get NC responses from statefile")
@@ -989,44 +990,69 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi
 // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return
 // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC
 func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) {
-	// Gets the number of NCs which will determine the number of IPs given to a pod
-	numOfNCs := len(service.state.ContainerStatus)
-	// if there are no NCs on the NNC there will be no IPs in the pool so return error
-	if numOfNCs == 0 {
+	//  Map used to get the number of IPFamilies across all NCs
+	ipFamilies := map[cns.IPFamily]struct{}{}
+
+	// checks to make sure we have at least one NC
+	if len(service.state.ContainerStatus) == 0 {
 		return nil, ErrNoNCs
 	}
+
+	// Gets the IPFamilies from all NCs and stores them in a map. This will be ued to determine the amount of IPs to return
+	for ncID := range service.state.ContainerStatus {
+		for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies {
+			ipFamilies[ipFamily] = struct{}{}
+		}
+	}
+
+	// Makes sure we have at least one IPFamily across all NCs
+	numOfIPFamilies := len(ipFamilies)
+	if numOfIPFamilies == 0 {
+		return nil, ErrNoIPFamilies
+	}
+
 	service.Lock()
 	defer service.Unlock()
 	// Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs
-	podIPInfo := make([]cns.PodIpInfo, numOfNCs)
+	podIPInfo := make([]cns.PodIpInfo, numOfIPFamilies)
 	// This map is used to store whether or not we have found an available IP from an NC when looping through the pool
-	ipsToAssign := make(map[string]cns.IPConfigurationStatus)
+	ipsToAssign := make(map[cns.IPFamily]cns.IPConfigurationStatus)
 
 	// Searches for available IPs in the pool
 	for _, ipState := range service.PodIPConfigState {
-		// check if an IP from this NC is already set side for assignment.
-		if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment {
+		// get the IPFamily of the current ipState
+		var ipStateFamily cns.IPFamily
+		if net.ParseIP(ipState.IPAddress).To4() != nil {
+			ipStateFamily = cns.IPv4Family
+		} else {
+			ipStateFamily = cns.IPv6Family
+		}
+
+		// check if the IP with the same family type exists already
+		if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[ipStateFamily]; IPFamilyAlreadyMarkedForAssignment {
 			continue
 		}
 		// Checks if the current IP is available
 		if ipState.GetState() != types.Available {
 			continue
 		}
-		ipsToAssign[ipState.NCID] = ipState
-		// Once one IP per container is found break out of the loop and stop searching
-		if len(ipsToAssign) == numOfNCs {
+		ipsToAssign[ipStateFamily] = ipState
+		// Once one IP per family is found break out of the loop and stop searching
+		if len(ipsToAssign) == numOfIPFamilies {
 			break
 		}
 	}
 
-	// Checks to make sure we found one IP for each NC
-	if len(ipsToAssign) != numOfNCs {
+	// Checks to make sure we found one IP for each IPFamily
+	if len(ipsToAssign) != numOfIPFamilies {
 		for ncID := range service.state.ContainerStatus {
-			if _, found := ipsToAssign[ncID]; found {
-				continue
+			for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies {
+				if _, found := ipsToAssign[ipFamily]; found {
+					continue
+				}
+				return podIPInfo, errors.Errorf("not enough IPs available of type %s for %s, waiting on Azure CNS to allocate more with NC Status: %s",
+					ipFamily, ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 			}
-			return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s",
-				ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 		}
 	}
 

cns/restserver/restserver.go

@@ -67,6 +67,8 @@ type iptablesGetter interface {
 }
 
 // HTTPRestService represents http listener for CNS - Container Networking Service.
+// TODO: Add a new value for IPFamily
+// If we add a new type of Middleware that will be reflected in the IPConfigsHandlerMiddleware value
 type HTTPRestService struct {
 	*cns.Service
 	dockerClient             *dockerclient.Client
@@ -92,6 +94,7 @@ type HTTPRestService struct {
 	PnpIDByMacAddress          map[string]string
 	imdsClient                 imdsClient
 	nodesubnetIPFetcher        *nodesubnet.IPFetcher
+	IPFamilies                 []cns.IPFamily
 }
 
 type CNIConflistGenerator interface {

cns/restserver/util.go

@@ -168,8 +168,8 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw
 	if hostVersion == "" {
 		// Host version is the NC version from NMAgent, set it -1 to indicate no result from NMAgent yet.
 		// TODO, query NMAgent and with aggresive time out and assign latest host version.
-		hostVersion = "-1"
 	}
+	hostVersion = "1"
 
 	// Remove the auth token before saving the containerStatus to cns json file
 	createNetworkContainerRequest := req

crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go

@@ -119,9 +119,12 @@ type NetworkContainer struct {
 	// +kubebuilder:default=vnet
 	Type               NCType         `json:"type,omitempty"`
 	PrimaryIP          string         `json:"primaryIP,omitempty"`
+	PrimaryIPV6        string         `json:"primaryIPV6,omitempty"`
 	SubnetName         string         `json:"subnetName,omitempty"`
 	IPAssignments      []IPAssignment `json:"ipAssignments,omitempty"`
 	DefaultGateway     string         `json:"defaultGateway,omitempty"`
+	DefaultGatewayV6   string         `json:"defaultGatewayV6,omitempty"`
+	MacAddress         string         `json:"macAddress,omitempty"`
 	SubnetAddressSpace string         `json:"subnetAddressSpace,omitempty"`
 	// +kubebuilder:default=0
 	// +kubebuilder:validation:Optional

crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.16.3
+    controller-gen.kubebuilder.io/version: v0.17.2
   name: nodenetworkconfigs.acn.azure.com
 spec:
   group: acn.azure.com
@@ -109,6 +109,8 @@ spec:
                       type: string
                     defaultGateway:
                       type: string
+                    defaultGatewayV6:
+                      type: string
                     id:
                       type: string
                     ipAssignments:
@@ -122,10 +124,14 @@ spec:
                             type: string
                         type: object
                       type: array
+                    macAddress:
+                      type: string
                     nodeIP:
                       type: string
                     primaryIP:
                       type: string
+                    primaryIPV6:
+                      type: string
                     resourceGroupID:
                       type: string
                     status: