adding changes to CNS to allow for dualstack NCs

Author: rjdenney

cns/NetworkContainerContract.go

@@ -127,6 +127,7 @@ type CreateNetworkContainerRequest struct {
 	EndpointPolicies           []NetworkContainerRequestPolicies
 	NCStatus                   v1alpha.NCStatus
 	NetworkInterfaceInfo       NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code
+	IPFamilies                 map[IPFamily]struct{}
 }
 
 func (req *CreateNetworkContainerRequest) Validate() error {
@@ -742,3 +743,11 @@ type NodeRegisterRequest struct {
 	NumCores             int
 	NmAgentSupportedApis []string
 }
+
+// IPFamily - Enum for determining IPFamily when retrieving IPs from network containers
+type IPFamily string
+
+const (
+	IPv4Family IPFamily = "ipv4"
+	IPv6Family IPFamily = "ipv6"
+)

cns/kubecontroller/nodenetworkconfig/conversion_linux.go

@@ -1,6 +1,7 @@
 package nodenetworkconfig
 
 import (
+	"fmt"
 	"net/netip"
 	"strconv"
 
@@ -15,6 +16,7 @@ import (
 //nolint:gocritic //ignore hugeparam
 func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) {
 	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
+	ipFamilies := map[cns.IPFamily]struct{}{}
 
 	// iterate through all IP addresses in the subnet described by primaryPrefix and
 	// add them to the request as secondary IPConfigs.
@@ -24,11 +26,17 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			NCVersion: int(nc.Version),
 		}
 	}
+	// adds the IPFamily of the primary CIDR to the set
+	if primaryIPPrefix.Addr().Is4() {
+		ipFamilies[cns.IPv4Family] = struct{}{}
+	} else {
+		ipFamilies[cns.IPv6Family] = struct{}{}
+	}
 
 	// Add IPs from CIDR block to the secondary IPConfigs
 	if nc.Type == v1alpha.VNETBlock {
-
 		for _, ipAssignment := range nc.IPAssignments {
+			// Here we would need to check all other assigned CIDR Blocks that aren't the primary.
 			cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP)
 			if err != nil {
 				return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP)
@@ -42,9 +50,17 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 					NCVersion: int(nc.Version),
 				}
 			}
+			// adds the IPFamily of the secondary CIDR to the set
+			if cidrPrefix.Addr().Is4() {
+				ipFamilies[cns.IPv4Family] = struct{}{}
+			} else {
+				ipFamilies[cns.IPv6Family] = struct{}{}
+			}
 		}
 	}
 
+	fmt.Printf("IPFamilies found on NC %+v are %+v", nc.ID, ipFamilies)
+
 	return &cns.CreateNetworkContainerRequest{
 		HostPrimaryIP:        nc.NodeIP,
 		SecondaryIPConfigs:   secondaryIPConfigs,
@@ -55,6 +71,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 			IPSubnet:         subnet,
 			GatewayIPAddress: nc.DefaultGateway,
 		},
-		NCStatus: nc.Status,
+		NCStatus:   nc.Status,
+		IPFamilies: ipFamilies,
 	}, nil
 }

cns/restserver/internalapi_test.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
+	"net/netip"
 	"os"
 	"reflect"
 	"strconv"
@@ -914,11 +915,22 @@ func generateNetworkContainerRequest(secondaryIps map[string]cns.SecondaryIPConf
 	ipSubnet.PrefixLength = subnetPrfixLength
 	ipConfig.IPSubnet = ipSubnet
 
+	ipFamilies := map[cns.IPFamily]struct{}{}
+	for _, secIPConfig := range secondaryIps {
+		IP, _ := netip.ParseAddr(secIPConfig.IPAddress)
+		if IP.Is4() {
+			ipFamilies[cns.IPv4Family] = struct{}{}
+		} else {
+			ipFamilies[cns.IPv6Family] = struct{}{}
+		}
+	}
+
 	req := cns.CreateNetworkContainerRequest{
 		NetworkContainerType: dockerContainerType,
 		NetworkContainerid:   ncID,
 		IPConfiguration:      ipConfig,
 		Version:              ncVersion,
+		IPFamilies:           ipFamilies,
 	}
 
 	ncVersionInInt, _ := strconv.Atoi(ncVersion)

cns/restserver/ipam.go

@@ -25,6 +25,7 @@ var (
 	ErrStoreEmpty             = errors.New("empty endpoint state store")
 	ErrParsePodIPFailed       = errors.New("failed to parse pod's ip")
 	ErrNoNCs                  = errors.New("no NCs found in the CNS internal state")
+	ErrNoIPFamilies           = errors.New("No IP Families found on NCs")
 	ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state")
 	ErrEndpointStateNotFound  = errors.New("endpoint state could not be found in the statefile")
 	ErrGetAllNCResponseEmpty  = errors.New("failed to get NC responses from statefile")
@@ -989,44 +990,69 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi
 // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return
 // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC
 func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) {
-	// Gets the number of NCs which will determine the number of IPs given to a pod
-	numOfNCs := len(service.state.ContainerStatus)
-	// if there are no NCs on the NNC there will be no IPs in the pool so return error
-	if numOfNCs == 0 {
+	//  Map used to get the number of IPFamilies across all NCs
+	ipFamilies := map[cns.IPFamily]struct{}{}
+
+	// checks to make sure we have at least one NC
+	if len(service.state.ContainerStatus) == 0 {
 		return nil, ErrNoNCs
 	}
+
+	// Gets the IPFamilies from all NCs and stores them in a map. This will be ued to determine the amount of IPs to return
+	for ncID := range service.state.ContainerStatus {
+		for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies {
+			ipFamilies[ipFamily] = struct{}{}
+		}
+	}
+
+	// Makes sure we have at least one IPFamily across all NCs
+	numOfIPFamilies := len(ipFamilies)
+	if numOfIPFamilies == 0 {
+		return nil, ErrNoIPFamilies
+	}
+
 	service.Lock()
 	defer service.Unlock()
 	// Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs
-	podIPInfo := make([]cns.PodIpInfo, numOfNCs)
+	podIPInfo := make([]cns.PodIpInfo, numOfIPFamilies)
 	// This map is used to store whether or not we have found an available IP from an NC when looping through the pool
-	ipsToAssign := make(map[string]cns.IPConfigurationStatus)
+	ipsToAssign := make(map[cns.IPFamily]cns.IPConfigurationStatus)
 
 	// Searches for available IPs in the pool
 	for _, ipState := range service.PodIPConfigState {
-		// check if an IP from this NC is already set side for assignment.
-		if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment {
+		// get the IPFamily of the current ipState
+		var ipStateFamily cns.IPFamily
+		if net.ParseIP(ipState.IPAddress).To4() != nil {
+			ipStateFamily = cns.IPv4Family
+		} else {
+			ipStateFamily = cns.IPv6Family
+		}
+
+		// check if the IP with the same family type exists already
+		if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[ipStateFamily]; IPFamilyAlreadyMarkedForAssignment {
 			continue
 		}
 		// Checks if the current IP is available
 		if ipState.GetState() != types.Available {
 			continue
 		}
-		ipsToAssign[ipState.NCID] = ipState
-		// Once one IP per container is found break out of the loop and stop searching
-		if len(ipsToAssign) == numOfNCs {
+		ipsToAssign[ipStateFamily] = ipState
+		// Once one IP per family is found break out of the loop and stop searching
+		if len(ipsToAssign) == numOfIPFamilies {
 			break
 		}
 	}
 
-	// Checks to make sure we found one IP for each NC
-	if len(ipsToAssign) != numOfNCs {
+	// Checks to make sure we found one IP for each IPFamily
+	if len(ipsToAssign) != numOfIPFamilies {
 		for ncID := range service.state.ContainerStatus {
-			if _, found := ipsToAssign[ncID]; found {
-				continue
+			for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies {
+				if _, found := ipsToAssign[ipFamily]; found {
+					continue
+				}
+				return podIPInfo, errors.Errorf("not enough IPs available of type %s for %s, waiting on Azure CNS to allocate more with NC Status: %s",
+					ipFamily, ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 			}
-			return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s",
-				ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 		}
 	}