Prefix on NIC v6 support

Author: kadevu

azure-ipam/ipam.go

@@ -115,7 +115,7 @@ func (p *IPAMPlugin) CmdAdd(args *cniSkel.CmdArgs) error {
 	p.logger.Debug("Received CNS IP config response", zap.Any("response", resp))
 
 	// Get Pod IP and gateway IP from ip config response
-	podIPNet, err := ipconfig.ProcessIPConfigsResp(resp)
+	podIPNet, gatewayIP, err := ipconfig.ProcessIPConfigsResp(resp)
 	if err != nil {
 		p.logger.Error("Failed to interpret CNS IPConfigResponse", zap.Error(err), zap.Any("response", resp))
 		return cniTypes.NewError(ErrProcessIPConfigResponse, err.Error(), "failed to interpret CNS IPConfigResponse")
@@ -136,9 +136,38 @@ func (p *IPAMPlugin) CmdAdd(args *cniSkel.CmdArgs) error {
 				Mask: net.CIDRMask(ipNet.Bits(), 128), // nolint
 			}
 		}
+		ipConfig.Gateway = (*gatewayIP)[i]
 		cniResult.IPs[i] = ipConfig
 	}
 
+	cniResult.Interfaces = []*types100.Interface{}
+	seenInterfaces := map[string]bool{}
+
+	for _, podIPInfo := range resp.PodIPInfo {
+		if podIPInfo.MacAddress == "" {
+			continue
+		}
+
+		// Skip if interface already seen
+		// This is to avoid duplicate interfaces in the result
+		// which can happen if multiple IPs are assigned to the same interface
+		// or if multiple interfaces are assigned to the same pod
+		if seenInterfaces[podIPInfo.MacAddress] {
+			continue
+		}
+
+		infMac, err := net.ParseMAC(podIPInfo.MacAddress)
+		if err != nil {
+			p.logger.Error("Failed to parse interface MAC address", zap.Error(err), zap.String("macAddress", podIPInfo.MacAddress))
+			return cniTypes.NewError(cniTypes.ErrUnsupportedField, err.Error(), "failed to parse interface MAC address")
+		}
+
+		cniResult.Interfaces = append(cniResult.Interfaces, &types100.Interface{
+			Mac: infMac.String(),
+		})
+		seenInterfaces[podIPInfo.MacAddress] = true
+	}
+
 	// Get versioned result
 	versionedCniResult, err := cniResult.GetAsVersion(nwCfg.CNIVersion)
 	if err != nil {

azure-ipam/ipconfig/ipconfig.go

@@ -3,6 +3,7 @@ package ipconfig
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/netip"
 
 	"github.com/Azure/azure-container-networking/cns"
@@ -63,23 +64,36 @@ func CreateIPConfigsReq(args *cniSkel.CmdArgs) (cns.IPConfigsRequest, error) {
 	return req, nil
 }
 
-func ProcessIPConfigsResp(resp *cns.IPConfigsResponse) (*[]netip.Prefix, error) {
+func ProcessIPConfigsResp(resp *cns.IPConfigsResponse) (*[]netip.Prefix, *[]net.IP, error) {
 	podIPNets := make([]netip.Prefix, len(resp.PodIPInfo))
+	gatewaysIPs := make([]net.IP, len(resp.PodIPInfo))
 
 	for i := range resp.PodIPInfo {
+		var gatewayIP net.IP
+
 		podCIDR := fmt.Sprintf(
 			"%s/%d",
 			resp.PodIPInfo[i].PodIPConfig.IPAddress,
 			resp.PodIPInfo[i].NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength,
 		)
 		podIPNet, err := netip.ParsePrefix(podCIDR)
 		if err != nil {
-			return nil, errors.Wrapf(err, "cns returned invalid pod CIDR %q", podCIDR)
+			return nil, nil, errors.Wrapf(err, "cns returned invalid pod CIDR %q", podCIDR)
 		}
 		podIPNets[i] = podIPNet
+
+		if podIPNet.Addr().Is4() {
+			gatewayIP = net.ParseIP(resp.PodIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPAddress)
+		} else if podIPNet.Addr().Is6() {
+			gatewayIP = net.ParseIP(resp.PodIPInfo[i].NetworkContainerPrimaryIPConfig.GatewayIPv6Address)
+		}
+
+		if gatewayIP != nil {
+			gatewaysIPs[i] = gatewayIP
+		}
 	}
 
-	return &podIPNets, nil
+	return &podIPNets, &gatewaysIPs, nil
 }
 
 type k8sPodEnvArgs struct {

cns/NetworkContainerContract.go

@@ -389,9 +389,10 @@ type NetworkInterfaceInfo struct {
 
 // IPConfiguration contains details about ip config to provision in the VM.
 type IPConfiguration struct {
-	IPSubnet         IPSubnet
-	DNSServers       []string
-	GatewayIPAddress string
+	IPSubnet           IPSubnet
+	DNSServers         []string
+	GatewayIPAddress   string
+	GatewayIPv6Address string
 }
 
 // SecondaryIPConfig contains IP info of SecondaryIP
@@ -746,3 +747,11 @@ type NodeRegisterRequest struct {
 	NumCores             int
 	NmAgentSupportedApis []string
 }
+
+// IPFamily - Enum for determining IPFamily when retrieving IPs from network containers
+type IPFamily string
+
+const (
+	IPv4Family IPFamily = "ipv4"
+	IPv6Family IPFamily = "ipv6"
+)

cns/kubecontroller/nodenetworkconfig/conversion_linux.go

@@ -16,12 +16,15 @@ import (
 func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) {
 	secondaryIPConfigs := map[string]cns.SecondaryIPConfig{}
 
-	// iterate through all IP addresses in the subnet described by primaryPrefix and
-	// add them to the request as secondary IPConfigs.
-	for addr := primaryIPPrefix.Masked().Addr(); primaryIPPrefix.Contains(addr); addr = addr.Next() {
-		secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{
-			IPAddress: addr.String(),
-			NCVersion: int(nc.Version),
+	// in the case of vnet prefix on swift v2 the primary IP is a /32 and should not be added to secondary IP configs
+	if !primaryIPPrefix.IsSingleIP() {
+		// iterate through all IP addresses in the subnet described by primaryPrefix and
+		// add them to the request as secondary IPConfigs.
+		for addr := primaryIPPrefix.Masked().Addr(); primaryIPPrefix.Contains(addr); addr = addr.Next() {
+			secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{
+				IPAddress: addr.String(),
+				NCVersion: int(nc.Version),
+			}
 		}
 	}
 
@@ -52,9 +55,13 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre
 		NetworkContainerType: cns.Docker,
 		Version:              strconv.FormatInt(nc.Version, 10), //nolint:gomnd // it's decimal
 		IPConfiguration: cns.IPConfiguration{
-			IPSubnet:         subnet,
-			GatewayIPAddress: nc.DefaultGateway,
+			IPSubnet:           subnet,
+			GatewayIPAddress:   nc.DefaultGateway,
+			GatewayIPv6Address: nc.DefaultGatewayV6,
 		},
 		NCStatus: nc.Status,
+		NetworkInterfaceInfo: cns.NetworkInterfaceInfo{
+			MACAddress: nc.MacAddress,
+		},
 	}, nil
 }

cns/restserver/internalapi_linux.go

@@ -68,6 +68,12 @@ func (service *HTTPRestService) programSNATRules(req *cns.CreateNetworkContainer
 
 	// use any secondary ip + the nnc prefix length to get an iptables rule to allow dns and imds traffic from the pods
 	for _, v := range req.SecondaryIPConfigs {
+		// check if the ip address is IPv4
+		if net.ParseIP(v.IPAddress).To4() == nil {
+			// skip if the ip address is not IPv4
+			continue
+		}
+
 		// put the ip address in standard cidr form (where we zero out the parts that are not relevant)
 		_, podSubnet, _ := net.ParseCIDR(v.IPAddress + "/" + fmt.Sprintf("%d", req.IPConfiguration.IPSubnet.PrefixLength))
 

cns/restserver/ipam.go

@@ -995,38 +995,87 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([
 	if numOfNCs == 0 {
 		return nil, ErrNoNCs
 	}
+
+	//  Map used to get the number of IPFamilies across all NCs
+	ipFamilies := map[cns.IPFamily]struct{}{}
+
+	// Gets the IPFamilies from all NCs and store them in a map. This will be used to determine the number of IPs to return
+	for ncID := range service.state.ContainerStatus {
+		if len(ipFamilies) == 2 {
+			break
+		}
+
+		for _, secIPConfig := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.SecondaryIPConfigs {
+			if len(ipFamilies) == 2 {
+				break
+			}
+
+			ip := net.ParseIP(secIPConfig.IPAddress)
+			if ip == nil {
+				continue
+			}
+
+			if ip.To4() != nil {
+				ipFamilies[cns.IPv4Family] = struct{}{}
+			} else {
+				ipFamilies[cns.IPv6Family] = struct{}{}
+			}
+		}
+	}
+
+	// Makes sure we have at least one IPFamily across all NCs
+	numOfIPFamilies := len(ipFamilies)
+
+	numberOfIPs := numOfNCs
+	if numOfIPFamilies != 0 {
+		numberOfIPs = numOfIPFamilies
+	}
+
 	service.Lock()
 	defer service.Unlock()
 	// Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs
-	podIPInfo := make([]cns.PodIpInfo, numOfNCs)
+	podIPInfo := make([]cns.PodIpInfo, numberOfIPs)
 	// This map is used to store whether or not we have found an available IP from an NC when looping through the pool
 	ipsToAssign := make(map[string]cns.IPConfigurationStatus)
 
 	// Searches for available IPs in the pool
 	for _, ipState := range service.PodIPConfigState {
-		// check if an IP from this NC is already set side for assignment.
-		if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment {
+
+		// get the IPFamily of the current ipState
+		var ipStateFamily cns.IPFamily
+		if net.ParseIP(ipState.IPAddress).To4() != nil {
+			ipStateFamily = cns.IPv4Family
+		} else {
+			ipStateFamily = cns.IPv6Family
+		}
+
+		key := generateAssignedIPKey(ipState.NCID, ipStateFamily)
+
+		// check if the IP with the same family type exists already
+		if _, ncIPFamilyAlreadyMarkedForAssignment := ipsToAssign[key]; ncIPFamilyAlreadyMarkedForAssignment {
 			continue
 		}
 		// Checks if the current IP is available
 		if ipState.GetState() != types.Available {
 			continue
 		}
-		ipsToAssign[ipState.NCID] = ipState
-		// Once one IP per container is found break out of the loop and stop searching
-		if len(ipsToAssign) == numOfNCs {
+		ipsToAssign[key] = ipState
+		// Once numberOfIPs per container is found break out of the loop and stop searching
+		if len(ipsToAssign) == numberOfIPs {
 			break
 		}
 	}
 
-	// Checks to make sure we found one IP for each NC
-	if len(ipsToAssign) != numOfNCs {
+	// Checks to make sure we found one IP for each NCxIPFamily
+	if len(ipsToAssign) != numberOfIPs {
 		for ncID := range service.state.ContainerStatus {
-			if _, found := ipsToAssign[ncID]; found {
-				continue
+			for ipFamily := range ipFamilies {
+				if _, found := ipsToAssign[generateAssignedIPKey(ncID, ipFamily)]; found {
+					continue
+				}
+				return podIPInfo, errors.Errorf("not enough IPs available of type %s for %s, waiting on Azure CNS to allocate more with NC Status: %s",
+					ipFamily, ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 			}
-			return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s",
-				ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus))
 		}
 	}
 
@@ -1061,10 +1110,14 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([
 		return podIPInfo, fmt.Errorf("not enough IPs available, waiting on Azure CNS to allocate more")
 	}
 
-	logger.Printf("[AssignDesiredIPConfigs] Successfully assigned IPs for pod %+v", podInfo)
+	logger.Printf("[AssignAvailableIPConfigs] Successfully assigned IPs for pod %+v", podInfo)
 	return podIPInfo, nil
 }
 
+func generateAssignedIPKey(ncID string, ipFamily cns.IPFamily) string {
+	return fmt.Sprintf("%s_%s", ncID, string(ipFamily))
+}
+
 // If IPConfigs are already assigned to the pod, it returns that else it returns the available ipconfigs.
 func requestIPConfigsHelper(service *HTTPRestService, req cns.IPConfigsRequest) ([]cns.PodIpInfo, error) {
 	// check if ipconfigs already assigned to this pod and return if exists or error