diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 394f871f09..37cdaa71f7 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -127,6 +127,7 @@ type CreateNetworkContainerRequest struct { EndpointPolicies []NetworkContainerRequestPolicies NCStatus v1alpha.NCStatus NetworkInterfaceInfo NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code + IPFamilies map[IPFamily]struct{} } func (req *CreateNetworkContainerRequest) Validate() error { @@ -742,3 +743,10 @@ type NodeRegisterRequest struct { NumCores int NmAgentSupportedApis []string } + +type IPFamily string + +const ( + IPv4Family IPFamily = "ipv4" + IPv6Family IPFamily = "ipv6" +) diff --git a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go index c89d41646b..4f430df655 100644 --- a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go +++ b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go @@ -1,6 +1,7 @@ package nodenetworkconfig import ( + "fmt" "net/netip" "strconv" @@ -15,6 +16,7 @@ import ( //nolint:gocritic //ignore hugeparam func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) { secondaryIPConfigs := map[string]cns.SecondaryIPConfig{} + ipFamilies := map[cns.IPFamily]struct{}{} // iterate through all IP addresses in the subnet described by primaryPrefix and // add them to the request as secondary IPConfigs. @@ -23,12 +25,20 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre IPAddress: addr.String(), NCVersion: int(nc.Version), } + + } + // adds the IPFamily of the primary CIDR to the set + if primaryIPPrefix.Addr().Is4() { + ipFamilies[cns.IPv4Family] = struct{}{} + } else { + ipFamilies[cns.IPv6Family] = struct{}{} } // Add IPs from CIDR block to the secondary IPConfigs if nc.Type == v1alpha.VNETBlock { for _, ipAssignment := range nc.IPAssignments { + // Here we would need to check all other assigned CIDR Blocks that aren't the primary. cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP) if err != nil { return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP) @@ -42,9 +52,17 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre NCVersion: int(nc.Version), } } + // adds the IPFamily of the secondary CIDR to the set + if cidrPrefix.Addr().Is4() { + ipFamilies[cns.IPv4Family] = struct{}{} + } else { + ipFamilies[cns.IPv6Family] = struct{}{} + } } } + fmt.Printf("IPFamilies found on NC %+v are %+v", nc.ID, ipFamilies) + return &cns.CreateNetworkContainerRequest{ HostPrimaryIP: nc.NodeIP, SecondaryIPConfigs: secondaryIPConfigs, @@ -55,6 +73,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre IPSubnet: subnet, GatewayIPAddress: nc.DefaultGateway, }, - NCStatus: nc.Status, + NCStatus: nc.Status, + IPFamilies: ipFamilies, }, nil } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 883d9aaef3..3873e82d96 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -25,6 +25,7 @@ var ( ErrStoreEmpty = errors.New("empty endpoint state store") ErrParsePodIPFailed = errors.New("failed to parse pod's ip") ErrNoNCs = errors.New("no NCs found in the CNS internal state") + ErrNoIPFamilies = errors.New("No IP Families found on NCs") ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state") ErrEndpointStateNotFound = errors.New("endpoint state could not be found in the statefile") ErrGetAllNCResponseEmpty = errors.New("failed to get NC responses from statefile") @@ -989,44 +990,69 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { - // Gets the number of NCs which will determine the number of IPs given to a pod - numOfNCs := len(service.state.ContainerStatus) - // if there are no NCs on the NNC there will be no IPs in the pool so return error - if numOfNCs == 0 { + // Map used to get the number of IPFamilies across all NCs + ipFamilies := map[cns.IPFamily]struct{}{} + + // checks to make sure we have at least one NC + if len(service.state.ContainerStatus) == 0 { return nil, ErrNoNCs } + + // Gets the IPFamilies from all NCs and stores them in a map. This will be ued to determine the amount of IPs to return + for ncID := range service.state.ContainerStatus { + for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { + ipFamilies[ipFamily] = struct{}{} + } + } + + // Makes sure we have at least one IPFamily across all NCs + numOfIPFamilies := len(ipFamilies) + if numOfIPFamilies == 0 { + return nil, ErrNoIPFamilies + } + service.Lock() defer service.Unlock() // Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs - podIPInfo := make([]cns.PodIpInfo, numOfNCs) + podIPInfo := make([]cns.PodIpInfo, numOfIPFamilies) // This map is used to store whether or not we have found an available IP from an NC when looping through the pool - ipsToAssign := make(map[string]cns.IPConfigurationStatus) + ipsToAssign := make(map[cns.IPFamily]cns.IPConfigurationStatus) // Searches for available IPs in the pool for _, ipState := range service.PodIPConfigState { - // check if an IP from this NC is already set side for assignment. - if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment { + // get the IPFamily of the current ipState + var ipStateFamily cns.IPFamily + if net.ParseIP(ipState.IPAddress).To4() != nil { + ipStateFamily = cns.IPv4Family + } else { + ipStateFamily = cns.IPv6Family + } + + // check if the IP with the same family type exists already + if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[ipStateFamily]; IPFamilyAlreadyMarkedForAssignment { continue } // Checks if the current IP is available if ipState.GetState() != types.Available { continue } - ipsToAssign[ipState.NCID] = ipState - // Once one IP per container is found break out of the loop and stop searching - if len(ipsToAssign) == numOfNCs { + ipsToAssign[ipStateFamily] = ipState + // Once one IP per family is found break out of the loop and stop searching + if len(ipsToAssign) == numOfIPFamilies { break } } - // Checks to make sure we found one IP for each NC - if len(ipsToAssign) != numOfNCs { + // Checks to make sure we found one IP for each IPFamily + if len(ipsToAssign) != numOfIPFamilies { for ncID := range service.state.ContainerStatus { - if _, found := ipsToAssign[ncID]; found { - continue + for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { + if _, found := ipsToAssign[ipFamily]; found { + continue + } + return podIPInfo, errors.Errorf("not enough IPs available of type %s for %s, waiting on Azure CNS to allocate more with NC Status: %s", + ipFamily, ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } - return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s", - ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 30a2b1e8d6..917ded0f26 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -79,6 +79,7 @@ type HTTPRestService struct { PnpIDByMacAddress map[string]string imdsClient imdsClient nodesubnetIPFetcher *nodesubnet.IPFetcher + IPFamilies []cns.IPFamily } type CNIConflistGenerator interface {