diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 5f64178a46..bc01396ce5 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -129,6 +129,7 @@ type CreateNetworkContainerRequest struct { EndpointPolicies []NetworkContainerRequestPolicies NCStatus v1alpha.NCStatus NetworkInterfaceInfo NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code + IPFamilies map[IPFamily]struct{} } func (req *CreateNetworkContainerRequest) Validate() error { @@ -746,3 +747,10 @@ type NodeRegisterRequest struct { NumCores int NmAgentSupportedApis []string } + +type IPFamily string + +const ( + IPv4Family IPFamily = "ipv4" + IPv6Family IPFamily = "ipv6" +) diff --git a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go index c89d41646b..4f430df655 100644 --- a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go +++ b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go @@ -1,6 +1,7 @@ package nodenetworkconfig import ( + "fmt" "net/netip" "strconv" @@ -15,6 +16,7 @@ import ( //nolint:gocritic //ignore hugeparam func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) { secondaryIPConfigs := map[string]cns.SecondaryIPConfig{} + ipFamilies := map[cns.IPFamily]struct{}{} // iterate through all IP addresses in the subnet described by primaryPrefix and // add them to the request as secondary IPConfigs. @@ -23,12 +25,20 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre IPAddress: addr.String(), NCVersion: int(nc.Version), } + + } + // adds the IPFamily of the primary CIDR to the set + if primaryIPPrefix.Addr().Is4() { + ipFamilies[cns.IPv4Family] = struct{}{} + } else { + ipFamilies[cns.IPv6Family] = struct{}{} } // Add IPs from CIDR block to the secondary IPConfigs if nc.Type == v1alpha.VNETBlock { for _, ipAssignment := range nc.IPAssignments { + // Here we would need to check all other assigned CIDR Blocks that aren't the primary. cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP) if err != nil { return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP) @@ -42,9 +52,17 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre NCVersion: int(nc.Version), } } + // adds the IPFamily of the secondary CIDR to the set + if cidrPrefix.Addr().Is4() { + ipFamilies[cns.IPv4Family] = struct{}{} + } else { + ipFamilies[cns.IPv6Family] = struct{}{} + } } } + fmt.Printf("IPFamilies found on NC %+v are %+v", nc.ID, ipFamilies) + return &cns.CreateNetworkContainerRequest{ HostPrimaryIP: nc.NodeIP, SecondaryIPConfigs: secondaryIPConfigs, @@ -55,6 +73,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre IPSubnet: subnet, GatewayIPAddress: nc.DefaultGateway, }, - NCStatus: nc.Status, + NCStatus: nc.Status, + IPFamilies: ipFamilies, }, nil } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 883d9aaef3..3873e82d96 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -25,6 +25,7 @@ var ( ErrStoreEmpty = errors.New("empty endpoint state store") ErrParsePodIPFailed = errors.New("failed to parse pod's ip") ErrNoNCs = errors.New("no NCs found in the CNS internal state") + ErrNoIPFamilies = errors.New("No IP Families found on NCs") ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state") ErrEndpointStateNotFound = errors.New("endpoint state could not be found in the statefile") ErrGetAllNCResponseEmpty = errors.New("failed to get NC responses from statefile") @@ -989,44 +990,69 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { - // Gets the number of NCs which will determine the number of IPs given to a pod - numOfNCs := len(service.state.ContainerStatus) - // if there are no NCs on the NNC there will be no IPs in the pool so return error - if numOfNCs == 0 { + // Map used to get the number of IPFamilies across all NCs + ipFamilies := map[cns.IPFamily]struct{}{} + + // checks to make sure we have at least one NC + if len(service.state.ContainerStatus) == 0 { return nil, ErrNoNCs } + + // Gets the IPFamilies from all NCs and stores them in a map. This will be ued to determine the amount of IPs to return + for ncID := range service.state.ContainerStatus { + for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { + ipFamilies[ipFamily] = struct{}{} + } + } + + // Makes sure we have at least one IPFamily across all NCs + numOfIPFamilies := len(ipFamilies) + if numOfIPFamilies == 0 { + return nil, ErrNoIPFamilies + } + service.Lock() defer service.Unlock() // Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs - podIPInfo := make([]cns.PodIpInfo, numOfNCs) + podIPInfo := make([]cns.PodIpInfo, numOfIPFamilies) // This map is used to store whether or not we have found an available IP from an NC when looping through the pool - ipsToAssign := make(map[string]cns.IPConfigurationStatus) + ipsToAssign := make(map[cns.IPFamily]cns.IPConfigurationStatus) // Searches for available IPs in the pool for _, ipState := range service.PodIPConfigState { - // check if an IP from this NC is already set side for assignment. - if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment { + // get the IPFamily of the current ipState + var ipStateFamily cns.IPFamily + if net.ParseIP(ipState.IPAddress).To4() != nil { + ipStateFamily = cns.IPv4Family + } else { + ipStateFamily = cns.IPv6Family + } + + // check if the IP with the same family type exists already + if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[ipStateFamily]; IPFamilyAlreadyMarkedForAssignment { continue } // Checks if the current IP is available if ipState.GetState() != types.Available { continue } - ipsToAssign[ipState.NCID] = ipState - // Once one IP per container is found break out of the loop and stop searching - if len(ipsToAssign) == numOfNCs { + ipsToAssign[ipStateFamily] = ipState + // Once one IP per family is found break out of the loop and stop searching + if len(ipsToAssign) == numOfIPFamilies { break } } - // Checks to make sure we found one IP for each NC - if len(ipsToAssign) != numOfNCs { + // Checks to make sure we found one IP for each IPFamily + if len(ipsToAssign) != numOfIPFamilies { for ncID := range service.state.ContainerStatus { - if _, found := ipsToAssign[ncID]; found { - continue + for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { + if _, found := ipsToAssign[ipFamily]; found { + continue + } + return podIPInfo, errors.Errorf("not enough IPs available of type %s for %s, waiting on Azure CNS to allocate more with NC Status: %s", + ipFamily, ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } - return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s", - ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index c467ab04e2..ab4892ffb1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -67,6 +67,8 @@ type iptablesGetter interface { } // HTTPRestService represents http listener for CNS - Container Networking Service. +// TODO: Add a new value for IPFamily +// If we add a new type of Middleware that will be reflected in the IPConfigsHandlerMiddleware value type HTTPRestService struct { *cns.Service dockerClient *dockerclient.Client @@ -92,6 +94,7 @@ type HTTPRestService struct { PnpIDByMacAddress map[string]string imdsClient imdsClient nodesubnetIPFetcher *nodesubnet.IPFetcher + IPFamilies []cns.IPFamily } type CNIConflistGenerator interface { diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 43d1e1aef9..ab26569e68 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -168,8 +168,8 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw if hostVersion == "" { // Host version is the NC version from NMAgent, set it -1 to indicate no result from NMAgent yet. // TODO, query NMAgent and with aggresive time out and assign latest host version. - hostVersion = "-1" } + hostVersion = "1" // Remove the auth token before saving the containerStatus to cns json file createNetworkContainerRequest := req diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 1137b28e96..7ded69a387 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -119,9 +119,12 @@ type NetworkContainer struct { // +kubebuilder:default=vnet Type NCType `json:"type,omitempty"` PrimaryIP string `json:"primaryIP,omitempty"` + PrimaryIPV6 string `json:"primaryIPV6,omitempty"` SubnetName string `json:"subnetName,omitempty"` IPAssignments []IPAssignment `json:"ipAssignments,omitempty"` DefaultGateway string `json:"defaultGateway,omitempty"` + DefaultGatewayV6 string `json:"defaultGatewayV6,omitempty"` + MacAddress string `json:"macAddress,omitempty"` SubnetAddressSpace string `json:"subnetAddressSpace,omitempty"` // +kubebuilder:default=0 // +kubebuilder:validation:Optional diff --git a/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml b/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml index 1410a85e2a..266fdb17a0 100644 --- a/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml +++ b/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.3 + controller-gen.kubebuilder.io/version: v0.17.2 name: nodenetworkconfigs.acn.azure.com spec: group: acn.azure.com @@ -109,6 +109,8 @@ spec: type: string defaultGateway: type: string + defaultGatewayV6: + type: string id: type: string ipAssignments: @@ -122,10 +124,14 @@ spec: type: string type: object type: array + macAddress: + type: string nodeIP: type: string primaryIP: type: string + primaryIPV6: + type: string resourceGroupID: type: string status: