From a2cb744e2dd7341c6597637d91b83a8225dfef1f Mon Sep 17 00:00:00 2001 From: ryandenney Date: Wed, 9 Oct 2024 11:56:31 -0400 Subject: [PATCH 01/16] initial TODOs for CNS --- .../nodenetworkconfig/reconciler.go | 1 + cns/middlewares/k8sSwiftV2.go | 106 +++++++++++++++++- cns/restserver/ipam.go | 5 + cns/restserver/restserver.go | 2 + cns/service/main.go | 2 + 5 files changed, 112 insertions(+), 4 deletions(-) diff --git a/cns/kubecontroller/nodenetworkconfig/reconciler.go b/cns/kubecontroller/nodenetworkconfig/reconciler.go index 4d5eb1bcd1..66faec9751 100644 --- a/cns/kubecontroller/nodenetworkconfig/reconciler.go +++ b/cns/kubecontroller/nodenetworkconfig/reconciler.go @@ -127,6 +127,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco ipAssignments += len(req.SecondaryIPConfigs) } + // TODO: Here we can assign a IPFamilies along with the allocated IPs // record assigned IPs metric allocatedIPs.Set(float64(ipAssignments)) diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index cdfba553dd..1d0673aa7f 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -53,11 +53,109 @@ func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context if respCode != types.Success { return nil, respCode, message } + ipConfigsResp, err := defaultHandler(ctx, req) + // If the pod is not v2, return the response from the handler + // we need to add the secondary Interface. Our current POC cluster is returning here + if !req.SecondaryInterfacesExist { + return ipConfigsResp, err + } + // TODO: the pod itself won't be "V2" as we aren't using multitenancy pods + // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config + defer func() { + // Release the default IP config if there is an error + if err != nil { + _, err = failureHandler(ctx, req) + if err != nil { + logger.Errorf("failed to release default IP config : %v", err) + } + } + }() + if err != nil { + return ipConfigsResp, err + } + SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) + } + ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...) + // Set routes for the pod + for i := range ipConfigsResp.PodIPInfo { + ipInfo := &ipConfigsResp.PodIPInfo[i] + // Backend nics doesn't need routes to be set + if ipInfo.NICType != cns.BackendNIC { + err = k.setRoutes(ipInfo) + if err != nil { + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: types.FailedToAllocateIPConfig, + Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), + }, + PodIPInfo: []cns.PodIpInfo{}, + }, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name()) + } + } + } + return ipConfigsResp, nil + } +} - // update ipConfigRequest - respCode, message = k.UpdateIPConfigRequest(mtpnc, req) - if respCode != types.Success { - return nil, respCode, message +// TODO: we will not be using multitenant pods. Need to look into what labels we are currently seeing and maybe compare to Vanilla swiftv2 +// For our purposes we would skip over this logic or need to replace it with something to check the delegated NIC +// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. +// nolint +func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) + if err != nil { + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) + return nil, types.UnexpectedError, errBuf.Error() + } + logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) + podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) + return nil, types.UnexpectedError, errBuf.Error() + } + + // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready + _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] + _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] + if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel { + + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() + } + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if !mtpnc.IsReady() { + return nil, types.UnexpectedError, errMTPNCNotReady.Error() + } + // If primary Ip is set in status field, it indicates the presence of secondary interfaces + if mtpnc.Status.PrimaryIP != "" { + req.SecondaryInterfacesExist = true + } + interfaceInfos := mtpnc.Status.InterfaceInfos + for _, interfaceInfo := range interfaceInfos { + if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { + if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { + return nil, types.UnexpectedError, errMTPNCNotReady.Error() + } + req.BackendInterfaceExist = true + req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) + + } + if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC { + req.SecondaryInterfacesExist = true + } } } logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 883d9aaef3..30d364d16e 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -282,6 +282,7 @@ func (service *HTTPRestService) RequestIPConfigsHandler(w http.ResponseWriter, r } var ipConfigsResp *cns.IPConfigsResponse + // TODO: Decide what middleware option we need to singetenancy swift v2 // Check if IPConfigsHandlerMiddleware is set if service.IPConfigsHandlerMiddleware != nil { // Wrap the default datapath handlers with the middleware depending on middleware type @@ -990,8 +991,11 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { // Gets the number of NCs which will determine the number of IPs given to a pod + // TODO: If we go with the One NC route then this logic would have to be updated + // Instead of using numOfNCs we would have a variable added to the HTTPRestService struct that determines IPFamilies and use the length of that numOfNCs := len(service.state.ContainerStatus) // if there are no NCs on the NNC there will be no IPs in the pool so return error + // We could still keep this check if numOfNCs == 0 { return nil, ErrNoNCs } @@ -1005,6 +1009,7 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([ // Searches for available IPs in the pool for _, ipState := range service.PodIPConfigState { // check if an IP from this NC is already set side for assignment. + // TODO: Instead of using ncAlreadyMarkedForAssignment it would be IPFamilyAlreadyMarkedForAssignment if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment { continue } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index c467ab04e2..0b973fcdf1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -67,6 +67,8 @@ type iptablesGetter interface { } // HTTPRestService represents http listener for CNS - Container Networking Service. +// TODO: Add a new value for IPFamily +// If we add a new type of Middleware that will be reflected in the IPConfigsHandlerMiddleware value type HTTPRestService struct { *cns.Service dockerClient *dockerclient.Client diff --git a/cns/service/main.go b/cns/service/main.go index fb1655accd..3c241aaacd 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1550,6 +1550,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // Start building the NNC Reconciler + // TODO: We need to return the IPFamilies from the reconciler so that they can be added to the Service struct // get CNS Node IP to compare NC Node IP with this Node IP to ensure NCs were created for this node nodeIP := configuration.NodeIP() nncReconciler := nncctrl.NewReconciler(httpRestServiceImplementation, poolMonitor, nodeIP) @@ -1583,6 +1584,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } } + // TODO: If we need special middleware this is where we would be setting it if cnsconfig.EnableSwiftV2 { if err := mtpncctrl.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") From d08a08f7d9048a57f98e0258b258f8a911b31d3b Mon Sep 17 00:00:00 2001 From: ryandenney Date: Thu, 10 Oct 2024 11:58:06 -0400 Subject: [PATCH 02/16] more TODOs --- cns/kubecontroller/nodenetworkconfig/conversion_linux.go | 6 ++++++ cns/middlewares/k8sSwiftV2_linux.go | 3 +++ cns/middlewares/k8sSwiftV2_windows.go | 2 ++ 3 files changed, 11 insertions(+) diff --git a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go index c89d41646b..8ffa3a9e98 100644 --- a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go +++ b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go @@ -18,17 +18,23 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre // iterate through all IP addresses in the subnet described by primaryPrefix and // add them to the request as secondary IPConfigs. + // TODO: If we decide to look to not change the contract to add an IPFamilies flag and just detect it we would have to do it here + // By doing this we can also have it work for Dualstack Overlay + // To do it we would need to pass in a pointer to an IPFamilies slice to this function and add the family of the IPs added in + // Here we check the primary CIDR which is needed for overlay and VNET Block for addr := primaryIPPrefix.Masked().Addr(); primaryIPPrefix.Contains(addr); addr = addr.Next() { secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{ IPAddress: addr.String(), NCVersion: int(nc.Version), } + } // Add IPs from CIDR block to the secondary IPConfigs if nc.Type == v1alpha.VNETBlock { for _, ipAssignment := range nc.IPAssignments { + // Here we would need to check all other assigned CIDR Blocks that aren't the primary. cidrPrefix, err := netip.ParsePrefix(ipAssignment.IP) if err != nil { return nil, errors.Wrapf(err, "invalid CIDR block: %s", ipAssignment.IP) diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index bb075cc3e0..7b7b23fb31 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -20,6 +20,9 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { var routes []cns.Route switch podIPInfo.NICType { + // TODO: We may want to create a new type of NIC. Currently we are using delegated NICs but this set routes method only + // takes in for the multitenant scenario. We should have a new case that behaves similarly to InfraNIC but for a delegated NIC + // Q: Does the code currently see the kube pods as Infra or Delegated? They currently use the delegated pod subnet for IPs case cns.DelegatedVMNIC: virtualGWRoute := cns.Route{ IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index 1dda583bd1..e2b01ee85f 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -49,6 +49,8 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { // assignSubnetPrefixLengthFields will assign the subnet-prefix length to some fields of podipinfo // this is required for the windows scenario so that HNS programming is successful for pods +// TODO: This is being used for the delegated NIC for Windows solution. Once we are testing Windows we will +// Need to confirm that this assigns the whole pod subnet that we expect for both v4 and v6 func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(podIPInfo *cns.PodIpInfo, interfaceInfo v1alpha1.InterfaceInfo, ip string) error { // Parse MTPNC SubnetAddressSpace to get the subnet prefix length subnet, subnetPrefix, err := utils.ParseIPAndPrefix(interfaceInfo.SubnetAddressSpace) From b8733cd2364128b98b6fa81a7792d7459d8d75f3 Mon Sep 17 00:00:00 2001 From: ryandenney Date: Wed, 16 Oct 2024 11:58:26 -0400 Subject: [PATCH 03/16] initial changes --- cns/NetworkContainerContract.go | 2 ++ .../nodenetworkconfig/conversion_linux.go | 23 +++++++++--- .../nodenetworkconfig/reconciler.go | 1 - cns/restserver/const.go | 7 ++++ cns/restserver/ipam.go | 35 +++++++++---------- cns/restserver/restserver.go | 1 + cns/restserver/util.go | 2 ++ 7 files changed, 47 insertions(+), 24 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index 5f64178a46..e1d84ebd0e 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/Azure/azure-container-networking/network/policy" @@ -129,6 +130,7 @@ type CreateNetworkContainerRequest struct { EndpointPolicies []NetworkContainerRequestPolicies NCStatus v1alpha.NCStatus NetworkInterfaceInfo NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code + IPFamilies map[restserver.IPFamily]struct{} } func (req *CreateNetworkContainerRequest) Validate() error { diff --git a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go index 8ffa3a9e98..10f453fd0f 100644 --- a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go +++ b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go @@ -1,10 +1,12 @@ package nodenetworkconfig import ( + "fmt" "net/netip" "strconv" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" ) @@ -15,12 +17,11 @@ import ( //nolint:gocritic //ignore hugeparam func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) { secondaryIPConfigs := map[string]cns.SecondaryIPConfig{} + ipFamilies := map[restserver.IPFamily]struct{}{} // iterate through all IP addresses in the subnet described by primaryPrefix and // add them to the request as secondary IPConfigs. - // TODO: If we decide to look to not change the contract to add an IPFamilies flag and just detect it we would have to do it here - // By doing this we can also have it work for Dualstack Overlay - // To do it we would need to pass in a pointer to an IPFamilies slice to this function and add the family of the IPs added in + // TODO: we need to pass in a pointer to an IPFamilies slice to this function and add the family of the IPs added in // Here we check the primary CIDR which is needed for overlay and VNET Block for addr := primaryIPPrefix.Masked().Addr(); primaryIPPrefix.Contains(addr); addr = addr.Next() { secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{ @@ -30,6 +31,12 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre } + if primaryIPPrefix.Addr().Is4() { + ipFamilies[restserver.IPv4Family] = struct{}{} + } else { + ipFamilies[restserver.IPv6Family] = struct{}{} + } + // Add IPs from CIDR block to the secondary IPConfigs if nc.Type == v1alpha.VNETBlock { @@ -48,9 +55,16 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre NCVersion: int(nc.Version), } } + if cidrPrefix.Addr().Is4() { + ipFamilies[restserver.IPv4Family] = struct{}{} + } else { + ipFamilies[restserver.IPv6Family] = struct{}{} + } } } + fmt.Printf("IPFamilies on NC %+v and %+v", nc.ID, ipFamilies) + return &cns.CreateNetworkContainerRequest{ HostPrimaryIP: nc.NodeIP, SecondaryIPConfigs: secondaryIPConfigs, @@ -61,6 +75,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre IPSubnet: subnet, GatewayIPAddress: nc.DefaultGateway, }, - NCStatus: nc.Status, + NCStatus: nc.Status, + IPFamilies: ipFamilies, }, nil } diff --git a/cns/kubecontroller/nodenetworkconfig/reconciler.go b/cns/kubecontroller/nodenetworkconfig/reconciler.go index 66faec9751..4d5eb1bcd1 100644 --- a/cns/kubecontroller/nodenetworkconfig/reconciler.go +++ b/cns/kubecontroller/nodenetworkconfig/reconciler.go @@ -127,7 +127,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco ipAssignments += len(req.SecondaryIPConfigs) } - // TODO: Here we can assign a IPFamilies along with the allocated IPs // record assigned IPs metric allocatedIPs.Set(float64(ipAssignments)) diff --git a/cns/restserver/const.go b/cns/restserver/const.go index 0782755596..eff738423a 100644 --- a/cns/restserver/const.go +++ b/cns/restserver/const.go @@ -13,3 +13,10 @@ const ( dncApiVersion = "?api-version=2018-03-01" nmaAPICallTimeout = 2 * time.Second ) + +type IPFamily string + +const ( + IPv4Family IPFamily = "ipv4" + IPv6Family IPFamily = "ipv6" +) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 30d364d16e..13c11e8a00 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -990,48 +990,45 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { - // Gets the number of NCs which will determine the number of IPs given to a pod - // TODO: If we go with the One NC route then this logic would have to be updated - // Instead of using numOfNCs we would have a variable added to the HTTPRestService struct that determines IPFamilies and use the length of that - numOfNCs := len(service.state.ContainerStatus) - // if there are no NCs on the NNC there will be no IPs in the pool so return error - // We could still keep this check - if numOfNCs == 0 { + // Gets the number of IPFamilies expected to be returned. + numOfIPFamilies := len(service.IPFamilies) + + if numOfIPFamilies == 0 { + // TODO: replace with it's own error return nil, ErrNoNCs } service.Lock() defer service.Unlock() // Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs - podIPInfo := make([]cns.PodIpInfo, numOfNCs) + podIPInfo := make([]cns.PodIpInfo, numOfIPFamilies) // This map is used to store whether or not we have found an available IP from an NC when looping through the pool - ipsToAssign := make(map[string]cns.IPConfigurationStatus) + ipsToAssign := make(map[IPFamily]cns.IPConfigurationStatus) // Searches for available IPs in the pool for _, ipState := range service.PodIPConfigState { - // check if an IP from this NC is already set side for assignment. - // TODO: Instead of using ncAlreadyMarkedForAssignment it would be IPFamilyAlreadyMarkedForAssignment - if _, ncAlreadyMarkedForAssignment := ipsToAssign[ipState.NCID]; ncAlreadyMarkedForAssignment { + // check if the IP with the same family type exists already + if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[net.ParseIP(ipState.IPAddress).To4() == nil]; IPFamilyAlreadyMarkedForAssignment { continue } // Checks if the current IP is available if ipState.GetState() != types.Available { continue } - ipsToAssign[ipState.NCID] = ipState + ipsToAssign[net.ParseIP(ipState.IPAddress).To4() == nil] = ipState // Once one IP per container is found break out of the loop and stop searching - if len(ipsToAssign) == numOfNCs { + if len(ipsToAssign) == numOfIPFamilies { break } } // Checks to make sure we found one IP for each NC - if len(ipsToAssign) != numOfNCs { - for ncID := range service.state.ContainerStatus { - if _, found := ipsToAssign[ncID]; found { + if len(ipsToAssign) != numOfIPFamilies { + for _, ipFamily := range service.IPFamilies { + if _, found := ipsToAssign[ipFamily]; found { continue } - return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s", - ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) + return podIPInfo, errors.Errorf("not enough IPs available of type %s, waiting on Azure CNS to allocate more with NC Status: %s", + ipFamily, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index 0b973fcdf1..b028727073 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -94,6 +94,7 @@ type HTTPRestService struct { PnpIDByMacAddress map[string]string imdsClient imdsClient nodesubnetIPFetcher *nodesubnet.IPFetcher + IPFamilies []IPFamily } type CNIConflistGenerator interface { diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 43d1e1aef9..2bd198c34f 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -183,6 +183,8 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw VfpUpdateComplete: vfpUpdateComplete, } + service.IPFamilies = req.IPFamilies + switch req.NetworkContainerType { case cns.AzureContainerInstance: fallthrough From 6327d1ed89bf1bc47d24d76cbe28fb01b8e3deb3 Mon Sep 17 00:00:00 2001 From: ryandenney Date: Thu, 17 Oct 2024 11:37:15 -0400 Subject: [PATCH 04/16] fixes overlay issue --- .../nodenetworkconfig/conversion_linux.go | 7 ++- cns/restserver/ipam.go | 45 ++++++++++++++----- cns/restserver/util.go | 2 - 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go index 10f453fd0f..03884de73b 100644 --- a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go +++ b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go @@ -21,8 +21,6 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre // iterate through all IP addresses in the subnet described by primaryPrefix and // add them to the request as secondary IPConfigs. - // TODO: we need to pass in a pointer to an IPFamilies slice to this function and add the family of the IPs added in - // Here we check the primary CIDR which is needed for overlay and VNET Block for addr := primaryIPPrefix.Masked().Addr(); primaryIPPrefix.Contains(addr); addr = addr.Next() { secondaryIPConfigs[addr.String()] = cns.SecondaryIPConfig{ IPAddress: addr.String(), @@ -30,7 +28,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre } } - + // adds the IPFamily of the primary CIDR to the set if primaryIPPrefix.Addr().Is4() { ipFamilies[restserver.IPv4Family] = struct{}{} } else { @@ -55,6 +53,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre NCVersion: int(nc.Version), } } + // adds the IPFamily of the secondary CIDR to the set if cidrPrefix.Addr().Is4() { ipFamilies[restserver.IPv4Family] = struct{}{} } else { @@ -63,7 +62,7 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre } } - fmt.Printf("IPFamilies on NC %+v and %+v", nc.ID, ipFamilies) + fmt.Printf("IPFamilies found on NC %+v are %+v", nc.ID, ipFamilies) return &cns.CreateNetworkContainerRequest{ HostPrimaryIP: nc.NodeIP, diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 13c11e8a00..6b5ed9bc8e 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -25,6 +25,7 @@ var ( ErrStoreEmpty = errors.New("empty endpoint state store") ErrParsePodIPFailed = errors.New("failed to parse pod's ip") ErrNoNCs = errors.New("no NCs found in the CNS internal state") + ErrNoIPFamilies = errors.New("No IP Families found on NCs") ErrOptManageEndpointState = errors.New("CNS is not set to manage the endpoint state") ErrEndpointStateNotFound = errors.New("endpoint state could not be found in the statefile") ErrGetAllNCResponseEmpty = errors.New("failed to get NC responses from statefile") @@ -990,13 +991,25 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { - // Gets the number of IPFamilies expected to be returned. - numOfIPFamilies := len(service.IPFamilies) + // Gets the number of IPFamilies expected to be returned across all NCs + ipFamilies := map[IPFamily]struct{}{} - if numOfIPFamilies == 0 { - // TODO: replace with it's own error + // checks to make sure we have at least one NC + if len(service.state.ContainerStatus) == 0 { return nil, ErrNoNCs } + + for ncID := range service.state.ContainerStatus { + for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { + ipFamilies[ipFamily] = struct{}{} + } + } + + numOfIPFamilies := len(ipFamilies) + if numOfIPFamilies == 0 { + return nil, ErrNoIPFamilies + } + service.Lock() defer service.Unlock() // Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs @@ -1006,29 +1019,37 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([ // Searches for available IPs in the pool for _, ipState := range service.PodIPConfigState { + // get the IPFamily of the current ipState + var ipStateFamily IPFamily + if net.ParseIP(ipState.IPAddress).To4() != nil { + ipStateFamily = IPv4Family + } else { + ipStateFamily = IPv6Family + } + // check if the IP with the same family type exists already - if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[net.ParseIP(ipState.IPAddress).To4() == nil]; IPFamilyAlreadyMarkedForAssignment { + if _, IPFamilyAlreadyMarkedForAssignment := ipsToAssign[ipStateFamily]; IPFamilyAlreadyMarkedForAssignment { continue } // Checks if the current IP is available if ipState.GetState() != types.Available { continue } - ipsToAssign[net.ParseIP(ipState.IPAddress).To4() == nil] = ipState - // Once one IP per container is found break out of the loop and stop searching + ipsToAssign[ipStateFamily] = ipState + // Once one IP per family is found break out of the loop and stop searching if len(ipsToAssign) == numOfIPFamilies { break } } - // Checks to make sure we found one IP for each NC + // Checks to make sure we found one IP for each IPFamily if len(ipsToAssign) != numOfIPFamilies { - for _, ipFamily := range service.IPFamilies { - if _, found := ipsToAssign[ipFamily]; found { + for ncID := range service.state.ContainerStatus { + if _, found := ipsToAssign[ncID]; found { continue } - return podIPInfo, errors.Errorf("not enough IPs available of type %s, waiting on Azure CNS to allocate more with NC Status: %s", - ipFamily, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) + return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s", + ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } } diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 2bd198c34f..43d1e1aef9 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -183,8 +183,6 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw VfpUpdateComplete: vfpUpdateComplete, } - service.IPFamilies = req.IPFamilies - switch req.NetworkContainerType { case cns.AzureContainerInstance: fallthrough From f43b6904ff3850f00c8a8b7e5bc3f116ca416409 Mon Sep 17 00:00:00 2001 From: ryandenney Date: Thu, 17 Oct 2024 11:42:05 -0400 Subject: [PATCH 05/16] cleaning up TODOs --- cns/restserver/ipusage.go | 58 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 cns/restserver/ipusage.go diff --git a/cns/restserver/ipusage.go b/cns/restserver/ipusage.go new file mode 100644 index 0000000000..e2ae30382b --- /dev/null +++ b/cns/restserver/ipusage.go @@ -0,0 +1,58 @@ +package restserver + +import ( + "github.com/Azure/azure-container-networking/cns/logger" + "github.com/Azure/azure-container-networking/cns/types" +) + +type ipState struct { + // allocatedIPs are all the IPs given to CNS by DNC. + allocatedIPs int64 + // assignedIPs are the IPs CNS gives to Pods. + assignedIPs int64 + // availableIPs are the IPs in state "Available". + availableIPs int64 + // programmingIPs are the IPs in state "PendingProgramming". + programmingIPs int64 + // releasingIPs are the IPs in state "PendingReleasr". + releasingIPs int64 +} + +func (service *HTTPRestService) buildIPState() *ipState { + service.Lock() + defer service.Unlock() + + state := ipState{ + allocatedIPs: 0, + assignedIPs: 0, + availableIPs: 0, + programmingIPs: 0, + releasingIPs: 0, + } + + //nolint:gocritic // This has to iterate over the IP Config state to get the counts. + for _, ipConfig := range service.PodIPConfigState { + state.allocatedIPs++ + if ipConfig.GetState() == types.Assigned { + state.assignedIPs++ + } + if ipConfig.GetState() == types.Available { + state.availableIPs++ + } + if ipConfig.GetState() == types.PendingProgramming { + state.programmingIPs++ + } + if ipConfig.GetState() == types.PendingRelease { + state.releasingIPs++ + } + } + + logger.Printf("[IP Usage] Allocated IPs: %d, Assigned IPs: %d, Available IPs: %d, PendingProgramming IPs: %d, PendingRelease IPs: %d", + state.allocatedIPs, + state.assignedIPs, + state.availableIPs, + state.programmingIPs, + state.releasingIPs, + ) + return &state +} From e4f62a381484dad951ed9eac8693d14c969c2b6a Mon Sep 17 00:00:00 2001 From: ryandenney Date: Mon, 11 Nov 2024 10:11:17 -0500 Subject: [PATCH 06/16] updating error check in ipam --- cns/restserver/ipam.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 6b5ed9bc8e..a1bd17a7cd 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -991,7 +991,7 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // Assigns an available IP from each NC on the NNC. If there is one NC then we expect to only have one IP return // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { - // Gets the number of IPFamilies expected to be returned across all NCs + // Map used to get the number of IPFamilies across all NCs ipFamilies := map[IPFamily]struct{}{} // checks to make sure we have at least one NC @@ -1045,11 +1045,13 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([ // Checks to make sure we found one IP for each IPFamily if len(ipsToAssign) != numOfIPFamilies { for ncID := range service.state.ContainerStatus { - if _, found := ipsToAssign[ncID]; found { - continue + for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { + if _, found := ipsToAssign[ipFamily]; found { + continue + } + return podIPInfo, errors.Errorf("not enough IPs available of type %s for %s, waiting on Azure CNS to allocate more with NC Status: %s", + ipFamily, ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } - return podIPInfo, errors.Errorf("not enough IPs available for %s, waiting on Azure CNS to allocate more with NC Status: %s", - ncID, string(service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.NCStatus)) } } From 826fedfb1bc564c76072cd5059dad27110bf566c Mon Sep 17 00:00:00 2001 From: ryandenney Date: Tue, 12 Nov 2024 14:58:09 -0500 Subject: [PATCH 07/16] updates to IPFamily and dependencies --- cns/NetworkContainerContract.go | 10 ++++++++-- .../nodenetworkconfig/conversion_linux.go | 11 +++++------ cns/restserver/const.go | 7 ------- cns/restserver/ipam.go | 10 +++++----- cns/restserver/ipusage.go | 13 ------------- cns/restserver/restserver.go | 2 +- 6 files changed, 19 insertions(+), 34 deletions(-) diff --git a/cns/NetworkContainerContract.go b/cns/NetworkContainerContract.go index e1d84ebd0e..bc01396ce5 100644 --- a/cns/NetworkContainerContract.go +++ b/cns/NetworkContainerContract.go @@ -7,7 +7,6 @@ import ( "strconv" "strings" - "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/Azure/azure-container-networking/network/policy" @@ -130,7 +129,7 @@ type CreateNetworkContainerRequest struct { EndpointPolicies []NetworkContainerRequestPolicies NCStatus v1alpha.NCStatus NetworkInterfaceInfo NetworkInterfaceInfo //nolint // introducing new field for backendnic, to be used later by cni code - IPFamilies map[restserver.IPFamily]struct{} + IPFamilies map[IPFamily]struct{} } func (req *CreateNetworkContainerRequest) Validate() error { @@ -748,3 +747,10 @@ type NodeRegisterRequest struct { NumCores int NmAgentSupportedApis []string } + +type IPFamily string + +const ( + IPv4Family IPFamily = "ipv4" + IPv6Family IPFamily = "ipv6" +) diff --git a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go index 03884de73b..4f430df655 100644 --- a/cns/kubecontroller/nodenetworkconfig/conversion_linux.go +++ b/cns/kubecontroller/nodenetworkconfig/conversion_linux.go @@ -6,7 +6,6 @@ import ( "strconv" "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" "github.com/pkg/errors" ) @@ -17,7 +16,7 @@ import ( //nolint:gocritic //ignore hugeparam func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPrefix netip.Prefix, subnet cns.IPSubnet) (*cns.CreateNetworkContainerRequest, error) { secondaryIPConfigs := map[string]cns.SecondaryIPConfig{} - ipFamilies := map[restserver.IPFamily]struct{}{} + ipFamilies := map[cns.IPFamily]struct{}{} // iterate through all IP addresses in the subnet described by primaryPrefix and // add them to the request as secondary IPConfigs. @@ -30,9 +29,9 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre } // adds the IPFamily of the primary CIDR to the set if primaryIPPrefix.Addr().Is4() { - ipFamilies[restserver.IPv4Family] = struct{}{} + ipFamilies[cns.IPv4Family] = struct{}{} } else { - ipFamilies[restserver.IPv6Family] = struct{}{} + ipFamilies[cns.IPv6Family] = struct{}{} } // Add IPs from CIDR block to the secondary IPConfigs @@ -55,9 +54,9 @@ func createNCRequestFromStaticNCHelper(nc v1alpha.NetworkContainer, primaryIPPre } // adds the IPFamily of the secondary CIDR to the set if cidrPrefix.Addr().Is4() { - ipFamilies[restserver.IPv4Family] = struct{}{} + ipFamilies[cns.IPv4Family] = struct{}{} } else { - ipFamilies[restserver.IPv6Family] = struct{}{} + ipFamilies[cns.IPv6Family] = struct{}{} } } } diff --git a/cns/restserver/const.go b/cns/restserver/const.go index eff738423a..0782755596 100644 --- a/cns/restserver/const.go +++ b/cns/restserver/const.go @@ -13,10 +13,3 @@ const ( dncApiVersion = "?api-version=2018-03-01" nmaAPICallTimeout = 2 * time.Second ) - -type IPFamily string - -const ( - IPv4Family IPFamily = "ipv4" - IPv6Family IPFamily = "ipv6" -) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index a1bd17a7cd..ba70279a19 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -992,7 +992,7 @@ func (service *HTTPRestService) AssignDesiredIPConfigs(podInfo cns.PodInfo, desi // In the case of dualstack we would expect to have one IPv6 from one NC and one IPv4 from a second NC func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { // Map used to get the number of IPFamilies across all NCs - ipFamilies := map[IPFamily]struct{}{} + ipFamilies := map[cns.IPFamily]struct{}{} // checks to make sure we have at least one NC if len(service.state.ContainerStatus) == 0 { @@ -1015,16 +1015,16 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([ // Creates a slice of PodIpInfo with the size as number of NCs to hold the result for assigned IP configs podIPInfo := make([]cns.PodIpInfo, numOfIPFamilies) // This map is used to store whether or not we have found an available IP from an NC when looping through the pool - ipsToAssign := make(map[IPFamily]cns.IPConfigurationStatus) + ipsToAssign := make(map[cns.IPFamily]cns.IPConfigurationStatus) // Searches for available IPs in the pool for _, ipState := range service.PodIPConfigState { // get the IPFamily of the current ipState - var ipStateFamily IPFamily + var ipStateFamily cns.IPFamily if net.ParseIP(ipState.IPAddress).To4() != nil { - ipStateFamily = IPv4Family + ipStateFamily = cns.IPv4Family } else { - ipStateFamily = IPv6Family + ipStateFamily = cns.IPv6Family } // check if the IP with the same family type exists already diff --git a/cns/restserver/ipusage.go b/cns/restserver/ipusage.go index e2ae30382b..ed87752104 100644 --- a/cns/restserver/ipusage.go +++ b/cns/restserver/ipusage.go @@ -5,19 +5,6 @@ import ( "github.com/Azure/azure-container-networking/cns/types" ) -type ipState struct { - // allocatedIPs are all the IPs given to CNS by DNC. - allocatedIPs int64 - // assignedIPs are the IPs CNS gives to Pods. - assignedIPs int64 - // availableIPs are the IPs in state "Available". - availableIPs int64 - // programmingIPs are the IPs in state "PendingProgramming". - programmingIPs int64 - // releasingIPs are the IPs in state "PendingReleasr". - releasingIPs int64 -} - func (service *HTTPRestService) buildIPState() *ipState { service.Lock() defer service.Unlock() diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index b028727073..ab4892ffb1 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -94,7 +94,7 @@ type HTTPRestService struct { PnpIDByMacAddress map[string]string imdsClient imdsClient nodesubnetIPFetcher *nodesubnet.IPFetcher - IPFamilies []IPFamily + IPFamilies []cns.IPFamily } type CNIConflistGenerator interface { From c408d53147841e5a767884787ed738b0dc6bfe17 Mon Sep 17 00:00:00 2001 From: ryandenney Date: Mon, 18 Nov 2024 11:45:30 -0500 Subject: [PATCH 08/16] removing todos --- cns/middlewares/k8sSwiftV2.go | 3 --- cns/middlewares/k8sSwiftV2_linux.go | 3 --- cns/middlewares/k8sSwiftV2_windows.go | 2 -- cns/restserver/ipam.go | 1 - cns/service/main.go | 2 -- 5 files changed, 11 deletions(-) diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index 1d0673aa7f..92dc4e5c7c 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -55,11 +55,9 @@ func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context } ipConfigsResp, err := defaultHandler(ctx, req) // If the pod is not v2, return the response from the handler - // we need to add the secondary Interface. Our current POC cluster is returning here if !req.SecondaryInterfacesExist { return ipConfigsResp, err } - // TODO: the pod itself won't be "V2" as we aren't using multitenancy pods // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config defer func() { // Release the default IP config if there is an error @@ -105,7 +103,6 @@ func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context } } -// TODO: we will not be using multitenant pods. Need to look into what labels we are currently seeing and maybe compare to Vanilla swiftv2 // For our purposes we would skip over this logic or need to replace it with something to check the delegated NIC // validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. // nolint diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index 7b7b23fb31..bb075cc3e0 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -20,9 +20,6 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { var routes []cns.Route switch podIPInfo.NICType { - // TODO: We may want to create a new type of NIC. Currently we are using delegated NICs but this set routes method only - // takes in for the multitenant scenario. We should have a new case that behaves similarly to InfraNIC but for a delegated NIC - // Q: Does the code currently see the kube pods as Infra or Delegated? They currently use the delegated pod subnet for IPs case cns.DelegatedVMNIC: virtualGWRoute := cns.Route{ IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index e2b01ee85f..1dda583bd1 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -49,8 +49,6 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { // assignSubnetPrefixLengthFields will assign the subnet-prefix length to some fields of podipinfo // this is required for the windows scenario so that HNS programming is successful for pods -// TODO: This is being used for the delegated NIC for Windows solution. Once we are testing Windows we will -// Need to confirm that this assigns the whole pod subnet that we expect for both v4 and v6 func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(podIPInfo *cns.PodIpInfo, interfaceInfo v1alpha1.InterfaceInfo, ip string) error { // Parse MTPNC SubnetAddressSpace to get the subnet prefix length subnet, subnetPrefix, err := utils.ParseIPAndPrefix(interfaceInfo.SubnetAddressSpace) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index ba70279a19..48b9dc6579 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -283,7 +283,6 @@ func (service *HTTPRestService) RequestIPConfigsHandler(w http.ResponseWriter, r } var ipConfigsResp *cns.IPConfigsResponse - // TODO: Decide what middleware option we need to singetenancy swift v2 // Check if IPConfigsHandlerMiddleware is set if service.IPConfigsHandlerMiddleware != nil { // Wrap the default datapath handlers with the middleware depending on middleware type diff --git a/cns/service/main.go b/cns/service/main.go index 3c241aaacd..fb1655accd 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -1550,7 +1550,6 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // Start building the NNC Reconciler - // TODO: We need to return the IPFamilies from the reconciler so that they can be added to the Service struct // get CNS Node IP to compare NC Node IP with this Node IP to ensure NCs were created for this node nodeIP := configuration.NodeIP() nncReconciler := nncctrl.NewReconciler(httpRestServiceImplementation, poolMonitor, nodeIP) @@ -1584,7 +1583,6 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn } } - // TODO: If we need special middleware this is where we would be setting it if cnsconfig.EnableSwiftV2 { if err := mtpncctrl.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup mtpnc reconciler with manager") From cd62086450fefd80df3fa23f5bdf46e5296ceafd Mon Sep 17 00:00:00 2001 From: ryandenney Date: Mon, 18 Nov 2024 11:48:02 -0500 Subject: [PATCH 09/16] fixing comments --- cns/middlewares/k8sSwiftV2.go | 1 - cns/restserver/ipam.go | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index 92dc4e5c7c..2df5c18e0f 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -103,7 +103,6 @@ func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context } } -// For our purposes we would skip over this logic or need to replace it with something to check the delegated NIC // validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. // nolint func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 48b9dc6579..3873e82d96 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -998,12 +998,14 @@ func (service *HTTPRestService) AssignAvailableIPConfigs(podInfo cns.PodInfo) ([ return nil, ErrNoNCs } + // Gets the IPFamilies from all NCs and stores them in a map. This will be ued to determine the amount of IPs to return for ncID := range service.state.ContainerStatus { for ipFamily := range service.state.ContainerStatus[ncID].CreateNetworkContainerRequest.IPFamilies { ipFamilies[ipFamily] = struct{}{} } } + // Makes sure we have at least one IPFamily across all NCs numOfIPFamilies := len(ipFamilies) if numOfIPFamilies == 0 { return nil, ErrNoIPFamilies From 58b7f3e21f6d175f900dc4cd82d13f623762c6bc Mon Sep 17 00:00:00 2001 From: ryandenney Date: Mon, 18 Nov 2024 11:50:21 -0500 Subject: [PATCH 10/16] remove old file --- cns/restserver/ipusage.go | 45 --------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 cns/restserver/ipusage.go diff --git a/cns/restserver/ipusage.go b/cns/restserver/ipusage.go deleted file mode 100644 index ed87752104..0000000000 --- a/cns/restserver/ipusage.go +++ /dev/null @@ -1,45 +0,0 @@ -package restserver - -import ( - "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/types" -) - -func (service *HTTPRestService) buildIPState() *ipState { - service.Lock() - defer service.Unlock() - - state := ipState{ - allocatedIPs: 0, - assignedIPs: 0, - availableIPs: 0, - programmingIPs: 0, - releasingIPs: 0, - } - - //nolint:gocritic // This has to iterate over the IP Config state to get the counts. - for _, ipConfig := range service.PodIPConfigState { - state.allocatedIPs++ - if ipConfig.GetState() == types.Assigned { - state.assignedIPs++ - } - if ipConfig.GetState() == types.Available { - state.availableIPs++ - } - if ipConfig.GetState() == types.PendingProgramming { - state.programmingIPs++ - } - if ipConfig.GetState() == types.PendingRelease { - state.releasingIPs++ - } - } - - logger.Printf("[IP Usage] Allocated IPs: %d, Assigned IPs: %d, Available IPs: %d, PendingProgramming IPs: %d, PendingRelease IPs: %d", - state.allocatedIPs, - state.assignedIPs, - state.availableIPs, - state.programmingIPs, - state.releasingIPs, - ) - return &state -} From 4cc3ed8b7f30f190e8789436fcd50e51f0e904ce Mon Sep 17 00:00:00 2001 From: keithnguyen Date: Fri, 24 Jan 2025 18:38:31 +0000 Subject: [PATCH 11/16] -1 to 1 --- cns/restserver/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 43d1e1aef9..05352b9f24 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -168,7 +168,7 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw if hostVersion == "" { // Host version is the NC version from NMAgent, set it -1 to indicate no result from NMAgent yet. // TODO, query NMAgent and with aggresive time out and assign latest host version. - hostVersion = "-1" + hostVersion = "1" } // Remove the auth token before saving the containerStatus to cns json file From 60cb1b8434e1a2aa964817e225acf45d489c59e5 Mon Sep 17 00:00:00 2001 From: keithnguyen Date: Mon, 27 Jan 2025 23:41:26 +0000 Subject: [PATCH 12/16] Fix --- .../manifests/acn.azure.com_nodenetworkconfigs.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml b/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml index 1410a85e2a..8d7fb5e08e 100644 --- a/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml +++ b/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml @@ -109,6 +109,8 @@ spec: type: string defaultGateway: type: string + defaultGatewayV6: + type: string id: type: string ipAssignments: @@ -126,6 +128,8 @@ spec: type: string primaryIP: type: string + primaryIPV6: + type: string resourceGroupID: type: string status: From ee1072ea26a2eb5bb7e693829dff6d823759edc5 Mon Sep 17 00:00:00 2001 From: keithnguyen Date: Tue, 28 Jan 2025 21:47:40 +0000 Subject: [PATCH 13/16] set it to always 1 --- cns/restserver/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cns/restserver/util.go b/cns/restserver/util.go index 05352b9f24..ab26569e68 100644 --- a/cns/restserver/util.go +++ b/cns/restserver/util.go @@ -168,8 +168,8 @@ func (service *HTTPRestService) saveNetworkContainerGoalState(req cns.CreateNetw if hostVersion == "" { // Host version is the NC version from NMAgent, set it -1 to indicate no result from NMAgent yet. // TODO, query NMAgent and with aggresive time out and assign latest host version. - hostVersion = "1" } + hostVersion = "1" // Remove the auth token before saving the containerStatus to cns json file createNetworkContainerRequest := req From 407a852b591806502bfcf0c32e44a4590bd4acfa Mon Sep 17 00:00:00 2001 From: keithnguyen Date: Fri, 14 Feb 2025 23:08:02 +0000 Subject: [PATCH 14/16] 3 new fields on NC. --- crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go | 3 +++ .../manifests/acn.azure.com_nodenetworkconfigs.yaml | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go index 1137b28e96..7ded69a387 100644 --- a/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go +++ b/crd/nodenetworkconfig/api/v1alpha/nodenetworkconfig.go @@ -119,9 +119,12 @@ type NetworkContainer struct { // +kubebuilder:default=vnet Type NCType `json:"type,omitempty"` PrimaryIP string `json:"primaryIP,omitempty"` + PrimaryIPV6 string `json:"primaryIPV6,omitempty"` SubnetName string `json:"subnetName,omitempty"` IPAssignments []IPAssignment `json:"ipAssignments,omitempty"` DefaultGateway string `json:"defaultGateway,omitempty"` + DefaultGatewayV6 string `json:"defaultGatewayV6,omitempty"` + MacAddress string `json:"macAddress,omitempty"` SubnetAddressSpace string `json:"subnetAddressSpace,omitempty"` // +kubebuilder:default=0 // +kubebuilder:validation:Optional diff --git a/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml b/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml index 8d7fb5e08e..266fdb17a0 100644 --- a/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml +++ b/crd/nodenetworkconfig/manifests/acn.azure.com_nodenetworkconfigs.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.3 + controller-gen.kubebuilder.io/version: v0.17.2 name: nodenetworkconfigs.acn.azure.com spec: group: acn.azure.com @@ -124,6 +124,8 @@ spec: type: string type: object type: array + macAddress: + type: string nodeIP: type: string primaryIP: From 55ca834b86f90b6b3e94fa3f562d96159724e25c Mon Sep 17 00:00:00 2001 From: keithnguyen Date: Fri, 7 Mar 2025 19:07:33 +0000 Subject: [PATCH 15/16] Hopefully recover this file from master --- cns/middlewares/k8sSwiftV2.go | 401 +++++++++------------------------- 1 file changed, 105 insertions(+), 296 deletions(-) diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index 2df5c18e0f..bb075cc3e0 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -3,6 +3,7 @@ package middlewares import ( "context" "fmt" + "net/netip" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" @@ -11,47 +12,121 @@ import ( "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" - v1 "k8s.io/api/core/v1" - k8stypes "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" ) -var ( - errMTPNCNotReady = errors.New("mtpnc is not ready") - errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") - errInvalidMTPNCPrefixLength = errors.New("invalid prefix length for MTPNC primaryIP, must be 32") -) +// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. +func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { + logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) + var routes []cns.Route -const ( - prefixLength = 32 - overlayGatewayv4 = "169.254.1.1" - virtualGW = "169.254.2.1" - overlayGatewayV6 = "fe80::1234:5678:9abc" -) + switch podIPInfo.NICType { + case cns.DelegatedVMNIC: + virtualGWRoute := cns.Route{ + IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), + } + // default route via SWIFT v2 interface + route := cns.Route{ + IPAddress: "0.0.0.0/0", + GatewayIPAddress: virtualGW, + } + routes = append(routes, virtualGWRoute, route) + + case cns.InfraNIC: + // Linux CNS middleware sets the infra routes(pod, infravnet and service cidrs) to infraNIC interface for the podIPInfo used in SWIFT V2 Linux scenario + infraRoutes, err := k.getInfraRoutes(podIPInfo) + if err != nil { + return errors.Wrap(err, "failed to get infra routes for infraNIC interface") + } + routes = infraRoutes + podIPInfo.SkipDefaultRoutes = true + + case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check + // No-op NIC types. + default: + return errInvalidSWIFTv2NICType + } + + podIPInfo.Routes = routes + return nil +} + +// Linux CNS gets pod CIDRs from configuration env +// Containerd reassigns the IP to the adapter and kernel configures the pod cidr route by default on Windows VM +// Hence the windows swiftv2 scenario does not require pod cidr +// GetPodCidrs() will return v4PodCidrs as first []string and v6PodCidrs as second []string +func (k *K8sSWIFTv2Middleware) GetPodCidrs() ([]string, []string, error) { //nolint + v4PodCidrs := []string{} + v6PodCidrs := []string{} + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs") + } -type K8sSWIFTv2Middleware struct { - Cli client.Client + v4PodCidrs = append(v4PodCidrs, podCIDRsV4...) + v6PodCidrs = append(v6PodCidrs, podCIDRv6...) + + return v4PodCidrs, v6PodCidrs, nil } -// Verify interface compliance at compile time -var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil) +// getInfraRoutes() returns the infra routes including infravnet/pod/service cidrs for the podIPInfo used in SWIFT V2 Linux scenario +// Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway +func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { + var routes []cns.Route + + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } -func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { - // gets pod info for the specified request - podInfo, pod, respCode, message := k.GetPodInfo(ctx, req) - if respCode != types.Success { - return nil, respCode, message + v4IPs, v6IPs, err := k.GetInfravnetAndServiceCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get infravnet and service CIDRs") } - // validates if pod is swiftv2 - isSwiftv2 := ValidateSwiftv2Pod(pod) + v4PodIPs, v6PodIPs, err := k.GetPodCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get pod CIDRs") + } + + v4IPs = append(v4IPs, v4PodIPs...) + v6IPs = append(v6IPs, v6PodIPs...) + + if ip.Is4() { + routes = append(routes, k.AddRoutes(v4IPs, overlayGatewayv4)...) + } else { + routes = append(routes, k.AddRoutes(v6IPs, overlayGatewayV6)...) + } + + return routes, nil +} + +// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient +func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error { + return nil +} + +// add default route is done on setRoutes() for Linux swiftv2 +func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} + +// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request +// and release IP configs handlers. +func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { + return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { + podInfo, respCode, message := k.GetPodInfoForIPConfigsRequest(ctx, &req) - var mtpnc v1alpha1.MultitenantPodNetworkConfig - // if swiftv2 is enabled, get mtpnc - if isSwiftv2 { - mtpnc, respCode, message = k.getMTPNC(ctx, podInfo) if respCode != types.Success { - return nil, respCode, message + return &cns.IPConfigsResponse{ + Response: cns.Response{ + ReturnCode: respCode, + Message: message, + }, + }, errors.New("failed to validate IP configs request") } ipConfigsResp, err := defaultHandler(ctx, req) // If the pod is not v2, return the response from the handler @@ -102,269 +177,3 @@ func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context return ipConfigsResp, nil } } - -// validateIPConfigsRequest validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. -// nolint -func (k *K8sSWIFTv2Middleware) validateIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { - // Retrieve the pod from the cluster - podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) - if err != nil { - errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) - return nil, types.UnexpectedError, errBuf.Error() - } - logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) - podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - pod := v1.Pod{} - if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { - errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) - return nil, types.UnexpectedError, errBuf.Error() - } - - // check the pod labels for Swift V2, set the request's SecondaryInterfaceSet flag to true and check if its MTPNC CRD is ready - _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] - _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] - if swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel { - - // Check if the MTPNC CRD exists for the pod, if not, return error - mtpnc := v1alpha1.MultitenantPodNetworkConfig{} - mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return nil, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() - } - // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if !mtpnc.IsReady() { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() - } - // If primary Ip is set in status field, it indicates the presence of secondary interfaces - if mtpnc.Status.PrimaryIP != "" { - req.SecondaryInterfacesExist = true - } - interfaceInfos := mtpnc.Status.InterfaceInfos - for _, interfaceInfo := range interfaceInfos { - if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { - if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { - return nil, types.UnexpectedError, errMTPNCNotReady.Error() - } - req.BackendInterfaceExist = true - req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) - - } - if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC { - req.SecondaryInterfacesExist = true - } - } - } - logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) - logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist) - - return podInfo, types.Success, "" -} - -// getIPConfig returns the pod's SWIFT V2 IP configuration. -func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { - // Check if the MTPNC CRD exists for the pod, if not, return error - mtpnc := v1alpha1.MultitenantPodNetworkConfig{} - mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return nil, errors.Wrapf(err, "failed to get pod's mtpnc from cache") - } - - // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if !mtpnc.IsReady() { - return nil, errMTPNCNotReady - } - logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) - - var podIPInfos []cns.PodIpInfo - - if len(mtpnc.Status.InterfaceInfos) == 0 { - // Use fields from mtpnc.Status if InterfaceInfos is empty - ip, prefixSize, err := utils.ParseIPAndPrefix(mtpnc.Status.PrimaryIP) - if err != nil { - return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix") - } - if prefixSize != prefixLength { - return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) - } - - podIPInfos = append(podIPInfos, cns.PodIpInfo{ - PodIPConfig: cns.IPSubnet{ - IPAddress: ip, - PrefixLength: uint8(prefixSize), - }, - MacAddress: mtpnc.Status.MacAddress, - NICType: cns.DelegatedVMNIC, - SkipDefaultRoutes: false, - // InterfaceName is empty for DelegatedVMNIC - }) - } else { - for _, interfaceInfo := range mtpnc.Status.InterfaceInfos { - var ( - nicType cns.NICType - ip string - prefixSize int - err error - ) - switch { - case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC: - nicType = cns.DelegatedVMNIC - case interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC: - nicType = cns.NodeNetworkInterfaceBackendNIC - default: - nicType = cns.DelegatedVMNIC - } - if nicType != cns.NodeNetworkInterfaceBackendNIC { - // Parse MTPNC primaryIP to get the IP address and prefix length - ip, prefixSize, err = utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP) - if err != nil { - return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix") - } - if prefixSize != prefixLength { - return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) - } - - podIPInfo := cns.PodIpInfo{ - PodIPConfig: cns.IPSubnet{ - IPAddress: ip, - PrefixLength: uint8(prefixSize), - }, - MacAddress: interfaceInfo.MacAddress, - NICType: nicType, - SkipDefaultRoutes: false, - // InterfaceName is empty for DelegatedVMNIC and AccelnetFrontendNIC - } - // for windows scenario, it is required to add additional fields with the exact subnetAddressSpace - // received from MTPNC, this function assigns them for windows while linux is a no-op - err = k.assignSubnetPrefixLengthFields(&podIPInfo, interfaceInfo, ip) - if err != nil { - return nil, errors.Wrap(err, "failed to parse mtpnc subnetAddressSpace prefix") - } - podIPInfos = append(podIPInfos, podIPInfo) - // for windows scenario, it is required to add default route with gatewayIP from CNS - k.addDefaultRoute(&podIPInfo, interfaceInfo.GatewayIP) - } - } - } - - return podIPInfos, nil -} - -func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode { - return cns.K8sSWIFTV2 -} - -// gets Pod Data -func (k *K8sSWIFTv2Middleware) GetPodInfo(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, k8sPod v1.Pod, respCode types.ResponseCode, message string) { - // Retrieve the pod from the cluster - podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) - if err != nil { - errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) - return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error() - } - logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) - podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - pod := v1.Pod{} - if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { - errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) - return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error() - } - return podInfo, pod, types.Success, "" -} - -// validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. -func ValidateSwiftv2Pod(pod v1.Pod) bool { - // check the pod labels for Swift V2 - _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] - _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] - return swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel -} - -func (k *K8sSWIFTv2Middleware) getMTPNC(ctx context.Context, podInfo cns.PodInfo) (mtpncResource v1alpha1.MultitenantPodNetworkConfig, respCode types.ResponseCode, message string) { - // Check if the MTPNC CRD exists for the pod, if not, return error - mtpnc := v1alpha1.MultitenantPodNetworkConfig{} - mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} - if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { - return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() - } - // Check if the MTPNC CRD is ready. If one of the fields is empty, return error - if !mtpnc.IsReady() { - return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, errMTPNCNotReady.Error() - } - return mtpnc, types.Success, "" -} - -// Updates Ip Config Request -func (k *K8sSWIFTv2Middleware) UpdateIPConfigRequest(mtpnc v1alpha1.MultitenantPodNetworkConfig, req *cns.IPConfigsRequest) ( - respCode types.ResponseCode, - message string, -) { - // If primary Ip is set in status field, it indicates the presence of secondary interfaces - if mtpnc.Status.PrimaryIP != "" { - req.SecondaryInterfacesExist = true - } - - interfaceInfos := mtpnc.Status.InterfaceInfos - for _, interfaceInfo := range interfaceInfos { - if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { - if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { - return types.UnexpectedError, errMTPNCNotReady.Error() - } - req.BackendInterfaceExist = true - req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) - - } - if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC { - req.SecondaryInterfacesExist = true - } - } - - return types.Success, "" -} - -func (k *K8sSWIFTv2Middleware) AddRoutes(cidrs []string, gatewayIP string) []cns.Route { - routes := make([]cns.Route, len(cidrs)) - for i, cidr := range cidrs { - routes[i] = cns.Route{ - IPAddress: cidr, - GatewayIPAddress: gatewayIP, - } - } - return routes -} - -// Both Linux and Windows CNS gets infravnet and service CIDRs from configuration env -// GetInfravnetAndServiceCidrs() returns v4CIDRs(infravnet and service cidrs) as first []string and v6CIDRs(infravnet and service) as second []string -func (k *K8sSWIFTv2Middleware) GetInfravnetAndServiceCidrs() ([]string, []string, error) { //nolint - v4Cidrs := []string{} - v6Cidrs := []string{} - - // Get and parse infraVNETCIDRs from env - infraVNETCIDRs, err := configuration.InfraVNETCIDRs() - if err != nil { - return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env") - } - infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) - if err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs") - } - - // Add infravnet CIDRs to v4 and v6 IPs - v4Cidrs = append(v4Cidrs, infraVNETCIDRsv4...) - v6Cidrs = append(v6Cidrs, infraVNETCIDRsv6...) - - // Get and parse serviceCIDRs from env - serviceCIDRs, err := configuration.ServiceCIDRs() - if err != nil { - return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env") - } - serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) - if err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs") - } - - // Add service CIDRs to v4 and v6 IPs - v4Cidrs = append(v4Cidrs, serviceCIDRsV4...) - v6Cidrs = append(v6Cidrs, serviceCIDRsV6...) - - return v4Cidrs, v6Cidrs, nil -} From 64b4d404d7f60adedaa03927d1a7a3e1d1e38dee Mon Sep 17 00:00:00 2001 From: keithnguyen Date: Fri, 7 Mar 2025 19:09:06 +0000 Subject: [PATCH 16/16] Hopefully recover forced push. --- cns/middlewares/k8sSwiftV2.go | 357 +++++++++++++++++++++------------- 1 file changed, 227 insertions(+), 130 deletions(-) diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index bb075cc3e0..cdfba553dd 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -3,7 +3,6 @@ package middlewares import ( "context" "fmt" - "net/netip" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" @@ -12,168 +11,266 @@ import ( "github.com/Azure/azure-container-networking/cns/types" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// setRoutes sets the routes for podIPInfo used in SWIFT V2 scenario. -func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { - logger.Printf("[SWIFTv2Middleware] set routes for pod with nic type : %s", podIPInfo.NICType) - var routes []cns.Route +var ( + errMTPNCNotReady = errors.New("mtpnc is not ready") + errInvalidSWIFTv2NICType = errors.New("invalid NIC type for SWIFT v2 scenario") + errInvalidMTPNCPrefixLength = errors.New("invalid prefix length for MTPNC primaryIP, must be 32") +) + +const ( + prefixLength = 32 + overlayGatewayv4 = "169.254.1.1" + virtualGW = "169.254.2.1" + overlayGatewayV6 = "fe80::1234:5678:9abc" +) + +type K8sSWIFTv2Middleware struct { + Cli client.Client +} + +// Verify interface compliance at compile time +var _ cns.IPConfigsHandlerMiddleware = (*K8sSWIFTv2Middleware)(nil) + +func (k *K8sSWIFTv2Middleware) GetPodInfoForIPConfigsRequest(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, respCode types.ResponseCode, message string) { + // gets pod info for the specified request + podInfo, pod, respCode, message := k.GetPodInfo(ctx, req) + if respCode != types.Success { + return nil, respCode, message + } + + // validates if pod is swiftv2 + isSwiftv2 := ValidateSwiftv2Pod(pod) - switch podIPInfo.NICType { - case cns.DelegatedVMNIC: - virtualGWRoute := cns.Route{ - IPAddress: fmt.Sprintf("%s/%d", virtualGW, prefixLength), + var mtpnc v1alpha1.MultitenantPodNetworkConfig + // if swiftv2 is enabled, get mtpnc + if isSwiftv2 { + mtpnc, respCode, message = k.getMTPNC(ctx, podInfo) + if respCode != types.Success { + return nil, respCode, message } - // default route via SWIFT v2 interface - route := cns.Route{ - IPAddress: "0.0.0.0/0", - GatewayIPAddress: virtualGW, + + // update ipConfigRequest + respCode, message = k.UpdateIPConfigRequest(mtpnc, req) + if respCode != types.Success { + return nil, respCode, message } - routes = append(routes, virtualGWRoute, route) + } + logger.Printf("[SWIFTv2Middleware] pod %s has secondary interface : %v", podInfo.Name(), req.SecondaryInterfacesExist) + logger.Printf("[SWIFTv2Middleware] pod %s has backend interface : %v", podInfo.Name(), req.BackendInterfaceExist) + + return podInfo, types.Success, "" +} + +// getIPConfig returns the pod's SWIFT V2 IP configuration. +func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodInfo) ([]cns.PodIpInfo, error) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return nil, errors.Wrapf(err, "failed to get pod's mtpnc from cache") + } + + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if !mtpnc.IsReady() { + return nil, errMTPNCNotReady + } + logger.Printf("[SWIFTv2Middleware] mtpnc for pod %s is : %+v", podInfo.Name(), mtpnc) + + var podIPInfos []cns.PodIpInfo - case cns.InfraNIC: - // Linux CNS middleware sets the infra routes(pod, infravnet and service cidrs) to infraNIC interface for the podIPInfo used in SWIFT V2 Linux scenario - infraRoutes, err := k.getInfraRoutes(podIPInfo) + if len(mtpnc.Status.InterfaceInfos) == 0 { + // Use fields from mtpnc.Status if InterfaceInfos is empty + ip, prefixSize, err := utils.ParseIPAndPrefix(mtpnc.Status.PrimaryIP) if err != nil { - return errors.Wrap(err, "failed to get infra routes for infraNIC interface") + return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix") + } + if prefixSize != prefixLength { + return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) } - routes = infraRoutes - podIPInfo.SkipDefaultRoutes = true - case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check - // No-op NIC types. - default: - return errInvalidSWIFTv2NICType + podIPInfos = append(podIPInfos, cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: ip, + PrefixLength: uint8(prefixSize), + }, + MacAddress: mtpnc.Status.MacAddress, + NICType: cns.DelegatedVMNIC, + SkipDefaultRoutes: false, + // InterfaceName is empty for DelegatedVMNIC + }) + } else { + for _, interfaceInfo := range mtpnc.Status.InterfaceInfos { + var ( + nicType cns.NICType + ip string + prefixSize int + err error + ) + switch { + case interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC: + nicType = cns.DelegatedVMNIC + case interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC: + nicType = cns.NodeNetworkInterfaceBackendNIC + default: + nicType = cns.DelegatedVMNIC + } + if nicType != cns.NodeNetworkInterfaceBackendNIC { + // Parse MTPNC primaryIP to get the IP address and prefix length + ip, prefixSize, err = utils.ParseIPAndPrefix(interfaceInfo.PrimaryIP) + if err != nil { + return nil, errors.Wrap(err, "failed to parse mtpnc primary IP and prefix") + } + if prefixSize != prefixLength { + return nil, errors.Wrapf(errInvalidMTPNCPrefixLength, "mtpnc primaryIP prefix length is %d", prefixSize) + } + + podIPInfo := cns.PodIpInfo{ + PodIPConfig: cns.IPSubnet{ + IPAddress: ip, + PrefixLength: uint8(prefixSize), + }, + MacAddress: interfaceInfo.MacAddress, + NICType: nicType, + SkipDefaultRoutes: false, + // InterfaceName is empty for DelegatedVMNIC and AccelnetFrontendNIC + } + // for windows scenario, it is required to add additional fields with the exact subnetAddressSpace + // received from MTPNC, this function assigns them for windows while linux is a no-op + err = k.assignSubnetPrefixLengthFields(&podIPInfo, interfaceInfo, ip) + if err != nil { + return nil, errors.Wrap(err, "failed to parse mtpnc subnetAddressSpace prefix") + } + podIPInfos = append(podIPInfos, podIPInfo) + // for windows scenario, it is required to add default route with gatewayIP from CNS + k.addDefaultRoute(&podIPInfo, interfaceInfo.GatewayIP) + } + } } - podIPInfo.Routes = routes - return nil + return podIPInfos, nil } -// Linux CNS gets pod CIDRs from configuration env -// Containerd reassigns the IP to the adapter and kernel configures the pod cidr route by default on Windows VM -// Hence the windows swiftv2 scenario does not require pod cidr -// GetPodCidrs() will return v4PodCidrs as first []string and v6PodCidrs as second []string -func (k *K8sSWIFTv2Middleware) GetPodCidrs() ([]string, []string, error) { //nolint - v4PodCidrs := []string{} - v6PodCidrs := []string{} +func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode { + return cns.K8sSWIFTV2 +} - // Get and parse podCIDRs from env - podCIDRs, err := configuration.PodCIDRs() +// gets Pod Data +func (k *K8sSWIFTv2Middleware) GetPodInfo(ctx context.Context, req *cns.IPConfigsRequest) (podInfo cns.PodInfo, k8sPod v1.Pod, respCode types.ResponseCode, message string) { + // Retrieve the pod from the cluster + podInfo, err := cns.UnmarshalPodInfo(req.OrchestratorContext) if err != nil { - return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env") + errBuf := errors.Wrapf(err, "failed to unmarshalling pod info from ipconfigs request %+v", req) + return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error() } - podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) - if err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs") + logger.Printf("[SWIFTv2Middleware] validate ipconfigs request for pod %s", podInfo.Name()) + podNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + pod := v1.Pod{} + if err := k.Cli.Get(ctx, podNamespacedName, &pod); err != nil { + errBuf := errors.Wrapf(err, "failed to get pod %+v", podNamespacedName) + return nil, v1.Pod{}, types.UnexpectedError, errBuf.Error() } - - v4PodCidrs = append(v4PodCidrs, podCIDRsV4...) - v6PodCidrs = append(v6PodCidrs, podCIDRv6...) - - return v4PodCidrs, v6PodCidrs, nil + return podInfo, pod, types.Success, "" } -// getInfraRoutes() returns the infra routes including infravnet/pod/service cidrs for the podIPInfo used in SWIFT V2 Linux scenario -// Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway -func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { - var routes []cns.Route +// validates if pod is multitenant by checking the pod labels, used in SWIFT V2 AKS scenario. +func ValidateSwiftv2Pod(pod v1.Pod) bool { + // check the pod labels for Swift V2 + _, swiftV2PodNetworkLabel := pod.Labels[configuration.LabelPodSwiftV2] + _, swiftV2PodNetworkInstanceLabel := pod.Labels[configuration.LabelPodNetworkInstanceSwiftV2] + return swiftV2PodNetworkLabel || swiftV2PodNetworkInstanceLabel +} - ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) - if err != nil { - return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) +func (k *K8sSWIFTv2Middleware) getMTPNC(ctx context.Context, podInfo cns.PodInfo) (mtpncResource v1alpha1.MultitenantPodNetworkConfig, respCode types.ResponseCode, message string) { + // Check if the MTPNC CRD exists for the pod, if not, return error + mtpnc := v1alpha1.MultitenantPodNetworkConfig{} + mtpncNamespacedName := k8stypes.NamespacedName{Namespace: podInfo.Namespace(), Name: podInfo.Name()} + if err := k.Cli.Get(ctx, mtpncNamespacedName, &mtpnc); err != nil { + return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, fmt.Errorf("failed to get pod's mtpnc from cache : %w", err).Error() } - - v4IPs, v6IPs, err := k.GetInfravnetAndServiceCidrs() - if err != nil { - return nil, errors.Wrap(err, "failed to get infravnet and service CIDRs") + // Check if the MTPNC CRD is ready. If one of the fields is empty, return error + if !mtpnc.IsReady() { + return v1alpha1.MultitenantPodNetworkConfig{}, types.UnexpectedError, errMTPNCNotReady.Error() } + return mtpnc, types.Success, "" +} - v4PodIPs, v6PodIPs, err := k.GetPodCidrs() - if err != nil { - return nil, errors.Wrap(err, "failed to get pod CIDRs") +// Updates Ip Config Request +func (k *K8sSWIFTv2Middleware) UpdateIPConfigRequest(mtpnc v1alpha1.MultitenantPodNetworkConfig, req *cns.IPConfigsRequest) ( + respCode types.ResponseCode, + message string, +) { + // If primary Ip is set in status field, it indicates the presence of secondary interfaces + if mtpnc.Status.PrimaryIP != "" { + req.SecondaryInterfacesExist = true } - v4IPs = append(v4IPs, v4PodIPs...) - v6IPs = append(v6IPs, v6PodIPs...) + interfaceInfos := mtpnc.Status.InterfaceInfos + for _, interfaceInfo := range interfaceInfos { + if interfaceInfo.DeviceType == v1alpha1.DeviceTypeInfiniBandNIC { + if interfaceInfo.MacAddress == "" || interfaceInfo.NCID == "" { + return types.UnexpectedError, errMTPNCNotReady.Error() + } + req.BackendInterfaceExist = true + req.BackendInterfaceMacAddresses = append(req.BackendInterfaceMacAddresses, interfaceInfo.MacAddress) - if ip.Is4() { - routes = append(routes, k.AddRoutes(v4IPs, overlayGatewayv4)...) - } else { - routes = append(routes, k.AddRoutes(v6IPs, overlayGatewayV6)...) + } + if interfaceInfo.DeviceType == v1alpha1.DeviceTypeVnetNIC { + req.SecondaryInterfacesExist = true + } } - return routes, nil + return types.Success, "" } -// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient -func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error { - return nil +func (k *K8sSWIFTv2Middleware) AddRoutes(cidrs []string, gatewayIP string) []cns.Route { + routes := make([]cns.Route, len(cidrs)) + for i, cidr := range cidrs { + routes[i] = cns.Route{ + IPAddress: cidr, + GatewayIPAddress: gatewayIP, + } + } + return routes } -// add default route is done on setRoutes() for Linux swiftv2 -func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} +// Both Linux and Windows CNS gets infravnet and service CIDRs from configuration env +// GetInfravnetAndServiceCidrs() returns v4CIDRs(infravnet and service cidrs) as first []string and v6CIDRs(infravnet and service) as second []string +func (k *K8sSWIFTv2Middleware) GetInfravnetAndServiceCidrs() ([]string, []string, error) { //nolint + v4Cidrs := []string{} + v6Cidrs := []string{} -// IPConfigsRequestHandlerWrapper is the middleware function for handling SWIFT v2 IP configs requests for AKS-SWIFT. This function wrapped the default SWIFT request -// and release IP configs handlers. -func (k *K8sSWIFTv2Middleware) IPConfigsRequestHandlerWrapper(defaultHandler, failureHandler cns.IPConfigsHandlerFunc) cns.IPConfigsHandlerFunc { - return func(ctx context.Context, req cns.IPConfigsRequest) (*cns.IPConfigsResponse, error) { - podInfo, respCode, message := k.GetPodInfoForIPConfigsRequest(ctx, &req) + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env") + } + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs") + } - if respCode != types.Success { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: respCode, - Message: message, - }, - }, errors.New("failed to validate IP configs request") - } - ipConfigsResp, err := defaultHandler(ctx, req) - // If the pod is not v2, return the response from the handler - if !req.SecondaryInterfacesExist { - return ipConfigsResp, err - } - // If the pod is v2, get the infra IP configs from the handler first and then add the SWIFTv2 IP config - defer func() { - // Release the default IP config if there is an error - if err != nil { - _, err = failureHandler(ctx, req) - if err != nil { - logger.Errorf("failed to release default IP config : %v", err) - } - } - }() - if err != nil { - return ipConfigsResp, err - } - SWIFTv2PodIPInfos, err := k.getIPConfig(ctx, podInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to get SWIFTv2 IP config : %v", req) - } - ipConfigsResp.PodIPInfo = append(ipConfigsResp.PodIPInfo, SWIFTv2PodIPInfos...) - // Set routes for the pod - for i := range ipConfigsResp.PodIPInfo { - ipInfo := &ipConfigsResp.PodIPInfo[i] - // Backend nics doesn't need routes to be set - if ipInfo.NICType != cns.BackendNIC { - err = k.setRoutes(ipInfo) - if err != nil { - return &cns.IPConfigsResponse{ - Response: cns.Response{ - ReturnCode: types.FailedToAllocateIPConfig, - Message: fmt.Sprintf("AllocateIPConfig failed: %v, IP config request is %v", err, req), - }, - PodIPInfo: []cns.PodIpInfo{}, - }, errors.Wrapf(err, "failed to set routes for pod %s", podInfo.Name()) - } - } - } - return ipConfigsResp, nil + // Add infravnet CIDRs to v4 and v6 IPs + v4Cidrs = append(v4Cidrs, infraVNETCIDRsv4...) + v6Cidrs = append(v6Cidrs, infraVNETCIDRsv6...) + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env") } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs") + } + + // Add service CIDRs to v4 and v6 IPs + v4Cidrs = append(v4Cidrs, serviceCIDRsV4...) + v6Cidrs = append(v6Cidrs, serviceCIDRsV6...) + + return v4Cidrs, v6Cidrs, nil }