diff --git a/cni/network/invoker_cns.go b/cni/network/invoker_cns.go index 02e0ce7351..8216c52412 100644 --- a/cni/network/invoker_cns.go +++ b/cni/network/invoker_cns.go @@ -444,6 +444,7 @@ func configureDefaultAddResult(info *IPResultInfo, addConfig *IPAMAddConfig, add Gw: ncgw, }) } + // if we have multiple infra ip result infos, we effectively append routes and ip configs to that same interface info each time // the host subnet prefix (in ipv4 or ipv6) will always refer to the same interface regardless of which ip result info we look at addResult.interfaceInfo[key] = network.InterfaceInfo{ diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index 0c62c3fa0b..b58d975fa9 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -3,6 +3,7 @@ package middlewares import ( "context" "fmt" + "net/netip" "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/configuration" @@ -247,3 +248,65 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode { return cns.K8sSWIFTV2 } + +func (k *K8sSWIFTv2Middleware) addRoutes(cidrs []string, gatewayIP string) []cns.Route { + routes := make([]cns.Route, len(cidrs)) + for i, cidr := range cidrs { + routes[i] = cns.Route{ + IPAddress: cidr, + GatewayIPAddress: gatewayIP, + } + } + return routes +} + +func (k *K8sSWIFTv2Middleware) SetInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { + var routes []cns.Route + + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + if err != nil { + return nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env") + } + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs") + } + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return nil, errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podCIDRs") + } + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return nil, errors.Wrapf(err, "failed to get serviceCIDRs from env") + } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse serviceCIDRs") + } + + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + + if ip.Is4() { + routes = append(routes, k.addRoutes(podCIDRsV4, overlayGatewayv4)...) + routes = append(routes, k.addRoutes(serviceCIDRsV4, overlayGatewayv4)...) + routes = append(routes, k.addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...) + } else { + routes = append(routes, k.addRoutes(podCIDRv6, overlayGatewayV6)...) + routes = append(routes, k.addRoutes(serviceCIDRsV6, overlayGatewayV6)...) + routes = append(routes, k.addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...) + } + + return routes, nil +} diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index 92835d6263..17caf9a83c 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -2,12 +2,9 @@ package middlewares import ( "fmt" - "net/netip" "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" ) @@ -30,50 +27,11 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { routes = append(routes, virtualGWRoute, route) case cns.InfraNIC: - // Get and parse infraVNETCIDRs from env - infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + infraRoutes, err := k.SetInfraRoutes(podIPInfo) if err != nil { - return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") - } - infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse infraVNETCIDRs") - } - - // Get and parse podCIDRs from env - podCIDRs, err := configuration.PodCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get podCIDRs from env") - } - podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse podCIDRs") - } - - // Get and parse serviceCIDRs from env - serviceCIDRs, err := configuration.ServiceCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get serviceCIDRs from env") - } - serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse serviceCIDRs") - } - - ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) - if err != nil { - return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) - } - - if ip.Is4() { - routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...) - routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...) - routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...) - } else { - routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...) - routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...) - routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...) + return errors.Wrap(err, "failed to set routes for infraNIC interface") } + routes = infraRoutes podIPInfo.SkipDefaultRoutes = true case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check @@ -86,17 +44,6 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { return nil } -func addRoutes(cidrs []string, gatewayIP string) []cns.Route { - routes := make([]cns.Route, len(cidrs)) - for i, cidr := range cidrs { - routes[i] = cns.Route{ - IPAddress: cidr, - GatewayIPAddress: gatewayIP, - } - } - return routes -} - // assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error { return nil diff --git a/cns/middlewares/k8sSwiftV2_linux_test.go b/cns/middlewares/k8sSwiftV2_linux_test.go index 76be6b2149..41ab474f6b 100644 --- a/cns/middlewares/k8sSwiftV2_linux_test.go +++ b/cns/middlewares/k8sSwiftV2_linux_test.go @@ -378,9 +378,10 @@ func TestSetRoutesFailure(t *testing.T) { } func TestAddRoutes(t *testing.T) { + middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"} gatewayIP := "192.168.1.1" - routes := addRoutes(cidrs, gatewayIP) + routes := middleware.addRoutes(cidrs, gatewayIP) expectedRoutes := []cns.Route{ { IPAddress: "10.0.0.0/24", diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index a2b34cd467..b9f4e69faa 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -1,8 +1,9 @@ package middlewares import ( + "fmt" + "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/logger" "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" @@ -12,7 +13,21 @@ import ( // default route is set for secondary interface NIC(i.e,delegatedNIC) func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { if podIPInfo.NICType == cns.InfraNIC { - logger.Printf("[SWIFTv2Middleware] skip setting default route on InfraNIC interface") + // as a workaround, set a default route with gw 0.0.0.0 to avoid HNS setting default route to infraNIC interface + // TODO: remove this once HNS supports custom routes adding to the pod + route := cns.Route{ + IPAddress: "0.0.0.0/0", + GatewayIPAddress: "0.0.0.0", + } + podIPInfo.Routes = append(podIPInfo.Routes, route) + + // add routes for infraNIC + routes, err := k.SetInfraRoutes(podIPInfo) + fmt.Printf("routes are %v", routes) + if err != nil { + return errors.Wrap(err, "failed to set routes for infraNIC interface") + } + podIPInfo.Routes = routes podIPInfo.SkipDefaultRoutes = true } return nil @@ -40,5 +55,12 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(podIPInfo *cns.Pod }, GatewayIPAddress: interfaceInfo.GatewayIP, } + // assign default route + route := cns.Route{ + IPAddress: "0.0.0.0/0", + GatewayIPAddress: interfaceInfo.GatewayIP, + } + podIPInfo.Routes = append(podIPInfo.Routes, route) + return nil } diff --git a/cns/middlewares/k8sSwiftV2_windows_test.go b/cns/middlewares/k8sSwiftV2_windows_test.go index 945d650ae9..d205ef5c6a 100644 --- a/cns/middlewares/k8sSwiftV2_windows_test.go +++ b/cns/middlewares/k8sSwiftV2_windows_test.go @@ -1,9 +1,12 @@ package middlewares import ( + "fmt" + "reflect" "testing" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "gotest.tools/v3/assert" @@ -11,6 +14,9 @@ import ( func TestSetRoutesSuccess(t *testing.T) { middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + t.Setenv(configuration.EnvPodCIDRs, "10.0.1.10/24,16A0:0010:AB00:001E::2/32") + t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16,16A0:0010:AB00:0000::/32") + t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.1/16,16A0:0020:AB00:0000::/32") podIPInfo := []cns.PodIpInfo{ { @@ -53,11 +59,19 @@ func TestAssignSubnetPrefixSuccess(t *testing.T) { MacAddress: "12:34:56:78:9a:bc", } + gatewayIP := "20.240.1.1" intInfo := v1alpha1.InterfaceInfo{ - GatewayIP: "20.240.1.1", + GatewayIP: gatewayIP, SubnetAddressSpace: "20.240.1.0/16", } + routes := []cns.Route{ + { + IPAddress: "0.0.0.0/0", + GatewayIPAddress: gatewayIP, + }, + } + ipInfo := podIPInfo err := middleware.assignSubnetPrefixLengthFields(&ipInfo, intInfo, ipInfo.PodIPConfig.IPAddress) assert.Equal(t, err, nil) @@ -65,4 +79,9 @@ func TestAssignSubnetPrefixSuccess(t *testing.T) { assert.Equal(t, ipInfo.PodIPConfig.PrefixLength, uint8(16)) assert.Equal(t, ipInfo.HostPrimaryIPInfo.Gateway, intInfo.GatewayIP) assert.Equal(t, ipInfo.HostPrimaryIPInfo.Subnet, intInfo.SubnetAddressSpace) + + // compare two slices of routes + if !reflect.DeepEqual(ipInfo.Routes, routes) { + t.Errorf("got '%+v', expected '%+v'", ipInfo.Routes, routes) + } } diff --git a/network/endpoint_windows.go b/network/endpoint_windows.go index edd52327f2..bab378742f 100644 --- a/network/endpoint_windows.go +++ b/network/endpoint_windows.go @@ -428,7 +428,7 @@ func (nw *network) newEndpointImplHnsV2(cli apipaClient, epInfo *EndpointInfo) ( } // Create the HCN endpoint. - logger.Info("Creating hcn endpoint", zap.Any("hcnEndpoint", hcnEndpoint), zap.String("computenetwork", hcnEndpoint.HostComputeNetwork)) + logger.Info("Creating hcn endpoint", zap.Any("hcnEndpoint", hcnEndpoint), zap.String("computenetwork", hcnEndpoint.HostComputeNetwork), zap.Any("routes", hcnEndpoint.Routes)) hnsResponse, err := Hnsv2.CreateEndpoint(hcnEndpoint) if err != nil { return nil, fmt.Errorf("Failed to create endpoint: %s due to error: %v", hcnEndpoint.Name, err)