diff --git a/cns/middlewares/k8sSwiftV2.go b/cns/middlewares/k8sSwiftV2.go index a11290c205..4cb3a8f2e0 100644 --- a/cns/middlewares/k8sSwiftV2.go +++ b/cns/middlewares/k8sSwiftV2.go @@ -249,3 +249,59 @@ func (k *K8sSWIFTv2Middleware) getIPConfig(ctx context.Context, podInfo cns.PodI func (k *K8sSWIFTv2Middleware) Type() cns.SWIFTV2Mode { return cns.K8sSWIFTV2 } + +// CNS gets pod CIDRs from configuration env and parse them to get the v4 and v6 IPs +// Containerd reassigns the IP to the adapter and kernel configures the pod cidr route by default, so windows swiftv2 does not require pod cidr +func (k *K8sSWIFTv2Middleware) GetPodCidrs() (v4IPs, v6IPs []string, err error) { + v4IPs = []string{} + v6IPs = []string{} + + // Get and parse podCIDRs from env + podCIDRs, err := configuration.PodCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env") + } + podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs") + } + + v4IPs = append(v4IPs, podCIDRsV4...) + v6IPs = append(v6IPs, podCIDRv6...) + + return v4IPs, v6IPs, nil +} + +// CNS gets node and service CIDRs from configuration env and parse them to get the v4 and v6 IPs +func (k *K8sSWIFTv2Middleware) GetCidrs() (v4IPs, v6IPs []string, err error) { + v4IPs = []string{} + v6IPs = []string{} + + // Get and parse infraVNETCIDRs from env + infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env") + } + infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs") + } + + // Get and parse serviceCIDRs from env + serviceCIDRs, err := configuration.ServiceCIDRs() + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env") + } + serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) + if err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs") + } + + v4IPs = append(v4IPs, infraVNETCIDRsv4...) + v4IPs = append(v4IPs, serviceCIDRsV4...) + + v6IPs = append(v6IPs, infraVNETCIDRsv6...) + v6IPs = append(v6IPs, serviceCIDRsV6...) + + return v4IPs, v6IPs, nil +} diff --git a/cns/middlewares/k8sSwiftV2_linux.go b/cns/middlewares/k8sSwiftV2_linux.go index e9a93de0e2..23dff175b3 100644 --- a/cns/middlewares/k8sSwiftV2_linux.go +++ b/cns/middlewares/k8sSwiftV2_linux.go @@ -5,9 +5,7 @@ import ( "net/netip" "github.com/Azure/azure-container-networking/cns" - "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/logger" - "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" ) @@ -30,50 +28,12 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { routes = append(routes, virtualGWRoute, route) case cns.InfraNIC: - // Get and parse infraVNETCIDRs from env - infraVNETCIDRs, err := configuration.InfraVNETCIDRs() + // Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway + infraRoutes, err := k.getInfraRoutes(podIPInfo) if err != nil { - return errors.Wrapf(err, "failed to get infraVNETCIDRs from env") - } - infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse infraVNETCIDRs") - } - - // Get and parse podCIDRs from env - podCIDRs, err := configuration.PodCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get podCIDRs from env") - } - podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse podCIDRs") - } - - // Get and parse serviceCIDRs from env - serviceCIDRs, err := configuration.ServiceCIDRs() - if err != nil { - return errors.Wrapf(err, "failed to get serviceCIDRs from env") - } - serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs) - if err != nil { - return errors.Wrapf(err, "failed to parse serviceCIDRs") - } - - ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) - if err != nil { - return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) - } - - if ip.Is4() { - routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...) - routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...) - routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...) - } else { - routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...) - routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...) - routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...) + return errors.Wrap(err, "failed to set routes for infraNIC interface") } + routes = infraRoutes podIPInfo.SkipDefaultRoutes = true case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check @@ -86,7 +46,14 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { return nil } -func addRoutes(cidrs []string, gatewayIP string) []cns.Route { +// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient +func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error { + return nil +} + +func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} + +func (k *K8sSWIFTv2Middleware) addRoutes(cidrs []string, gatewayIP string) []cns.Route { routes := make([]cns.Route, len(cidrs)) for i, cidr := range cidrs { routes[i] = cns.Route{ @@ -97,9 +64,32 @@ func addRoutes(cidrs []string, gatewayIP string) []cns.Route { return routes } -// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient -func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(_ *cns.PodIpInfo, _ v1alpha1.InterfaceInfo, _ string) error { - return nil -} +func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { + var routes []cns.Route -func (k *K8sSWIFTv2Middleware) addDefaultRoute(*cns.PodIpInfo, string) {} + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + + v4IPs, v6IPs, err := k.GetCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get node and service CIDRs") + } + + v4PodIPs, v6PodIPs, err := k.GetPodCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get pod CIDRs") + } + + v4IPs = append(v4IPs, v4PodIPs...) + v6IPs = append(v6IPs, v6PodIPs...) + + if ip.Is4() { + routes = append(routes, k.addRoutes(v4IPs, overlayGatewayv4)...) + } else { + routes = append(routes, k.addRoutes(v6IPs, overlayGatewayV6)...) + } + + return routes, nil +} diff --git a/cns/middlewares/k8sSwiftV2_linux_test.go b/cns/middlewares/k8sSwiftV2_linux_test.go index 76be6b2149..ede3a9a92b 100644 --- a/cns/middlewares/k8sSwiftV2_linux_test.go +++ b/cns/middlewares/k8sSwiftV2_linux_test.go @@ -3,6 +3,7 @@ package middlewares import ( "context" "fmt" + "reflect" "testing" "github.com/Azure/azure-container-networking/cns" @@ -342,10 +343,10 @@ func TestSetRoutesSuccess(t *testing.T) { } else { assert.Equal(t, ipInfo.SkipDefaultRoutes, false) } - } + for i := range podIPInfo { - assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) + reflect.DeepEqual(podIPInfo[i].Routes, desiredPodIPInfo[i].Routes) } } @@ -378,9 +379,10 @@ func TestSetRoutesFailure(t *testing.T) { } func TestAddRoutes(t *testing.T) { + middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"} gatewayIP := "192.168.1.1" - routes := addRoutes(cidrs, gatewayIP) + routes := middleware.addRoutes(cidrs, gatewayIP) expectedRoutes := []cns.Route{ { IPAddress: "10.0.0.0/24", diff --git a/cns/middlewares/k8sSwiftV2_windows.go b/cns/middlewares/k8sSwiftV2_windows.go index 2be2fbd1df..1aef392723 100644 --- a/cns/middlewares/k8sSwiftV2_windows.go +++ b/cns/middlewares/k8sSwiftV2_windows.go @@ -1,12 +1,18 @@ package middlewares import ( + "net/netip" + "github.com/Azure/azure-container-networking/cns" "github.com/Azure/azure-container-networking/cns/middlewares/utils" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/pkg/errors" ) +const ( + defaultGateway = "0.0.0.0" +) + // for AKS L1VH, do not set default route on infraNIC to avoid customer pod reaching all infra vnet services // default route is set for secondary interface NIC(i.e,delegatedNIC) func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { @@ -16,10 +22,17 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error { // TODO: Remove this once HNS fix is ready route := cns.Route{ IPAddress: "0.0.0.0/0", - GatewayIPAddress: "0.0.0.0", + GatewayIPAddress: defaultGateway, } podIPInfo.Routes = append(podIPInfo.Routes, route) + // set routes(pod/node/service cidrs) for infraNIC interface + // Swiftv2 Windows does not support IPv6 + infraRoutes, err := k.getInfraRoutes(podIPInfo) + if err != nil { + return errors.Wrap(err, "failed to set routes for infraNIC interface") + } + podIPInfo.Routes = append(podIPInfo.Routes, infraRoutes...) podIPInfo.SkipDefaultRoutes = true } return nil @@ -50,11 +63,47 @@ func (k *K8sSWIFTv2Middleware) assignSubnetPrefixLengthFields(podIPInfo *cns.Pod return nil } -// add default route with gateway IP to podIPInfo -func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gwIP string) { +// add default route with gateway IP to podIPInfo for delegated interface +func (k *K8sSWIFTv2Middleware) addDefaultRoute(podIPInfo *cns.PodIpInfo, gatewayIP string) { route := cns.Route{ IPAddress: "0.0.0.0/0", - GatewayIPAddress: gwIP, + GatewayIPAddress: gatewayIP, } podIPInfo.Routes = append(podIPInfo.Routes, route) } + +// add routes to podIPInfo for the given CIDRs and gateway IP +// always use default gateway IP for containerd to configure routes; +// containerd will set route with default gateway ip like 10.0.0.0/16 via 0.0.0.0 dev eth0 +func (k *K8sSWIFTv2Middleware) addRoutes(cidrs []string) []cns.Route { + routes := make([]cns.Route, len(cidrs)) + for i, cidr := range cidrs { + routes[i] = cns.Route{ + IPAddress: cidr, + GatewayIPAddress: defaultGateway, + } + } + return routes +} + +func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) { + var routes []cns.Route + + ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress) + } + + v4IPs, v6IPs, err := k.GetCidrs() + if err != nil { + return nil, errors.Wrap(err, "failed to get CIDRs") + } + + if ip.Is4() { + routes = append(routes, k.addRoutes(v4IPs)...) + } else { + routes = append(routes, k.addRoutes(v6IPs)...) + } + + return routes, nil +} diff --git a/cns/middlewares/k8sSwiftV2_windows_test.go b/cns/middlewares/k8sSwiftV2_windows_test.go index dab24685f9..799bce675b 100644 --- a/cns/middlewares/k8sSwiftV2_windows_test.go +++ b/cns/middlewares/k8sSwiftV2_windows_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/configuration" "github.com/Azure/azure-container-networking/cns/middlewares/mock" "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "gotest.tools/v3/assert" @@ -12,11 +13,13 @@ import ( func TestSetRoutesSuccess(t *testing.T) { middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()} + t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16") + t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.10/16") podIPInfo := []cns.PodIpInfo{ { PodIPConfig: cns.IPSubnet{ - IPAddress: "10.0.1.10", + IPAddress: "10.0.1.100", PrefixLength: 32, }, NICType: cns.InfraNIC, @@ -30,6 +33,30 @@ func TestSetRoutesSuccess(t *testing.T) { MacAddress: "12:34:56:78:9a:bc", }, } + desiredPodIPInfo := []cns.PodIpInfo{ + { + PodIPConfig: cns.IPSubnet{ + IPAddress: "10.0.1.100", + PrefixLength: 32, + }, + NICType: cns.InfraNIC, + Routes: []cns.Route{ + { + IPAddress: "10.0.0.0/16", + GatewayIPAddress: "0.0.0.0", + }, + { + IPAddress: "10.240.0.10/16", + GatewayIPAddress: "0.0.0.0", + }, + { + IPAddress: "0.0.0.0/0", + GatewayIPAddress: "0.0.0.0", + }, + }, + }, + } + for i := range podIPInfo { ipInfo := &podIPInfo[i] err := middleware.setRoutes(ipInfo) @@ -40,6 +67,9 @@ func TestSetRoutesSuccess(t *testing.T) { assert.Equal(t, ipInfo.SkipDefaultRoutes, false) } } + + // check if the routes are set as expected + reflect.DeepEqual(podIPInfo[0].Routes, desiredPodIPInfo[0].Routes) } func TestAssignSubnetPrefixSuccess(t *testing.T) {