Skip to content

Commit 2e4b8d9

Browse files
paulyufan2paulyufan2
committed
fix swiftv2 route issues
1 parent 9023374 commit 2e4b8d9

File tree

5 files changed

+172
-54
lines changed

5 files changed

+172
-54
lines changed

‎cns/middlewares/k8sSwiftV2.go‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,3 +226,50 @@ func (k *K8sSWIFTv2Middleware) UpdateIPConfigRequest(mtpnc v1alpha1.MultitenantP
226226

227227
return types.Success, ""
228228
}
229+
230+
func (k *K8sSWIFTv2Middleware) AddRoutes(cidrs []string, gatewayIP string) []cns.Route {
231+
routes := make([]cns.Route, len(cidrs))
232+
for i, cidr := range cidrs {
233+
routes[i] = cns.Route{
234+
IPAddress: cidr,
235+
GatewayIPAddress: gatewayIP,
236+
}
237+
}
238+
return routes
239+
}
240+
241+
// CNS gets node and service CIDRs from configuration env and parse them to get the v4 and v6 IPs
242+
func (k *K8sSWIFTv2Middleware) GetCidrs() (v4IPs, v6IPs []string, err error) {
243+
v4Cidrs := []string{}
244+
v6Cidrs := []string{}
245+
246+
// Get and parse infraVNETCIDRs from env
247+
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
248+
if err != nil {
249+
return nil, nil, errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
250+
}
251+
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
252+
if err != nil {
253+
return nil, nil, errors.Wrapf(err, "failed to parse infraVNETCIDRs")
254+
}
255+
256+
// Add infravnet CIDRs to v4 and v6 IPs
257+
v4Cidrs = append(v4Cidrs, infraVNETCIDRsv4...)
258+
v6Cidrs = append(v6Cidrs, infraVNETCIDRsv6...)
259+
260+
// Get and parse serviceCIDRs from env
261+
serviceCIDRs, err := configuration.ServiceCIDRs()
262+
if err != nil {
263+
return nil, nil, errors.Wrapf(err, "failed to get serviceCIDRs from env")
264+
}
265+
serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
266+
if err != nil {
267+
return nil, nil, errors.Wrapf(err, "failed to parse serviceCIDRs")
268+
}
269+
270+
// Add service CIDRs to v4 and v6 IPs
271+
v4Cidrs = append(v4Cidrs, serviceCIDRsV4...)
272+
v6Cidrs = append(v6Cidrs, serviceCIDRsV6...)
273+
274+
return v4Cidrs, v6Cidrs, nil
275+
}

‎cns/middlewares/k8sSwiftV2_linux.go‎

Lines changed: 54 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -32,50 +32,12 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
3232
routes = append(routes, virtualGWRoute, route)
3333

3434
case cns.InfraNIC:
35-
// Get and parse infraVNETCIDRs from env
36-
infraVNETCIDRs, err := configuration.InfraVNETCIDRs()
35+
// get service and infravnet routes
36+
infraRoutes, err := k.getInfraRoutes(podIPInfo)
3737
if err != nil {
38-
return errors.Wrapf(err, "failed to get infraVNETCIDRs from env")
39-
}
40-
infraVNETCIDRsv4, infraVNETCIDRsv6, err := utils.ParseCIDRs(infraVNETCIDRs)
41-
if err != nil {
42-
return errors.Wrapf(err, "failed to parse infraVNETCIDRs")
43-
}
44-
45-
// Get and parse podCIDRs from env
46-
podCIDRs, err := configuration.PodCIDRs()
47-
if err != nil {
48-
return errors.Wrapf(err, "failed to get podCIDRs from env")
49-
}
50-
podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
51-
if err != nil {
52-
return errors.Wrapf(err, "failed to parse podCIDRs")
53-
}
54-
55-
// Get and parse serviceCIDRs from env
56-
serviceCIDRs, err := configuration.ServiceCIDRs()
57-
if err != nil {
58-
return errors.Wrapf(err, "failed to get serviceCIDRs from env")
59-
}
60-
serviceCIDRsV4, serviceCIDRsV6, err := utils.ParseCIDRs(serviceCIDRs)
61-
if err != nil {
62-
return errors.Wrapf(err, "failed to parse serviceCIDRs")
63-
}
64-
65-
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
66-
if err != nil {
67-
return errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
68-
}
69-
70-
if ip.Is4() {
71-
routes = append(routes, addRoutes(podCIDRsV4, overlayGatewayv4)...)
72-
routes = append(routes, addRoutes(serviceCIDRsV4, overlayGatewayv4)...)
73-
routes = append(routes, addRoutes(infraVNETCIDRsv4, overlayGatewayv4)...)
74-
} else {
75-
routes = append(routes, addRoutes(podCIDRv6, overlayGatewayV6)...)
76-
routes = append(routes, addRoutes(serviceCIDRsV6, overlayGatewayV6)...)
77-
routes = append(routes, addRoutes(infraVNETCIDRsv6, overlayGatewayV6)...)
38+
return errors.Wrap(err, "failed to get infra routes for infraNIC interface")
7839
}
40+
routes = infraRoutes
7941
podIPInfo.SkipDefaultRoutes = true
8042

8143
case cns.NodeNetworkInterfaceBackendNIC: //nolint:exhaustive // ignore exhaustive types check
@@ -88,15 +50,57 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
8850
return nil
8951
}
9052

91-
func addRoutes(cidrs []string, gatewayIP string) []cns.Route {
92-
routes := make([]cns.Route, len(cidrs))
93-
for i, cidr := range cidrs {
94-
routes[i] = cns.Route{
95-
IPAddress: cidr,
96-
GatewayIPAddress: gatewayIP,
97-
}
53+
// CNS gets pod CIDRs from configuration env and parse them to get the v4 and v6 IPs
54+
// Containerd reassigns the IP to the adapter and kernel configures the pod cidr route by default, so windows swiftv2 does not require pod cidr
55+
func (k *K8sSWIFTv2Middleware) GetPodCidrs() (v4IPs, v6IPs []string, err error) {
56+
v4PodCidrs := []string{}
57+
v6PodCidrs := []string{}
58+
59+
// Get and parse podCIDRs from env
60+
podCIDRs, err := configuration.PodCIDRs()
61+
if err != nil {
62+
return nil, nil, errors.Wrapf(err, "failed to get podCIDRs from env")
63+
}
64+
podCIDRsV4, podCIDRv6, err := utils.ParseCIDRs(podCIDRs)
65+
if err != nil {
66+
return nil, nil, errors.Wrapf(err, "failed to parse podCIDRs")
67+
}
68+
69+
v4PodCidrs = append(v4PodCidrs, podCIDRsV4...)
70+
v6PodCidrs = append(v6PodCidrs, podCIDRv6...)
71+
72+
return v4PodCidrs, v6PodCidrs, nil
73+
}
74+
75+
func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) {
76+
var routes []cns.Route
77+
78+
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
79+
if err != nil {
80+
return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
9881
}
99-
return routes
82+
83+
v4IPs, v6IPs, err := k.GetCidrs()
84+
if err != nil {
85+
return nil, errors.Wrap(err, "failed to get node and service CIDRs")
86+
}
87+
88+
v4PodIPs, v6PodIPs, err := k.GetPodCidrs()
89+
if err != nil {
90+
return nil, errors.Wrap(err, "failed to get pod CIDRs")
91+
}
92+
93+
v4IPs = append(v4IPs, v4PodIPs...)
94+
v6IPs = append(v6IPs, v6PodIPs...)
95+
96+
// Linux uses 169.254.1.1 as the default ipv4 gateway and fe80::1234:5678:9abc as the default ipv6 gateway
97+
if ip.Is4() {
98+
routes = append(routes, k.AddRoutes(v4IPs, overlayGatewayv4)...)
99+
} else {
100+
routes = append(routes, k.AddRoutes(v6IPs, overlayGatewayV6)...)
101+
}
102+
103+
return routes, nil
100104
}
101105

102106
// assignSubnetPrefixLengthFields is a no-op for linux swiftv2 as the default prefix-length is sufficient

‎cns/middlewares/k8sSwiftV2_linux_test.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"testing"
7+
"reflect"
78

89
"github.com/Azure/azure-container-networking/cns"
910
"github.com/Azure/azure-container-networking/cns/configuration"
@@ -345,7 +346,7 @@ func TestSetRoutesSuccess(t *testing.T) {
345346

346347
}
347348
for i := range podIPInfo {
348-
assert.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes)
349+
reflect.DeepEqual(t, podIPInfo[i].Routes, desiredPodIPInfo[i].Routes)
349350
}
350351
}
351352

@@ -378,9 +379,10 @@ func TestSetRoutesFailure(t *testing.T) {
378379
}
379380

380381
func TestAddRoutes(t *testing.T) {
382+
middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()}
381383
cidrs := []string{"10.0.0.0/24", "20.0.0.0/24"}
382384
gatewayIP := "192.168.1.1"
383-
routes := addRoutes(cidrs, gatewayIP)
385+
routes := middleware.addRoutes(cidrs, gatewayIP)
384386
expectedRoutes := []cns.Route{
385387
{
386388
IPAddress: "10.0.0.0/24",

‎cns/middlewares/k8sSwiftV2_windows.go‎

Lines changed: 36 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"encoding/json"
66
"fmt"
7+
"net/netip"
78

89
"github.com/Azure/azure-container-networking/cns"
910
"github.com/Azure/azure-container-networking/cns/logger"
@@ -18,6 +19,10 @@ var defaultDenyEgressPolicy policy.Policy = mustGetEndpointPolicy(cns.DirectionT
1819

1920
var defaultDenyIngressPolicy policy.Policy = mustGetEndpointPolicy(cns.DirectionTypeIn)
2021

22+
const (
23+
defaultGateway = "0.0.0.0"
24+
)
25+
2126
// for AKS L1VH, do not set default route on infraNIC to avoid customer pod reaching all infra vnet services
2227
// default route is set for secondary interface NIC(i.e,delegatedNIC)
2328
func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
@@ -27,10 +32,16 @@ func (k *K8sSWIFTv2Middleware) setRoutes(podIPInfo *cns.PodIpInfo) error {
2732
// TODO: Remove this once HNS fix is ready
2833
route := cns.Route{
2934
IPAddress: "0.0.0.0/0",
30-
GatewayIPAddress: "0.0.0.0",
35+
GatewayIPAddress: defaultGateway,
3136
}
3237
podIPInfo.Routes = append(podIPInfo.Routes, route)
3338

39+
// set routes(infravnet and service cidrs) for infraNIC interface
40+
infraRoutes, err := k.getInfraRoutes(podIPInfo)
41+
if err != nil {
42+
return errors.Wrap(err, "failed to set routes for infraNIC interface")
43+
}
44+
podIPInfo.Routes = append(podIPInfo.Routes, infraRoutes...)
3445
podIPInfo.SkipDefaultRoutes = true
3546
}
3647
return nil
@@ -208,3 +219,27 @@ func GetDefaultDenyBool(mtpnc v1alpha1.MultitenantPodNetworkConfig) bool {
208219
// returns the value of DefaultDenyACL from mtpnc
209220
return mtpnc.Status.DefaultDenyACL
210221
}
222+
223+
func (k *K8sSWIFTv2Middleware) getInfraRoutes(podIPInfo *cns.PodIpInfo) ([]cns.Route, error) {
224+
var routes []cns.Route
225+
226+
ip, err := netip.ParseAddr(podIPInfo.PodIPConfig.IPAddress)
227+
if err != nil {
228+
return nil, errors.Wrapf(err, "failed to parse podIPConfig IP address %s", podIPInfo.PodIPConfig.IPAddress)
229+
}
230+
231+
// swiftv2 windows does not support ipv6
232+
v4IPs, _, err := k.GetCidrs()
233+
if err != nil {
234+
return nil, errors.Wrap(err, "failed to get CIDRs")
235+
}
236+
237+
if ip.Is4() {
238+
// add routes to podIPInfo for the given CIDRs and gateway IP
239+
// always use default gateway IP for containerd to configure routes;
240+
// containerd will set route with default gateway ip like 10.0.0.0/16 via 0.0.0.0 dev eth0
241+
routes = append(routes, k.AddRoutes(v4IPs, defaultGateway)...)
242+
}
243+
244+
return routes, nil
245+
}

‎cns/middlewares/k8sSwiftV2_windows_test.go‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"testing"
88

99
"github.com/Azure/azure-container-networking/cns"
10+
"github.com/Azure/azure-container-networking/cns/configuration"
1011
"github.com/Azure/azure-container-networking/cns/middlewares/mock"
1112
"github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1"
1213
"github.com/Azure/azure-container-networking/network/policy"
@@ -17,11 +18,13 @@ import (
1718

1819
func TestSetRoutesSuccess(t *testing.T) {
1920
middleware := K8sSWIFTv2Middleware{Cli: mock.NewClient()}
21+
t.Setenv(configuration.EnvServiceCIDRs, "10.0.0.0/16")
22+
t.Setenv(configuration.EnvInfraVNETCIDRs, "10.240.0.10/16")
2023

2124
podIPInfo := []cns.PodIpInfo{
2225
{
2326
PodIPConfig: cns.IPSubnet{
24-
IPAddress: "10.0.1.10",
27+
IPAddress: "10.0.1.100",
2528
PrefixLength: 32,
2629
},
2730
NICType: cns.InfraNIC,
@@ -35,6 +38,30 @@ func TestSetRoutesSuccess(t *testing.T) {
3538
MacAddress: "12:34:56:78:9a:bc",
3639
},
3740
}
41+
42+
desiredPodIPInfo := []cns.PodIpInfo{
43+
{
44+
PodIPConfig: cns.IPSubnet{
45+
IPAddress: "10.0.1.100",
46+
PrefixLength: 32,
47+
},
48+
NICType: cns.InfraNIC,
49+
Routes: []cns.Route{
50+
{
51+
IPAddress: "10.0.0.0/16",
52+
GatewayIPAddress: "0.0.0.0",
53+
},
54+
{
55+
IPAddress: "10.240.0.10/16",
56+
GatewayIPAddress: "0.0.0.0",
57+
},
58+
{
59+
IPAddress: "0.0.0.0/0",
60+
GatewayIPAddress: "0.0.0.0",
61+
},
62+
},
63+
},
64+
}
3865
for i := range podIPInfo {
3966
ipInfo := &podIPInfo[i]
4067
err := middleware.setRoutes(ipInfo)
@@ -45,6 +72,9 @@ func TestSetRoutesSuccess(t *testing.T) {
4572
assert.Equal(t, ipInfo.SkipDefaultRoutes, false)
4673
}
4774
}
75+
76+
// check if the routes are set as expected
77+
reflect.DeepEqual(podIPInfo[0].Routes, desiredPodIPInfo[0].Routes)
4878
}
4979

5080
func TestAssignSubnetPrefixSuccess(t *testing.T) {

0 commit comments

Comments
 (0)