Skip to content

Commit 182ba9c

Browse files
yboaronyboaron
committed
Unit tests for node ingress snat exclude annotation
Signed-off-by: Yossi Boaron <[email protected]>
1 parent 7077765 commit 182ba9c

File tree

5 files changed

+500
-3
lines changed

5 files changed

+500
-3
lines changed

‎go-controller/pkg/node/default_node_network_controller_test.go‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"fmt"
66
"net"
7+
"strings"
78
"sync"
89
"time"
910

@@ -21,6 +22,7 @@ import (
2122
adminpolicybasedrouteclient "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/clientset/versioned/fake"
2223
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/factory"
2324
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/kube/mocks"
25+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/managementport"
2426
nodenft "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/nftables"
2527
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager"
2628
ovntest "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/testing"
@@ -1238,4 +1240,325 @@ add element inet ovn-kubernetes no-pmtud-remote-node-ips-v6 { 2002:db8:1::4 }
12381240

12391241
})
12401242

1243+
Describe("node ingress snat exclude subnets", func() {
1244+
1245+
var (
1246+
testNS ns.NetNS
1247+
nc *DefaultNodeNetworkController
1248+
app *cli.App
1249+
)
1250+
1251+
const (
1252+
nodeName = "my-node"
1253+
)
1254+
1255+
BeforeEach(func() {
1256+
var err error
1257+
testNS, err = testutils.NewNS()
1258+
Expect(err).NotTo(HaveOccurred())
1259+
Expect(config.PrepareTestConfig()).To(Succeed())
1260+
1261+
app = cli.NewApp()
1262+
app.Name = "test"
1263+
app.Flags = config.Flags
1264+
})
1265+
1266+
AfterEach(func() {
1267+
util.ResetNetLinkOpMockInst() // other tests in this package rely directly on netlink (e.g. gateway_init_linux_test.go)
1268+
Expect(testNS.Close()).To(Succeed())
1269+
})
1270+
1271+
Context("with a cluster in IPv4 mode", func() {
1272+
const (
1273+
ethName string = "lo1337"
1274+
nodeIP string = "169.254.254.60"
1275+
ethCIDR string = nodeIP + "/24"
1276+
)
1277+
var link netlink.Link
1278+
1279+
BeforeEach(func() {
1280+
config.IPv4Mode = true
1281+
config.IPv6Mode = false
1282+
config.Gateway.Mode = config.GatewayModeShared
1283+
1284+
// Note we must do this in default netNS because
1285+
// nc.WatchNodes() will spawn goroutines which we cannot lock to the testNS
1286+
ovntest.AddLink(ethName)
1287+
1288+
var err error
1289+
link, err = netlink.LinkByName(ethName)
1290+
Expect(err).NotTo(HaveOccurred())
1291+
err = netlink.LinkSetUp(link)
1292+
Expect(err).NotTo(HaveOccurred())
1293+
1294+
// Add an IP address
1295+
addr, err := netlink.ParseAddr(ethCIDR)
1296+
Expect(err).NotTo(HaveOccurred())
1297+
addr.Scope = int(netlink.SCOPE_UNIVERSE)
1298+
err = netlink.AddrAdd(link, addr)
1299+
Expect(err).NotTo(HaveOccurred())
1300+
1301+
})
1302+
1303+
AfterEach(func() {
1304+
err := netlink.LinkDel(link)
1305+
Expect(err).NotTo(HaveOccurred())
1306+
})
1307+
1308+
ovntest.OnSupportedPlatformsIt("empty annotation on startup", func() {
1309+
1310+
app.Action = func(_ *cli.Context) error {
1311+
node := corev1.Node{
1312+
ObjectMeta: metav1.ObjectMeta{
1313+
Name: nodeName,
1314+
Annotations: map[string]string{},
1315+
},
1316+
Status: corev1.NodeStatus{
1317+
Addresses: []corev1.NodeAddress{
1318+
{
1319+
Type: corev1.NodeInternalIP,
1320+
Address: nodeIP,
1321+
},
1322+
},
1323+
},
1324+
}
1325+
1326+
nft := nodenft.SetFakeNFTablesHelper()
1327+
1328+
kubeFakeClient := fake.NewSimpleClientset(&corev1.NodeList{
1329+
Items: []corev1.Node{node},
1330+
})
1331+
fakeClient := &util.OVNNodeClientset{
1332+
KubeClient: kubeFakeClient,
1333+
AdminPolicyRouteClient: adminpolicybasedrouteclient.NewSimpleClientset(),
1334+
NetworkAttchDefClient: nadfake.NewSimpleClientset(),
1335+
}
1336+
1337+
stop := make(chan struct{})
1338+
wf, err := factory.NewNodeWatchFactory(fakeClient, nodeName)
1339+
Expect(err).NotTo(HaveOccurred())
1340+
wg := &sync.WaitGroup{}
1341+
defer func() {
1342+
close(stop)
1343+
wg.Wait()
1344+
wf.Shutdown()
1345+
}()
1346+
1347+
err = wf.Start()
1348+
Expect(err).NotTo(HaveOccurred())
1349+
routeManager := routemanager.NewController()
1350+
cnnci := NewCommonNodeNetworkControllerInfo(kubeFakeClient, fakeClient.AdminPolicyRouteClient, wf, nil, nodeName, routeManager)
1351+
nc = newDefaultNodeNetworkController(cnnci, stop, wg, routeManager, nil)
1352+
nc.initRetryFrameworkForNode()
1353+
err = setupPMTUDNFTSets()
1354+
Expect(err).NotTo(HaveOccurred())
1355+
err = setupPMTUDNFTChain()
1356+
Expect(err).NotTo(HaveOccurred())
1357+
defaultNetConfig := &bridgeUDNConfiguration{
1358+
ofPortPatch: "patch-breth0_ov",
1359+
}
1360+
nc.Gateway = &gateway{
1361+
openflowManager: &openflowManager{
1362+
flowCache: map[string][]string{},
1363+
defaultBridge: &bridgeConfiguration{
1364+
netConfig: map[string]*bridgeUDNConfiguration{
1365+
types.DefaultNetworkName: defaultNetConfig,
1366+
},
1367+
},
1368+
},
1369+
}
1370+
1371+
err = managementport.SetupManagementPortNFTSets()
1372+
Expect(err).NotTo(HaveOccurred())
1373+
1374+
// must run route manager manually which is usually started with nc.Start()
1375+
wg.Add(1)
1376+
go func() {
1377+
defer GinkgoRecover()
1378+
defer wg.Done()
1379+
nc.routeManager.Run(stop, 10*time.Second)
1380+
Expect(err).NotTo(HaveOccurred())
1381+
}()
1382+
By("no nftables elements should present at startup")
1383+
1384+
err = nc.WatchNodes()
1385+
Expect(err).NotTo(HaveOccurred())
1386+
Expect(nft.Dump()).NotTo(ContainSubstring("add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.168.1.0/24 }"))
1387+
Expect(nft.Dump()).NotTo(ContainSubstring("add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }"))
1388+
1389+
By("adding subnets to node annotation should update nftables elements")
1390+
node.Annotations[util.OvnNodeDontSNATSubnets] = `["192.167.1.0/24"]`
1391+
1392+
_, err = kubeFakeClient.CoreV1().Nodes().Update(context.TODO(), &node, metav1.UpdateOptions{})
1393+
Expect(err).NotTo(HaveOccurred())
1394+
1395+
Eventually(func() bool {
1396+
cleanDump := strings.ReplaceAll(nft.Dump(), "\r", "")
1397+
return strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.167.1.0/24 }") &&
1398+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.168.1.0/24 }") &&
1399+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }")
1400+
}).WithTimeout(2 * time.Second).Should(BeTrue())
1401+
1402+
By("adding extra subnets to node annotation should update nftables elements")
1403+
1404+
node.Annotations[util.OvnNodeDontSNATSubnets] = `["192.167.1.0/24","fd00::/64","192.169.1.0/24","fd11::/64"]`
1405+
1406+
_, err = kubeFakeClient.CoreV1().Nodes().Update(context.TODO(), &node, metav1.UpdateOptions{})
1407+
Expect(err).NotTo(HaveOccurred())
1408+
1409+
Eventually(func() bool {
1410+
cleanDump := strings.ReplaceAll(nft.Dump(), "\r", "")
1411+
return strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.167.1.0/24 }") &&
1412+
strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.169.1.0/24 }") &&
1413+
strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }")
1414+
}).WithTimeout(2 * time.Second).Should(BeTrue())
1415+
1416+
By("deleting node should remove nftables elements")
1417+
err = kubeFakeClient.CoreV1().Nodes().Delete(context.TODO(), nodeName, metav1.DeleteOptions{})
1418+
Expect(err).NotTo(HaveOccurred())
1419+
1420+
Eventually(func() bool {
1421+
cleanDump := strings.ReplaceAll(nft.Dump(), "\r", "")
1422+
return !strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.167.1.0/24 }") &&
1423+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.169.1.0/24 }") &&
1424+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }")
1425+
1426+
}).WithTimeout(2 * time.Second).Should(BeTrue())
1427+
return nil
1428+
}
1429+
1430+
err := app.Run([]string{app.Name})
1431+
Expect(err).NotTo(HaveOccurred())
1432+
})
1433+
1434+
ovntest.OnSupportedPlatformsIt("non-empty annotation on startup", func() {
1435+
1436+
app.Action = func(_ *cli.Context) error {
1437+
node := corev1.Node{
1438+
ObjectMeta: metav1.ObjectMeta{
1439+
Name: nodeName,
1440+
Annotations: map[string]string{
1441+
util.OvnNodeDontSNATSubnets: `["192.168.1.0/24","fd00::/64"]`,
1442+
},
1443+
},
1444+
Status: corev1.NodeStatus{
1445+
Addresses: []corev1.NodeAddress{
1446+
{
1447+
Type: corev1.NodeInternalIP,
1448+
Address: nodeIP,
1449+
},
1450+
},
1451+
},
1452+
}
1453+
1454+
nft := nodenft.SetFakeNFTablesHelper()
1455+
1456+
kubeFakeClient := fake.NewSimpleClientset(&corev1.NodeList{
1457+
Items: []corev1.Node{node},
1458+
})
1459+
fakeClient := &util.OVNNodeClientset{
1460+
KubeClient: kubeFakeClient,
1461+
AdminPolicyRouteClient: adminpolicybasedrouteclient.NewSimpleClientset(),
1462+
NetworkAttchDefClient: nadfake.NewSimpleClientset(),
1463+
}
1464+
1465+
stop := make(chan struct{})
1466+
wf, err := factory.NewNodeWatchFactory(fakeClient, nodeName)
1467+
Expect(err).NotTo(HaveOccurred())
1468+
wg := &sync.WaitGroup{}
1469+
defer func() {
1470+
close(stop)
1471+
wg.Wait()
1472+
wf.Shutdown()
1473+
}()
1474+
1475+
err = wf.Start()
1476+
Expect(err).NotTo(HaveOccurred())
1477+
routeManager := routemanager.NewController()
1478+
cnnci := NewCommonNodeNetworkControllerInfo(kubeFakeClient, fakeClient.AdminPolicyRouteClient, wf, nil, nodeName, routeManager)
1479+
nc = newDefaultNodeNetworkController(cnnci, stop, wg, routeManager, nil)
1480+
nc.initRetryFrameworkForNode()
1481+
err = setupPMTUDNFTSets()
1482+
Expect(err).NotTo(HaveOccurred())
1483+
err = setupPMTUDNFTChain()
1484+
Expect(err).NotTo(HaveOccurred())
1485+
defaultNetConfig := &bridgeUDNConfiguration{
1486+
ofPortPatch: "patch-breth0_ov",
1487+
}
1488+
nc.Gateway = &gateway{
1489+
openflowManager: &openflowManager{
1490+
flowCache: map[string][]string{},
1491+
defaultBridge: &bridgeConfiguration{
1492+
netConfig: map[string]*bridgeUDNConfiguration{
1493+
types.DefaultNetworkName: defaultNetConfig,
1494+
},
1495+
},
1496+
},
1497+
}
1498+
1499+
err = managementport.SetupManagementPortNFTSets()
1500+
Expect(err).NotTo(HaveOccurred())
1501+
1502+
// must run route manager manually which is usually started with nc.Start()
1503+
wg.Add(1)
1504+
go func() {
1505+
defer GinkgoRecover()
1506+
defer wg.Done()
1507+
nc.routeManager.Run(stop, 10*time.Second)
1508+
Expect(err).NotTo(HaveOccurred())
1509+
}()
1510+
By("expected nftables elements should present at startup")
1511+
1512+
err = nc.WatchNodes()
1513+
Expect(err).NotTo(HaveOccurred())
1514+
Expect(nft.Dump()).To(ContainSubstring("add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.168.1.0/24 }"))
1515+
Expect(nft.Dump()).To(ContainSubstring("add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }"))
1516+
1517+
By("editing subnets on node annotation should update nftables elements")
1518+
node.Annotations[util.OvnNodeDontSNATSubnets] = `["192.167.1.0/24"]`
1519+
1520+
_, err = kubeFakeClient.CoreV1().Nodes().Update(context.TODO(), &node, metav1.UpdateOptions{})
1521+
Expect(err).NotTo(HaveOccurred())
1522+
1523+
Eventually(func() bool {
1524+
cleanDump := strings.ReplaceAll(nft.Dump(), "\r", "")
1525+
return strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.167.1.0/24 }") &&
1526+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.168.1.0/24 }") &&
1527+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }")
1528+
}).WithTimeout(2 * time.Second).Should(BeTrue())
1529+
1530+
By("adding extra subnets to node annotation should update nftables elements")
1531+
1532+
node.Annotations[util.OvnNodeDontSNATSubnets] = `["192.167.1.0/24","fd00::/64","192.169.1.0/24","fd11::/64"]`
1533+
1534+
_, err = kubeFakeClient.CoreV1().Nodes().Update(context.TODO(), &node, metav1.UpdateOptions{})
1535+
Expect(err).NotTo(HaveOccurred())
1536+
1537+
Eventually(func() bool {
1538+
cleanDump := strings.ReplaceAll(nft.Dump(), "\r", "")
1539+
return strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.167.1.0/24 }") &&
1540+
strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.169.1.0/24 }") &&
1541+
strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }")
1542+
}).WithTimeout(2 * time.Second).Should(BeTrue())
1543+
1544+
By("deleting node should remove nftables elements")
1545+
err = kubeFakeClient.CoreV1().Nodes().Delete(context.TODO(), nodeName, metav1.DeleteOptions{})
1546+
Expect(err).NotTo(HaveOccurred())
1547+
1548+
Eventually(func() bool {
1549+
cleanDump := strings.ReplaceAll(nft.Dump(), "\r", "")
1550+
return !strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.167.1.0/24 }") &&
1551+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { 192.169.1.0/24 }") &&
1552+
!strings.Contains(cleanDump, "add element inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { fd00::/64 }")
1553+
1554+
}).WithTimeout(2 * time.Second).Should(BeTrue())
1555+
return nil
1556+
}
1557+
1558+
err := app.Run([]string{app.Name})
1559+
Expect(err).NotTo(HaveOccurred())
1560+
})
1561+
1562+
})
1563+
})
12411564
})

‎go-controller/pkg/node/gateway_init_linux_test.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,14 @@ add table inet ovn-kubernetes
5656
add set inet ovn-kubernetes mgmtport-no-snat-nodeports { type inet_proto . inet_service ; comment "NodePorts not subject to management port SNAT" ; }
5757
add set inet ovn-kubernetes mgmtport-no-snat-services-v4 { type ipv4_addr . inet_proto . inet_service ; comment "eTP:Local short-circuit not subject to management port SNAT (IPv4)" ; }
5858
add set inet ovn-kubernetes mgmtport-no-snat-services-v6 { type ipv6_addr . inet_proto . inet_service ; comment "eTP:Local short-circuit not subject to management port SNAT (IPv6)" ; }
59+
add set inet ovn-kubernetes mgmtport-no-snat-subnets-v4 { type ipv4_addr ; flags interval ; comment "subnets not subject to management port SNAT (IPv4)" ; }
60+
add set inet ovn-kubernetes mgmtport-no-snat-subnets-v6 { type ipv6_addr ; flags interval ; comment "subnets not subject to management port SNAT (IPv6)" ; }
5961
add chain inet ovn-kubernetes mgmtport-snat { type nat hook postrouting priority 100 ; comment "OVN SNAT to Management Port" ; }
6062
add rule inet ovn-kubernetes mgmtport-snat oifname != %s return
6163
add rule inet ovn-kubernetes mgmtport-snat meta nfproto ipv4 ip saddr 10.1.1.2 counter return
6264
add rule inet ovn-kubernetes mgmtport-snat meta l4proto . th dport @mgmtport-no-snat-nodeports counter return
6365
add rule inet ovn-kubernetes mgmtport-snat ip daddr . meta l4proto . th dport @mgmtport-no-snat-services-v4 counter return
66+
add rule inet ovn-kubernetes mgmtport-snat ip saddr @mgmtport-no-snat-subnets-v4 counter return
6467
add rule inet ovn-kubernetes mgmtport-snat counter snat ip to 10.1.1.2
6568
`
6669

‎go-controller/pkg/node/managementport/port_linux.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ func NewManagementPortController(
101101
}
102102

103103
// setup NFT sets early as gateway initialization depends on it
104-
err = setupManagementPortNFTSets()
104+
err = SetupManagementPortNFTSets()
105105
if err != nil {
106106
return nil, err
107107
}
@@ -299,7 +299,7 @@ func setupManagementPortConfig(link netlink.Link, cfg *managementPortConfig, rou
299299
// setupManagementPortNFTSets sets up the NFT sets that the management port SNAR
300300
// rules rely on. These sets are written to by other componets so they are setup
301301
// independantly and as early as possible.
302-
func setupManagementPortNFTSets() error {
302+
func SetupManagementPortNFTSets() error {
303303
nft, err := nodenft.GetNFTablesHelper()
304304
if err != nil {
305305
return err

‎go-controller/pkg/node/managementport/port_linux_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -783,7 +783,7 @@ var _ = Describe("Management Port tests", func() {
783783
ipv4: &fakeMgmtPortIPFamilyConfig,
784784
netInfo: netInfo,
785785
}
786-
err := setupManagementPortNFTSets()
786+
err := SetupManagementPortNFTSets()
787787
Expect(err).NotTo(HaveOccurred())
788788
err = setupManagementPortNFTChain(types.K8sMgmtIntfName, &fakeMgmtPortConfig)
789789
Expect(err).NotTo(HaveOccurred())

0 commit comments

Comments
 (0)