Skip to content

Commit 23e4eb2

Browse files
committed
Move management port implementation to new package
Move the existing implementaiton to a new package. It should be functionally equivalent except: * nft sets are configured as early as possible, the rest is configured upon Start (synchronously) * moved management port routing table, rules, routes etc from gateway code to this package, configued upon Start (synchronously) Signed-off-by: Jaime Caamaño Ruiz <[email protected]>
1 parent ccf448c commit 23e4eb2

17 files changed

+1552
-1314
lines changed

go-controller/hack/test-go.sh

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ function testrun {
3333
if [[ ! -z "${RACE:-}" ]]; then
3434
args="-race "
3535
fi
36-
if [[ "$USER" != root && " ${root_pkgs[@]} " =~ " $pkg " && -z "${DOCKER_TEST:-}" ]]; then
36+
if [[ "$USER" != root && " ${root_pkgs[*]} " =~ " $pkg " && -z "${DOCKER_TEST:-}" ]]; then
3737
testfile=$(mktemp --tmpdir ovn-test.XXXXXXXX)
3838
echo "sudo required for ${pkg}, compiling test to ${testfile}"
3939
if [[ ! -z "${RACE:-}" ]]; then
@@ -55,7 +55,7 @@ function testrun {
5555
if [ ! -z "${COVERALLS:-}" ]; then
5656
args="${args} -test.coverprofile=${idx}.coverprofile "
5757
fi
58-
if [[ " ${big_pkgs[@]} " =~ " $pkg " ]]; then
58+
if [[ " ${big_pkgs[*]} " =~ " $pkg " ]]; then
5959
echo "Increasing timeout to 20m for package ${pkg}"
6060
args="${args} -test.timeout=20m"
6161
fi
@@ -72,7 +72,16 @@ function testrun {
7272
}
7373

7474
# These packages requires root for network namespace manipulation in unit tests
75-
root_pkgs=("github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controllermanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/rulemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/vrfmanager" "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressip")
75+
root_pkgs=(
76+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/controllermanager"
77+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node"
78+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/controllers/egressip"
79+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/iptables"
80+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/managementport"
81+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/routemanager"
82+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/rulemanager"
83+
"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/node/vrfmanager"
84+
)
7685

7786
# These packages are big and require more than the 10m default to run the unit tests
7887
big_pkgs=("github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn")

go-controller/pkg/node/gateway_init.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -592,7 +592,7 @@ func CleanupClusterNode(name string) error {
592592
}
593593

594594
// Clean up legacy IPTables rules for management port
595-
DelLegacyMgtPortIptRules()
595+
managementport.DelLegacyMgtPortIptRules()
596596

597597
// Delete nftables rules
598598
nodenft.CleanupNFTables()

go-controller/pkg/node/gateway_init_linux_test.go

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,28 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS,
116116
)
117117

118118
fexec := ovntest.NewLooseCompareFakeExec()
119+
120+
// management port commands
121+
mpPortName := types.K8sMgmtIntfName
122+
mpPortRepName := types.K8sMgmtIntfName + "_0"
123+
mpPortLegacyName := types.K8sPrefix + nodeName
124+
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
125+
Cmd: "ovs-vsctl --timeout=15 --no-headings --data bare --format csv --columns type,name find Interface name=" + mpPortName,
126+
Output: "internal," + mpPortName,
127+
})
128+
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
129+
Cmd: "ovs-vsctl --timeout=15 --no-headings --data bare --format csv --columns type,name find Interface name=" + mpPortRepName,
130+
Output: "internal," + mpPortRepName,
131+
})
132+
fexec.AddFakeCmdsNoOutputNoError([]string{
133+
"ovs-vsctl --timeout=15 -- --if-exists del-port br-int " + mpPortLegacyName + " -- --may-exist add-port br-int " + mpPortName + " -- set interface " + mpPortName + " mac=\"0a:58:0a:01:01:02\" type=internal mtu_request=" + mtu + " external-ids:iface-id=" + mpPortLegacyName,
134+
})
135+
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
136+
Cmd: "sysctl -w net.ipv4.conf.ovn-k8s-mp0.forwarding=1",
137+
Output: "net.ipv4.conf.ovn-k8s-mp0.forwarding = 1",
138+
})
139+
140+
// gateway commands
119141
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
120142
Cmd: "ovs-vsctl --timeout=15 port-to-br eth0",
121143
Err: fmt.Errorf(""),
@@ -288,6 +310,14 @@ func shareGatewayInterfaceTest(app *cli.App, testNS ns.NetNS,
288310
}()
289311
err = testNS.Do(func(ns.NetNS) error {
290312
defer GinkgoRecover()
313+
314+
// create dummy management interface
315+
err := netlink.LinkAdd(&netlink.Dummy{
316+
LinkAttrs: netlink.LinkAttrs{
317+
Name: types.K8sMgmtIntfName,
318+
},
319+
})
320+
Expect(err).NotTo(HaveOccurred())
291321
// start management port
292322
err = mp.Start(stop)
293323
Expect(err).NotTo(HaveOccurred())
@@ -969,6 +999,27 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0`
969999

9701000
fexec := ovntest.NewLooseCompareFakeExec()
9711001

1002+
// management port commands
1003+
mpPortName := types.K8sMgmtIntfName
1004+
mpPortRepName := types.K8sMgmtIntfName + "_0"
1005+
mpPortLegacyName := types.K8sPrefix + nodeName
1006+
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
1007+
Cmd: "ovs-vsctl --timeout=15 --no-headings --data bare --format csv --columns type,name find Interface name=" + mpPortName,
1008+
Output: "internal," + mpPortName,
1009+
})
1010+
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
1011+
Cmd: "ovs-vsctl --timeout=15 --no-headings --data bare --format csv --columns type,name find Interface name=" + mpPortRepName,
1012+
Output: "internal," + mpPortRepName,
1013+
})
1014+
fexec.AddFakeCmdsNoOutputNoError([]string{
1015+
"ovs-vsctl --timeout=15 -- --if-exists del-port br-int " + mpPortLegacyName + " -- --may-exist add-port br-int " + mpPortName + " -- set interface " + mpPortName + " mac=\"0a:58:0a:01:01:02\" type=internal mtu_request=" + mtu + " external-ids:iface-id=" + mpPortLegacyName,
1016+
})
1017+
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
1018+
Cmd: "sysctl -w net.ipv4.conf.ovn-k8s-mp0.forwarding=1",
1019+
Output: "net.ipv4.conf.ovn-k8s-mp0.forwarding = 1",
1020+
})
1021+
1022+
// gateway commands
9721023
fexec.AddFakeCmd(&ovntest.ExpectedCmd{
9731024
Cmd: "ovs-vsctl --timeout=15 port-to-br eth0",
9741025
Err: fmt.Errorf(""),
@@ -1173,6 +1224,14 @@ OFPT_GET_CONFIG_REPLY (xid=0x4): frags=normal miss_send_len=0`
11731224
}()
11741225
err = testNS.Do(func(ns.NetNS) error {
11751226
defer GinkgoRecover()
1227+
1228+
// create dummy management interface
1229+
err := netlink.LinkAdd(&netlink.Dummy{
1230+
LinkAttrs: netlink.LinkAttrs{
1231+
Name: types.K8sMgmtIntfName,
1232+
},
1233+
})
1234+
Expect(err).NotTo(HaveOccurred())
11761235
// start management port
11771236
err = mp.Start(stop)
11781237
Expect(err).NotTo(HaveOccurred())

go-controller/pkg/node/gateway_localnet_linux_test.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -299,7 +299,7 @@ var _ = Describe("Node Operations", func() {
299299
})
300300

301301
Context("on startup", func() {
302-
It("removes stale iptables/nftables rules while keeping remaining intact", func() {
302+
It("removes stale iptables rules while keeping remaining intact", func() {
303303
app.Action = func(*cli.Context) error {
304304
// Depending on the order of informer event processing the initial
305305
// Service might be "added" once or twice. Take that into account.
@@ -353,10 +353,6 @@ var _ = Describe("Node Operations", func() {
353353
err := f4.MatchState(expectedTables, nil)
354354
Expect(err).NotTo(HaveOccurred())
355355

356-
expectedNFT := getBaseNFTRules(types.K8sMgmtIntfName) + "\nadd rule inet ovn-kubernetes mgmtport-snat blah blah blah\n"
357-
err = nodenft.MatchNFTRules(expectedNFT, nft.Dump())
358-
Expect(err).NotTo(HaveOccurred())
359-
360356
stopChan := make(chan struct{})
361357
fakeClient := util.GetOVNClientset(&service).GetNodeClientset()
362358
wf, err := factory.NewNodeWatchFactory(fakeClient, "node")
@@ -405,7 +401,7 @@ var _ = Describe("Node Operations", func() {
405401
err = f4.MatchState(expectedTables, nil)
406402
Expect(err).NotTo(HaveOccurred())
407403

408-
expectedNFT = getBaseNFTRules(types.K8sMgmtIntfName)
404+
expectedNFT := getBaseNFTRules(types.K8sMgmtIntfName)
409405
err = nodenft.MatchNFTRules(expectedNFT, nft.Dump())
410406
Expect(err).NotTo(HaveOccurred())
411407

go-controller/pkg/node/gateway_shared_intf.go

Lines changed: 0 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -50,9 +50,6 @@ const (
5050
ctMarkOVN = "0x1"
5151
// ctMarkHost is the conntrack mark value for host traffic
5252
ctMarkHost = "0x2"
53-
// ovnkubeSvcViaMgmPortRT is the number of the custom routing table used to steer host->service
54-
// traffic packets into OVN via ovn-k8s-mp0. Currently only used for ITP=local traffic.
55-
ovnkubeSvcViaMgmPortRT = "7"
5653
// ovnKubeNodeSNATMark is used to mark packets that need to be SNAT-ed to nodeIP for
5754
// traffic originating from egressIP and egressService controlled pods towards other nodes in the cluster.
5855
ovnKubeNodeSNATMark = "0x3f0"
@@ -2169,62 +2166,6 @@ func setBridgeOfPorts(bridge *bridgeConfiguration) error {
21692166
return nil
21702167
}
21712168

2172-
// initSvcViaMgmPortRoutingRules creates the svc2managementport routing table, routes and rules
2173-
// that let's us forward service traffic to ovn-k8s-mp0 as opposed to the default route towards breth0
2174-
func initSvcViaMgmPortRoutingRules(hostSubnets []*net.IPNet) error {
2175-
// create ovnkubeSvcViaMgmPortRT and service route towards ovn-k8s-mp0
2176-
for _, hostSubnet := range hostSubnets {
2177-
isIPv6 := utilnet.IsIPv6CIDR(hostSubnet)
2178-
gatewayIP := util.GetNodeGatewayIfAddr(hostSubnet).IP.String()
2179-
for _, svcCIDR := range config.Kubernetes.ServiceCIDRs {
2180-
if isIPv6 == utilnet.IsIPv6CIDR(svcCIDR) {
2181-
if stdout, stderr, err := util.RunIP("route", "replace", "table", ovnkubeSvcViaMgmPortRT, svcCIDR.String(), "via", gatewayIP, "dev", types.K8sMgmtIntfName); err != nil {
2182-
return fmt.Errorf("error adding routing table entry into custom routing table: %s: stdout: %s, stderr: %s, err: %v", ovnkubeSvcViaMgmPortRT, stdout, stderr, err)
2183-
}
2184-
klog.V(5).Infof("Successfully added route into custom routing table: %s", ovnkubeSvcViaMgmPortRT)
2185-
}
2186-
}
2187-
}
2188-
2189-
createRule := func(family string) error {
2190-
stdout, stderr, err := util.RunIP(family, "rule")
2191-
if err != nil {
2192-
return fmt.Errorf("error listing routing rules, stdout: %s, stderr: %s, err: %v", stdout, stderr, err)
2193-
}
2194-
if !strings.Contains(stdout, fmt.Sprintf("from all fwmark %s lookup %s", types.OVNKubeITPMark, ovnkubeSvcViaMgmPortRT)) {
2195-
if stdout, stderr, err := util.RunIP(family, "rule", "add", "fwmark", types.OVNKubeITPMark, "lookup", ovnkubeSvcViaMgmPortRT, "prio", "30"); err != nil {
2196-
return fmt.Errorf("error adding routing rule for service via management table (%s): stdout: %s, stderr: %s, err: %v", ovnkubeSvcViaMgmPortRT, stdout, stderr, err)
2197-
}
2198-
}
2199-
return nil
2200-
}
2201-
2202-
// create ip rule that will forward ovnkubeITPMark marked packets to ovnkubeITPRoutingTable
2203-
if config.IPv4Mode {
2204-
if err := createRule("-4"); err != nil {
2205-
return fmt.Errorf("could not add IPv4 rule: %v", err)
2206-
}
2207-
}
2208-
if config.IPv6Mode {
2209-
if err := createRule("-6"); err != nil {
2210-
return fmt.Errorf("could not add IPv6 rule: %v", err)
2211-
}
2212-
}
2213-
2214-
// lastly update the reverse path filtering options for ovn-k8s-mp0 interface to avoid dropping return packets
2215-
// NOTE: v6 doesn't have rp_filter strict mode block
2216-
rpFilterLooseMode := "2"
2217-
// TODO: Convert testing framework to mock golang module utilities. Example:
2218-
// result, err := sysctl.Sysctl(fmt.Sprintf("net/ipv4/conf/%s/rp_filter", types.K8sMgmtIntfName), rpFilterLooseMode)
2219-
stdout, stderr, err := util.RunSysctl("-w", fmt.Sprintf("net.ipv4.conf.%s.rp_filter=%s", types.K8sMgmtIntfName, rpFilterLooseMode))
2220-
if err != nil || stdout != fmt.Sprintf("net.ipv4.conf.%s.rp_filter = %s", types.K8sMgmtIntfName, rpFilterLooseMode) {
2221-
return fmt.Errorf("could not set the correct rp_filter value for interface %s: stdout: %v, stderr: %v, err: %v",
2222-
types.K8sMgmtIntfName, stdout, stderr, err)
2223-
}
2224-
2225-
return nil
2226-
}
2227-
22282169
func newGateway(
22292170
nodeName string,
22302171
subnets []*net.IPNet,
@@ -2362,12 +2303,6 @@ func newGateway(
23622303
}
23632304

23642305
if config.Gateway.NodeportEnable {
2365-
if config.OvnKubeNode.Mode == types.NodeModeFull {
2366-
// (TODO): Internal Traffic Policy is not supported in DPU mode
2367-
if err := initSvcViaMgmPortRoutingRules(subnets); err != nil {
2368-
return err
2369-
}
2370-
}
23712306
klog.Info("Creating Gateway Node Port Watcher")
23722307
gw.nodePortWatcher, err = newNodePortWatcher(gwBridge, gw.openflowManager, gw.nodeIPManager, watchFactory, networkManager)
23732308
if err != nil {

0 commit comments

Comments
 (0)