diff --git a/npm/iptm/iptm_test.go b/npm/iptm/iptm_test.go index 912be83495..630035f86e 100644 --- a/npm/iptm/iptm_test.go +++ b/npm/iptm/iptm_test.go @@ -13,95 +13,103 @@ import ( var ( initCalls = []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-ACCEPT"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, // NOTE the following grep call stdouts are misleading. The first grep returns 3, and the second one returns "" (i.e. line 0) // a fix is coming for fakeexec stdout and exit code problems from piping commands (e.g. what we do with grep) - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL {Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "4 "}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL {Cmd: []string{"grep", "AZURE-NPM"}, Stdout: "4 "}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-INGRESS"}}, // broken here - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-and-EGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-mark-0x2000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "ACCEPT-on-EGRESS-mark-0x1000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-on-connection-state"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "MARK", "--set-mark", "0x0", "-m", "comment", "--comment", "Clear-AZURE-NPM-MARKS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-All-packets"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-INGRESS"}}, // broken here + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-and-EGRESS-mark-0x3000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "ACCEPT-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-on-connection-state"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "MARK", "--set-mark", "0x0", "-m", "comment", "--comment", "Clear-AZURE-NPM-MARKS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-All-packets"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, // ///////// - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "AZURE-NPM-INGRESS-FROM", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "AZURE-NPM-EGRESS-TO", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "AZURE-NPM-INGRESS-FROM", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{ + "iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000", + }}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "AZURE-NPM-EGRESS-TO", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{ + "iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000", + }}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, } initWithJumpToAzureAtTopCalls = []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-ACCEPT"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-INGRESS"}}, // broken here - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-and-EGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-mark-0x2000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "ACCEPT-on-EGRESS-mark-0x1000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM", "-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-on-connection-state"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "MARK", "--set-mark", "0x0", "-m", "comment", "--comment", "Clear-AZURE-NPM-MARKS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-All-packets"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-INGRESS"}}, // broken here + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-and-EGRESS-mark-0x3000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "ACCEPT-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-j", "AZURE-NPM-ACCEPT", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "ACCEPT-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM", "-m", "state", "--state", "RELATED,ESTABLISHED", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-on-connection-state"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "MARK", "--set-mark", "0x0", "-m", "comment", "--comment", "Clear-AZURE-NPM-MARKS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-ACCEPT", "-j", "ACCEPT", "-m", "comment", "--comment", "ACCEPT-All-packets"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS", "-j", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, // ///////// - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "AZURE-NPM-INGRESS-FROM", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "AZURE-NPM-EGRESS-TO", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-INGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS-PORT", "-j", "AZURE-NPM-INGRESS-FROM", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS", "-j", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{ + "iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000", + }}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-PORT", "-j", "AZURE-NPM-EGRESS-TO", "-m", "comment", "--comment", "ALL-JUMP-TO-AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-INGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x2000", "-m", "comment", "--comment", "RETURN-on-INGRESS-mark-0x2000"}}, + {Cmd: []string{ + "iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x3000", "-m", "comment", "--comment", "RETURN-on-EGRESS-and-INGRESS-mark-0x3000", + }}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "AZURE-NPM-EGRESS-DROPS", "-j", "RETURN", "-m", "mark", "--mark", "0x1000", "-m", "comment", "--comment", "RETURN-on-EGRESS-mark-0x1000"}}, } ) @@ -124,69 +132,69 @@ func TestUninitNpmChains(t *testing.T) { { name: "no v2 npm chains exist", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, {Cmd: []string{"grep", "Chain AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-ACCEPT"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-TARGET-SETS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INRGESS-DROPS"}}, // can we remove this rule now? - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-ACCEPT"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-TARGET-SETS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INRGESS-DROPS"}}, // can we delete this rule now? + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-TARGET-SETS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INRGESS-DROPS"}}, // can we remove this rule now? + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-TARGET-SETS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INRGESS-DROPS"}}, // can we delete this rule now? }, }, { name: " v2 exists chian exists", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, {Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: "Chain AZURE-NPM-INGRESS-ALLOW-MARK (1 references)\n"}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-ACCEPT"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-TARGET-SETS"}}, - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INRGESS-DROPS"}}, // can we remove this rule now? - {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS-ALLOW-MARK"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-ACCEPT"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-FROM"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS-PORT"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS-TO"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-EGRESS-DROPS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-TARGET-SETS"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INRGESS-DROPS"}}, // can we delete this rule now? - {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS-ALLOW-MARK"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-TARGET-SETS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INRGESS-DROPS"}}, // can we remove this rule now? + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM-INGRESS-ALLOW-MARK"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-FROM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS-PORT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS-TO"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-EGRESS-DROPS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-TARGET-SETS"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INRGESS-DROPS"}}, // can we delete this rule now? + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "AZURE-NPM-INGRESS-ALLOW-MARK"}}, }, }, // currently can't test multiple v2 chains existing because AllCurrentAzureChains() returns a map and fexec needs the exact order of commands @@ -228,9 +236,9 @@ func TestCheckAndAddForwardChain(t *testing.T) { name: "add missing jump to azure at top", args: args{ calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, // "rule does not exist" - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, // "rule does not exist" + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, }, @@ -239,11 +247,11 @@ func TestCheckAndAddForwardChain(t *testing.T) { name: "add missing jump to azure after kube services", args: args{ calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL STDOUT - {Cmd: []string{"grep", "KUBE-SERVICES"}, ExitCode: 1}, // THIS IS THE EXIT CODE FOR CHECK command below ("rule doesn't exist") - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"grep", "KUBE-SERVICES"}, ExitCode: 1}, // THIS IS THE EXIT CODE FOR CHECK command below ("rule doesn't exist") + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, }, @@ -252,9 +260,9 @@ func TestCheckAndAddForwardChain(t *testing.T) { name: "jump to azure already at top", args: args{ calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "1 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "1 "}, // THIS IS THE GREP CALL STDOUT {Cmd: []string{"grep", "AZURE-NPM"}}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, @@ -264,11 +272,11 @@ func TestCheckAndAddForwardChain(t *testing.T) { name: "jump to azure already after kube services", args: args{ calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL STDOUT {Cmd: []string{"grep", "KUBE-SERVICES"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, Stdout: "4 "}, // THIS IS THE GREP CALL STDOUT - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, Stdout: "4 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}}, {Cmd: []string{"grep", "AZURE-NPM"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, @@ -278,12 +286,12 @@ func TestCheckAndAddForwardChain(t *testing.T) { name: "move jump to azure to top", args: args{ calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "5 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "5 "}, // THIS IS THE GREP CALL STDOUT {Cmd: []string{"grep", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, }, @@ -292,14 +300,14 @@ func TestCheckAndAddForwardChain(t *testing.T) { name: "move jump to azure after kube services", args: args{ calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "AZURE-NPM"}}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, Stdout: "3 "}, // THIS IS THE GREP CALL STDOUT {Cmd: []string{"grep", "KUBE-SERVICES"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, Stdout: "2 "}, // THIS IS THE GREP CALL STDOUT - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, Stdout: "2 "}, // THIS IS THE GREP CALL STDOUT + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}}, {Cmd: []string{"grep", "AZURE-NPM"}}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, }, @@ -319,7 +327,7 @@ func TestCheckAndAddForwardChain(t *testing.T) { func TestExists(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "ACCEPT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "ACCEPT"}}, } fexec := testutils.GetFakeExecWithScripts(calls) @@ -341,7 +349,7 @@ func TestExists(t *testing.T) { func TestAddChain(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "TEST-CHAIN"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "TEST-CHAIN"}}, } fexec := testutils.GetFakeExecWithScripts(calls) @@ -355,8 +363,8 @@ func TestAddChain(t *testing.T) { func TestDeleteChain(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "TEST-CHAIN"}}, - {Cmd: []string{"iptables", "-w", "60", "-X", "TEST-CHAIN"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "TEST-CHAIN"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-X", "TEST-CHAIN"}}, } fexec := testutils.GetFakeExecWithScripts(calls) @@ -374,7 +382,7 @@ func TestDeleteChain(t *testing.T) { func TestAdd(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "REJECT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "REJECT"}}, } fexec := testutils.GetFakeExecWithScripts(calls) @@ -420,9 +428,9 @@ func testPrometheusMetrics(t *testing.T, expectedNumACLRules, expectedExecCount func TestDelete(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "REJECT"}}, - {Cmd: []string{"iptables", "-w", "60", "-C", "FORWARD", "-j", "REJECT"}}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "REJECT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "REJECT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-C", "FORWARD", "-j", "REJECT"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "REJECT"}}, } fexec := testutils.GetFakeExecWithScripts(calls) @@ -450,7 +458,7 @@ func TestDelete(t *testing.T) { func TestRun(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-N", "TEST-CHAIN"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-N", "TEST-CHAIN"}}, } fexec := testutils.GetFakeExecWithScripts(calls) @@ -508,7 +516,7 @@ func TestGetChainLineNumber(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-t", "filter", "-n", "--list", "FORWARD", "--line-numbers"}, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, Stdout: tt.stdout, ExitCode: tt.exitCode}, } fexec := testutils.GetFakeExecWithScripts(calls) diff --git a/npm/pkg/dataplane/dataplane_linux.go b/npm/pkg/dataplane/dataplane_linux.go index 368e499d24..99769bf0ae 100644 --- a/npm/pkg/dataplane/dataplane_linux.go +++ b/npm/pkg/dataplane/dataplane_linux.go @@ -2,7 +2,6 @@ package dataplane import ( "github.com/Azure/azure-container-networking/npm/pkg/dataplane/policies" - "github.com/Azure/azure-container-networking/npm/util" npmerrors "github.com/Azure/azure-container-networking/npm/util/errors" ) @@ -21,8 +20,6 @@ func (dp *DataPlane) updatePod(pod *updateNPMPod) error { } func (dp *DataPlane) bootupDataPlane() error { - util.DetectIptablesVersion(dp.ioShim) - // It is important to keep order to clean-up ACLs before ipsets. Otherwise we won't be able to delete ipsets referenced by ACLs if err := dp.policyMgr.Bootup(nil); err != nil { return npmerrors.ErrorWrapper(npmerrors.BootupDataplane, false, "failed to reset policy dataplane", err) diff --git a/npm/pkg/dataplane/dataplane_linux_test.go b/npm/pkg/dataplane/dataplane_linux_test.go index 7c48ac1d3b..837a33445d 100644 --- a/npm/pkg/dataplane/dataplane_linux_test.go +++ b/npm/pkg/dataplane/dataplane_linux_test.go @@ -1,7 +1,6 @@ package dataplane import ( - "fmt" "testing" "time" @@ -74,9 +73,6 @@ func TestNetPolInBackgroundUpdatePolicy(t *testing.T) { calls := append(getBootupTestCalls(), getAddPolicyTestCallsForDP(&testPolicyobj)...) calls = append(calls, getRemovePolicyTestCallsForDP(&testPolicyobj)...) calls = append(calls, getAddPolicyTestCallsForDP(&updatedTestPolicyobj)...) - for _, call := range calls { - fmt.Println(call) - } ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) dp, err := NewDataPlane("testnode", ioshim, netpolInBackgroundCfg, nil) @@ -133,31 +129,31 @@ func TestNetPolInBackgroundFailureToAddFirstTime(t *testing.T) { }, // restore will try twice per pMgr.AddPolicies() call testutils.TestCmd{ - Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 1, }, testutils.TestCmd{ - Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 1, }, // first policy succeeds testutils.TestCmd{ - Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 0, }, // second policy succeeds testutils.TestCmd{ - Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 0, }, // third policy fails // restore will try twice per pMgr.AddPolicies() call testutils.TestCmd{ - Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 1, }, testutils.TestCmd{ - Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 1, }, ) diff --git a/npm/pkg/dataplane/dataplane_test.go b/npm/pkg/dataplane/dataplane_test.go index fd32c547f3..00e40aaf29 100644 --- a/npm/pkg/dataplane/dataplane_test.go +++ b/npm/pkg/dataplane/dataplane_test.go @@ -1,7 +1,6 @@ package dataplane import ( - "fmt" "testing" "github.com/Azure/azure-container-networking/common" @@ -262,9 +261,6 @@ func TestUpdatePolicy(t *testing.T) { calls := append(getBootupTestCalls(), getAddPolicyTestCallsForDP(&testPolicyobj)...) calls = append(calls, getRemovePolicyTestCallsForDP(&testPolicyobj)...) calls = append(calls, getAddPolicyTestCallsForDP(&updatedTestPolicyobj)...) - for _, call := range calls { - fmt.Println(call) - } ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) dp, err := NewDataPlane("testnode", ioshim, dpCfg, nil) @@ -420,7 +416,7 @@ func TestUpdatePodCache(t *testing.T) { } func getBootupTestCalls() []testutils.TestCmd { - return append(policies.GetBootupTestCalls(true), ipsets.GetResetTestCalls()...) + return append(policies.GetBootupTestCalls(), ipsets.GetResetTestCalls()...) } func getAddPolicyTestCallsForDP(networkPolicy *policies.NPMNetworkPolicy) []testutils.TestCmd { diff --git a/npm/pkg/dataplane/parse/parser_test.go b/npm/pkg/dataplane/parse/parser_test.go index a5d880e644..daa58c2a42 100644 --- a/npm/pkg/dataplane/parse/parser_test.go +++ b/npm/pkg/dataplane/parse/parser_test.go @@ -30,7 +30,7 @@ func TestParseIptablesObjectFileV2(t *testing.T) { func TestParseIptablesObject(t *testing.T) { calls := []testutils.TestCmd{ - {Cmd: []string{"iptables-save", "-t", "filter"}}, + {Cmd: []string{"iptables-nft-save", "-t", "filter"}}, } parser := IPTablesParser{ diff --git a/npm/pkg/dataplane/policies/chain-management_linux.go b/npm/pkg/dataplane/policies/chain-management_linux.go index 0ad7c0ae7d..3691e8014f 100644 --- a/npm/pkg/dataplane/policies/chain-management_linux.go +++ b/npm/pkg/dataplane/policies/chain-management_linux.go @@ -86,6 +86,9 @@ var ( util.IptablesJumpFlag, util.IptablesAzureChain, } + + listHintChainArgs = []string{"KUBE-IPTABLES-HINT", util.IptablesTableFlag, util.IptablesMangleTable, util.IptablesNumericFlag} + listCanaryChainArgs = []string{"KUBE-KUBELET-CANARY", util.IptablesTableFlag, util.IptablesMangleTable, util.IptablesNumericFlag} ) type exitErrorInfo struct { @@ -162,6 +165,8 @@ func isBaseChain(chain string) bool { Called once at startup. Like the rest of PolicyManager, minimizes the number of OS calls by consolidating all possible actions into one iptables-restore call. +0.1. Detect iptables version. +0.2. Clean up legacy tables if using nft and vice versa. 1. Delete the deprecated jump from FORWARD to AZURE-NPM chain (if it exists). 2. Cleanup old NPM chains, and configure base chains and their rules. 1. Do the following via iptables-restore --noflush: @@ -181,87 +186,27 @@ TODO: could use one grep call instead of separate calls for getting jump line nu func (pMgr *PolicyManager) bootup(_ []string) error { klog.Infof("booting up iptables Azure chains") + // 0.1. Detect iptables version + pMgr.detectIptablesVersion() + // Stop reconciling so we don't contend for iptables, and so we don't update the staleChains at the same time as reconcile() // Reconciling would only be happening if this function were called to reset iptables well into the azure-npm pod lifecycle. pMgr.reconcileManager.forceLock() defer pMgr.reconcileManager.forceUnlock() - if strings.Contains(util.Iptables, "nft") { - klog.Info("detected nft iptables. cleaning up legacy iptables") - util.Iptables = util.IptablesLegacy - util.IptablesSave = util.IptablesSaveLegacy - util.IptablesRestore = util.IptablesRestoreLegacy - - // 0. delete the deprecated jump to deprecated AZURE-NPM in legacy iptables - deprecatedErrCode, deprecatedErr := pMgr.ignoreErrorsAndRunIPTablesCommand(removeDeprecatedJumpIgnoredErrors, util.IptablesDeletionFlag, deprecatedJumpFromForwardToAzureChainArgs...) - if deprecatedErrCode == 0 { - klog.Infof("deleted deprecated jump rule from FORWARD chain to AZURE-NPM chain") - } else if deprecatedErr != nil { - metrics.SendErrorLogAndMetric(util.IptmID, - "failed to delete deprecated jump rule from FORWARD chain to AZURE-NPM chain for unexpected reason with exit code %d and error: %s", - deprecatedErrCode, deprecatedErr.Error()) - } - - // 0. delete the deprecated jump to current AZURE-NPM in legacy iptables - deprecatedErrCode, deprecatedErr = pMgr.ignoreErrorsAndRunIPTablesCommand(removeDeprecatedJumpIgnoredErrors, util.IptablesDeletionFlag, jumpFromForwardToAzureChainArgs...) - if deprecatedErrCode == 0 { - klog.Infof("deleted deprecated jump rule from FORWARD chain to AZURE-NPM chain") - } else if deprecatedErr != nil { - metrics.SendErrorLogAndMetric(util.IptmID, - "failed to delete deprecated jump rule from FORWARD chain to AZURE-NPM chain for unexpected reason with exit code %d and error: %s", - deprecatedErrCode, deprecatedErr.Error()) - } - - // clean up current chains in legacy iptables - currentChains, err := ioutil.AllCurrentAzureChains(pMgr.ioShim.Exec, util.IptablesDefaultWaitTime) - if err != nil { - return npmerrors.SimpleErrorWrapper("failed to get current chains for bootup", err) - } - - // We have only one chance to clean existing legacy iptables chains. - // So flush all the chains and then destroy them - var aggregateError error - for chain := range currentChains { - errCode, err := pMgr.runIPTablesCommand(util.IptablesFlushFlag, chain) - if err != nil && errCode != doesNotExistErrorCode { - // add to staleChains if it's not one of the iptablesAzureChains - pMgr.staleChains.add(chain) - currentErrString := fmt.Sprintf("failed to flush chain %s with err [%v]", chain, err) - if aggregateError == nil { - aggregateError = npmerrors.SimpleError(currentErrString) - } else { - aggregateError = npmerrors.SimpleErrorWrapper(fmt.Sprintf("%s and had previous error", currentErrString), aggregateError) - } - } - } - - for chain := range currentChains { - errCode, err := pMgr.runIPTablesCommand(util.IptablesDestroyFlag, chain) - if err != nil && errCode != doesNotExistErrorCode { - // add to staleChains if it's not one of the iptablesAzureChains - pMgr.staleChains.add(chain) - currentErrString := fmt.Sprintf("failed to delete chain %s with err [%v]", chain, err) - if aggregateError == nil { - aggregateError = npmerrors.SimpleError(currentErrString) - } else { - aggregateError = npmerrors.SimpleErrorWrapper(fmt.Sprintf("%s and had previous error", currentErrString), aggregateError) - } - } - } - - if aggregateError != nil { - metrics.SendErrorLogAndMetric(util.IptmID, - "failed to flush and delete stale chain in legacy iptables with error: %s", - aggregateError.Error()) - } + // 0.2. cleanup + if err := pMgr.cleanupOtherIptables(); err != nil { + return npmerrors.SimpleErrorWrapper("failed to cleanup other iptables chains", err) + } - util.Iptables = util.IptablesNft - util.IptablesSave = util.IptablesSaveNft - util.IptablesRestore = util.IptablesRestoreNft + if err := pMgr.bootupAfterDetectAndCleanup(); err != nil { + return err } - klog.Info("cleaning up default iptables") + return nil +} +func (pMgr *PolicyManager) bootupAfterDetectAndCleanup() error { // 1. delete the deprecated jump to AZURE-NPM deprecatedErrCode, deprecatedErr := pMgr.ignoreErrorsAndRunIPTablesCommand(removeDeprecatedJumpIgnoredErrors, util.IptablesDeletionFlag, deprecatedJumpFromForwardToAzureChainArgs...) if deprecatedErrCode == 0 { @@ -294,6 +239,215 @@ func (pMgr *PolicyManager) bootup(_ []string) error { return nil } +// detectIptablesVersion sets the global iptables variable to nft if detected or legacy if detected. +// NPM will crash if it fails to detect either. +// This global variable is referenced in all iptables related functions. +// NPM should use the same iptables version as kube-proxy. +// kube-proxy creates an iptables chain as a hint for which version it uses. +// For more details, see: https://kubernetes.io/blog/2022/09/07/iptables-chains-not-api/#use-case-iptables-mode +func (pMgr *PolicyManager) detectIptablesVersion() { + klog.Info("first attempt detecting iptables version. looking for hint/canary chain in iptables-nft") + if pMgr.hintOrCanaryChainExist(util.IptablesNft) { + util.SetIptablesToNft() + return + } + + klog.Info("second attempt detecting iptables version. looking for hint/canary chain in iptables-legacy") + if pMgr.hintOrCanaryChainExist(util.IptablesLegacy) { + util.SetIptablesToLegacy() + return + } + + // default to nft if nothing is found + util.SetIptablesToNft() +} + +func (pMgr *PolicyManager) hintOrCanaryChainExist(iptablesCmd string) bool { + // hint chain should exist since k8s 1.24 (see https://kubernetes.io/blog/2022/09/07/iptables-chains-not-api/#use-case-iptables-mode) + prevIptables := util.Iptables + util.Iptables = iptablesCmd + defer func() { + util.Iptables = prevIptables + }() + + _, hintErr := pMgr.runIPTablesCommand(util.IptablesListFlag, listHintChainArgs...) + if hintErr == nil { + metrics.SendLog(util.IptmID, "found hint chain. will use iptables version: %s"+iptablesCmd, metrics.DonotPrint) + return true + } + + // check for canary chain + _, canaryErr := pMgr.runIPTablesCommand(util.IptablesListFlag, listCanaryChainArgs...) + if canaryErr != nil { + return false + } + + metrics.SendLog(util.IptmID, "found canary chain. will use iptables version: "+iptablesCmd, metrics.DonotPrint) + return true +} + +// clenaupOtherIptablesChains cleans up legacy tables if using nft and vice versa. +// It will only return an error if it fails to delete a jump rule and flush the AZURE-NPM chain (see comment about #3088 below). +// Cleanup logic: +// 1. delete jump rules to AZURE-NPM +// 2. flush all chains +// 3. delete all chains +func (pMgr *PolicyManager) cleanupOtherIptables() error { + hadNFT := util.Iptables == util.IptablesNft + if hadNFT { + klog.Info("detected nft iptables. cleaning up legacy iptables") + util.SetIptablesToLegacy() + } else { + klog.Info("detected legacy iptables. cleaning up nft iptables") + util.SetIptablesToNft() + } + + defer func() { + if hadNFT { + klog.Info("cleaned up legacy iptables") + util.SetIptablesToNft() + } else { + klog.Info("cleaned up nft tables") + util.SetIptablesToLegacy() + } + }() + + deletedJumpRule := false + + // 1.1. delete the deprecated jump to AZURE-NPM + errCode, err := pMgr.ignoreErrorsAndRunIPTablesCommand(removeDeprecatedJumpIgnoredErrors, util.IptablesDeletionFlag, deprecatedJumpFromForwardToAzureChainArgs...) + if errCode == 0 { + klog.Infof("[cleanup] deleted deprecated jump rule from FORWARD chain to AZURE-NPM chain") + deletedJumpRule = true + } else if err != nil { + metrics.SendErrorLogAndMetric(util.IptmID, + "[cleanup] failed to delete deprecated jump rule from FORWARD chain to AZURE-NPM chain for unexpected reason with exit code %d and error: %s", + errCode, err.Error()) + } + + // 1.2. delete the jump to AZURE-NPM + errCode, err = pMgr.ignoreErrorsAndRunIPTablesCommand(removeDeprecatedJumpIgnoredErrors, util.IptablesDeletionFlag, jumpFromForwardToAzureChainArgs...) + if errCode == 0 { + deletedJumpRule = true + klog.Infof("[cleanup] deleted jump rule from FORWARD chain to AZURE-NPM chain") + } else if err != nil { + metrics.SendErrorLogAndMetric(util.IptmID, + "[cleanup] failed to delete jump rule from FORWARD chain to AZURE-NPM chain for unexpected reason with exit code %d and error: %s", + errCode, err.Error()) + } + + // 2. get current chains + currentChains, err := ioutil.AllCurrentAzureChains(pMgr.ioShim.Exec, util.IptablesDefaultWaitTime) + if err != nil { + return npmerrors.SimpleErrorWrapper("[cleanup] failed to get current chains for bootup", err) + } + + if len(currentChains) == 0 { + klog.Info("no chains to cleanup") + return nil + } + + klog.Infof("[cleanup] %d chains to clean up", len(currentChains)) + + // 3.1. try to flush all chains at once + chains := make([]string, 0, len(currentChains)) + _, hasAzureChain := currentChains[util.IptablesAzureChain] + if hasAzureChain { + // putting AZURE-NPM chain first is required for proper unit testing (for determinancy in destroying chains) + chains = append(chains, util.IptablesAzureChain) + } + for chain := range currentChains { + if chain == util.IptablesAzureChain { + // putting AZURE-NPM chain first is required for proper unit testing (for determinancy in destroying chains) + continue + } + chains = append(chains, chain) + } + + creator := pMgr.creatorForCleanup(chains) + if err := restore(creator); err != nil { + msg := "[cleanup] failed to flush all chains with error: %s" + klog.Infof(msg, err.Error()) + metrics.SendErrorLogAndMetric(util.IptmID, msg, err.Error()) + + // 3.2. if we failed to flush all chains, then try to flush and delete them one by one + var aggregateError error + if _, ok := currentChains[util.IptablesAzureChain]; ok { + _, err := pMgr.runIPTablesCommand(util.IptablesFlushFlag, util.IptablesAzureChain) + aggregateError = err + if err != nil && !deletedJumpRule { + // fixes #3088 + // if we failed to delete a jump rule to AZURE-NPM and we failed to flush AZURE-NPM chain, + // then there is risk that there is a jump rule to AZURE-NPM, which in turn has rules which could lead to allowing or dropping a packet. + // We have failed to cleanup the other iptables rules, and there is no guarantee that packets will be processed correctly now. + // So we must crash and retry. + return npmerrors.SimpleErrorWrapper("[cleanup] must crash and retry. failed to delete jump rule and flush AZURE-NPM chain with error", err) + } + } + + for chain := range currentChains { + if chain == util.IptablesAzureChain { + // already flushed above + continue + } + + errCode, err := pMgr.runIPTablesCommand(util.IptablesFlushFlag, chain) + if err != nil && errCode != doesNotExistErrorCode { + // NOTE: if we fail to flush or delete the chain, then we will never clean it up in the future. + // This is zero-day behavior since NPM supported nft (we used to mark the chain stale, but this would not have worked as expected). + // NPM currently has no mechanism for retrying flush/delete for a chain from the other iptables version (other than the AZURE-NPM chain which is handled above). + currentErrString := fmt.Sprintf("failed to flush chain %s with err [%v]", chain, err) + if aggregateError == nil { + aggregateError = npmerrors.SimpleError(currentErrString) + } else { + aggregateError = npmerrors.SimpleErrorWrapper(currentErrString+" and had previous error", aggregateError) + } + } + } + + if aggregateError != nil { + metrics.SendErrorLogAndMetric(util.IptmID, + "[cleanup] benign failure to flush chains with error: %s", + aggregateError.Error()) + } + } + + // 4. delete all chains + var aggregateError error + for _, chain := range chains { + errCode, err := pMgr.runIPTablesCommand(util.IptablesDestroyFlag, chain) + if err != nil && errCode != doesNotExistErrorCode { + // NOTE: if we fail to flush or delete the chain, then we will never clean it up in the future. + // This is zero-day behavior since NPM supported nft (we used to mark the chain stale, but this would not have worked as expected). + // NPM currently has no mechanism for retrying flush/delete for a chain from the other iptables version (other than the AZURE-NPM chain which is handled above). + currentErrString := fmt.Sprintf("failed to delete chain %s with err [%v]", chain, err) + if aggregateError == nil { + aggregateError = npmerrors.SimpleError(currentErrString) + } else { + aggregateError = npmerrors.SimpleErrorWrapper(currentErrString+" and had previous error", aggregateError) + } + } + } + + if aggregateError != nil { + metrics.SendErrorLogAndMetric(util.IptmID, + "[cleanup] benign failure to delete chains with error: %s", + aggregateError.Error()) + } + + return nil +} + +func (pMgr *PolicyManager) creatorForCleanup(chains []string) *ioutil.FileCreator { + // pass nil because we don't need to add any lines like ":CHAIN-NAME - -" because that is for creating chains + creator := pMgr.newCreatorWithChains(nil) + for _, chain := range chains { + creator.AddLine("", nil, "-F "+chain) + } + creator.AddLine("", nil, util.IptablesRestoreCommit) + return creator +} + // reconcile does the following: // - creates the jump rule from FORWARD chain to AZURE-NPM chain (if it does not exist) and makes sure it's after the jumps to KUBE-FORWARD & KUBE-SERVICES chains (if they exist). // - cleans up stale policy chains. It can be forced to stop this process if reconcileManager.forceLock() is called. @@ -363,7 +517,7 @@ func (pMgr *PolicyManager) ignoreErrorsAndRunIPTablesCommand(ignored []*exitErro allArgs := []string{util.IptablesWaitFlag, util.IptablesDefaultWaitTime, operationFlag} allArgs = append(allArgs, args...) - klog.Infof("Executing iptables command with args %v", allArgs) + klog.Infof("executing iptables command [%s] with args %v", util.Iptables, allArgs) command := pMgr.ioShim.Exec.Command(util.Iptables, allArgs...) output, err := command.CombinedOutput() diff --git a/npm/pkg/dataplane/policies/chain-management_linux_test.go b/npm/pkg/dataplane/policies/chain-management_linux_test.go index 75fa6994b5..a211e033a1 100644 --- a/npm/pkg/dataplane/policies/chain-management_linux_test.go +++ b/npm/pkg/dataplane/policies/chain-management_linux_test.go @@ -26,8 +26,7 @@ Chain AZURE-NPM-INGRESS (1 references) Chain AZURE-NPM-INGRESS-ALLOW-MARK (1 references) ` - grepOutputAzureV1Chains = `Chain AZURE-NPM -Chain AZURE-NPM (1 references) + grepOutputAzureV1Chains = `Chain AZURE-NPM (1 references) Chain AZURE-NPM-INGRESS (1 references) Chain AZURE-NPM-INGRESS-DROPS (1 references) Chain AZURE-NPM-INGRESS-TO (1 references) @@ -38,28 +37,12 @@ Chain AZURE-NPM-EGRESS-FROM (1 references) Chain AZURE-NPM-EGRESS-PORTS (1 references) Chain AZURE-NPM-ACCEPT (1 references) ` -) - -// similar to TestBootup in policymanager.go except an error occurs -func TestBootupFailure(t *testing.T) { - metrics.ReinitializeAll() - calls := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, //nolint // AZURE-NPM chain didn't exist - {Cmd: listAllCommandStrings, PipedToCommand: true, HasStartError: true, ExitCode: 1}, - {Cmd: []string{"grep", "Chain AZURE-NPM"}}, - } - ioshim := common.NewMockIOShim(calls) - defer ioshim.VerifyCalls(t, calls) - pMgr := NewPolicyManager(ioshim, ipsetConfig) - - metrics.IncNumACLRules() - metrics.IncNumACLRules() - - require.Error(t, pMgr.Bootup(nil)) - // make sure that the metrics were reset - promVals{0, 0}.testPrometheusMetrics(t) -} + // pMgr.cleanupOtherIptables() can't be tested deterministically for more than two chains + grepOutputTwoAzureChains = `Chain AZURE-NPM (1 references) +Chain AZURE-NPM-INGRESS (1 references) +` +) func TestStaleChainsForceLock(t *testing.T) { testChains := []string{} @@ -364,25 +347,36 @@ func TestBootupLinux(t *testing.T) { // all tests with "no NPM prior" work for any situation (with v1 or v2 prior), // but the fake command exit codes and stdouts are in line with having no NPM prior { - name: "success (no NPM prior)", - calls: GetBootupTestCalls(false), + name: "success (no NPM prior)", + calls: []testutils.TestCmd{ + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, //nolint // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + ExitCode: 1, + }, + fakeIPTablesRestoreCommand, + {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, + {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + }, wantErr: false, }, { name: "success after restore failure (no NPM prior)", calls: []testutils.TestCmd{ { - Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2, Stdout: "iptables v1.8.4 (legacy): Couldn't load target `AZURE-NPM':No such file or directory", }, // AZURE-NPM chain didn't exist - {Cmd: listAllCommandStrings, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, {Cmd: []string{"grep", "Chain AZURE-NPM"}, ExitCode: 1}, fakeIPTablesRestoreFailureCommand, // e.g. xtables lock held by another app. Currently the stdout doesn't matter for retrying fakeIPTablesRestoreCommand, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, wantErr: false, }, @@ -390,11 +384,11 @@ func TestBootupLinux(t *testing.T) { name: "success: v2 existed prior", calls: []testutils.TestCmd{ { - Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 1, Stdout: "No chain/target/match by that name", }, // deprecated rule did not exist - {Cmd: listAllCommandStrings, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: grepOutputAzureChainsWithoutPolicies, @@ -402,15 +396,15 @@ func TestBootupLinux(t *testing.T) { fakeIPTablesRestoreCommand, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, wantErr: false, }, { name: "v1 existed prior: successfully delete deprecated jump", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, // deprecated rule existed - {Cmd: listAllCommandStrings, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, // deprecated rule existed + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: grepOutputAzureV1Chains, @@ -418,15 +412,15 @@ func TestBootupLinux(t *testing.T) { fakeIPTablesRestoreCommand, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, wantErr: false, }, { name: "v1 existed prior: unknown error while deleting deprecated jump", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 3}, // unknown error - {Cmd: listAllCommandStrings, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 3}, // unknown error + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: grepOutputAzureV1Chains, @@ -434,15 +428,15 @@ func TestBootupLinux(t *testing.T) { fakeIPTablesRestoreCommand, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, wantErr: false, }, { name: "failure while finding current chains (no NPM prior)", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, // AZURE-NPM chain didn't exist - {Cmd: listAllCommandStrings, PipedToCommand: true, HasStartError: true, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true, HasStartError: true, ExitCode: 1}, {Cmd: []string{"grep", "Chain AZURE-NPM"}}, }, wantErr: true, @@ -450,8 +444,8 @@ func TestBootupLinux(t *testing.T) { { name: "failure twice on restore (no NPM prior)", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, // AZURE-NPM chain didn't exist - {Cmd: listAllCommandStrings, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, {Cmd: []string{"grep", "Chain AZURE-NPM"}, ExitCode: 1}, fakeIPTablesRestoreFailureCommand, fakeIPTablesRestoreFailureCommand, @@ -461,8 +455,8 @@ func TestBootupLinux(t *testing.T) { { name: "failure on position (no NPM prior)", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, // AZURE-NPM chain didn't exist - {Cmd: listAllCommandStrings, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: grepOutputAzureChainsWithoutPolicies, @@ -471,7 +465,7 @@ func TestBootupLinux(t *testing.T) { {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, { - Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1, }, }, @@ -484,7 +478,7 @@ func TestBootupLinux(t *testing.T) { ioshim := common.NewMockIOShim(tt.calls) defer ioshim.VerifyCalls(t, tt.calls) pMgr := NewPolicyManager(ioshim, ipsetConfig) - err := pMgr.bootup(nil) + err := pMgr.bootupAfterDetectAndCleanup() if tt.wantErr { require.Error(t, err) } else { @@ -506,7 +500,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { calls: []testutils.TestCmd{ {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, wantErr: false, @@ -516,7 +510,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { calls: []testutils.TestCmd{ {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, wantErr: true, @@ -550,8 +544,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "AZURE-NPM"}, Stdout: "2 AZURE-NPM all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, wantErr: false, @@ -564,7 +558,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "AZURE-NPM"}, Stdout: "2 AZURE-NPM all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, wantErr: true, @@ -577,8 +571,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "AZURE-NPM"}, Stdout: "2 AZURE-NPM all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, }, placeAzureChainFirst: util.PlaceAzureChainFirst, wantErr: true, @@ -590,7 +584,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "KUBE-SERVICES"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: false, @@ -605,7 +599,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: false, @@ -620,8 +614,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { }, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "KUBE-SERVICES"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: false, @@ -636,8 +630,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { }, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "KUBE-SERVICES"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: false, @@ -672,8 +666,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: false, @@ -691,8 +685,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: false, @@ -719,7 +713,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "4", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: true, @@ -737,7 +731,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: true, @@ -755,8 +749,8 @@ func TestPositionAzureChainJumpRule(t *testing.T) { Cmd: []string{"grep", "KUBE-SERVICES"}, Stdout: "3 KUBE-SERVICES all -- 0.0.0.0/0 0.0.0.0/0 ...", }, - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "3", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 1}, }, placeAzureChainFirst: util.PlaceAzureChainAfterKubeServices, wantErr: true, @@ -772,6 +766,7 @@ func TestPositionAzureChainJumpRule(t *testing.T) { PlaceAzureChainFirst: tt.placeAzureChainFirst, } pMgr := NewPolicyManager(ioshim, cfg) + err := pMgr.positionAzureChainJumpRule() if tt.wantErr { require.Error(t, err) @@ -863,6 +858,7 @@ func TestChainLineNumber(t *testing.T) { ioshim := common.NewMockIOShim(tt.calls) defer ioshim.VerifyCalls(t, tt.calls) pMgr := NewPolicyManager(ioshim, ipsetConfig) + lineNum, err := pMgr.chainLineNumber(testChainName) if tt.wantErr { require.Error(t, err) @@ -875,7 +871,7 @@ func TestChainLineNumber(t *testing.T) { } func getFakeDestroyCommand(chain string) testutils.TestCmd { - return testutils.TestCmd{Cmd: []string{"iptables", "-w", "60", "-X", chain}} + return testutils.TestCmd{Cmd: []string{"iptables-nft", "-w", "60", "-X", chain}} } func getFakeDestroyCommandWithExitCode(chain string, exitCode int) testutils.TestCmd { @@ -894,3 +890,508 @@ func stringsToMap(items []string) map[string]struct{} { } return m } + +func TestDetectIptablesVersion(t *testing.T) { + type args struct { + name string + calls []testutils.TestCmd + expectedIptablesVersion string + } + + tests := []args{ + { + name: "nft has hint chain", + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 0, + }, + }, + expectedIptablesVersion: util.IptablesNft, + }, + { + name: "nft has only canary chain", + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-KUBELET-CANARY", "-t", "mangle", "-n"}, + ExitCode: 0, + }, + }, + expectedIptablesVersion: util.IptablesNft, + }, + { + name: "legacy has hint chain", + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-KUBELET-CANARY", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 0, + }, + }, + expectedIptablesVersion: util.IptablesLegacy, + }, + { + name: "no kube chains: default nft", + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-KUBELET-CANARY", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-L", "KUBE-KUBELET-CANARY", "-t", "mangle", "-n"}, + ExitCode: 1, + }, + }, + expectedIptablesVersion: util.IptablesNft, + }, + { + name: "nft and legacy both fail: default nft", + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 2, + }, + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-KUBELET-CANARY", "-t", "mangle", "-n"}, + ExitCode: 2, + }, + { + Cmd: []string{"iptables", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 2, + }, + { + Cmd: []string{"iptables", "-w", "60", "-L", "KUBE-KUBELET-CANARY", "-t", "mangle", "-n"}, + ExitCode: 2, + }, + }, + expectedIptablesVersion: util.IptablesNft, + }, + } + + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + metrics.InitializeAll() + + ioshim := common.NewMockIOShim(tt.calls) + defer ioshim.VerifyCalls(t, tt.calls) + cfg := &PolicyManagerCfg{ + NodeIP: "6.7.8.9", + PolicyMode: IPSetPolicyMode, + PlaceAzureChainFirst: util.PlaceAzureChainFirst, + } + pMgr := NewPolicyManager(ioshim, cfg) + pMgr.detectIptablesVersion() + + require.Equal(t, tt.expectedIptablesVersion, util.Iptables) + }) + } +} + +func TestCleanupOtherChains(t *testing.T) { + type args struct { + name string + startWithNft bool + calls []testutils.TestCmd + expectedErr bool + } + + tests := []args{ + { + name: "cleanup legacy jump no chains", + startWithNft: true, + calls: []testutils.TestCmd{ + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, // deprecated rule existed + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + ExitCode: 1, + }, + }, + expectedErr: false, + }, + { + name: "cleanup legacy jump and chains", + startWithNft: true, + calls: []testutils.TestCmd{ + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, // deprecated rule existed + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + {Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}}, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}}, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, + }, + expectedErr: false, + }, + { + name: "cleanup legacy retry flushes", + startWithNft: true, + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}}, + {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}}, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, + }, + expectedErr: false, + }, + { + name: "cleanup legacy error: delete/flush errors", + startWithNft: true, + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}, + ExitCode: 1, + }, + }, + expectedErr: true, + }, + { + name: "don't flush azure chain if it isn't there", + startWithNft: true, + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: "Chain AZURE-NPM-INGRESS (1 references)\n", + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}}, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}}, + }, + expectedErr: false, + }, + { + name: "cleanup legacy errors ok if deleted jump (non-deprecated)", + startWithNft: true, + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}, + ExitCode: 1, + }, + }, + expectedErr: false, + }, + { + name: "cleanup legacy errors ok if deleted jump (deprecated)", + startWithNft: true, + calls: []testutils.TestCmd{ + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}, + ExitCode: 2, + }, + { + Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}, + ExitCode: 2, + }, + { + Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}, + ExitCode: 2, + }, + { + Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}, + ExitCode: 2, + }, + }, + expectedErr: false, + }, + { + name: "cleanup legacy other flush errors ok", + startWithNft: true, + calls: []testutils.TestCmd{ + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true, + ExitCode: 1, + }, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM"}}, + { + Cmd: []string{"iptables", "-w", "60", "-F", "AZURE-NPM-INGRESS"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}}, + { + Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM-INGRESS"}, + ExitCode: 1, + }, + }, + expectedErr: false, + }, + { + name: "cleanup legacy error: list error", + startWithNft: true, + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true, HasStartError: true, + ExitCode: 1, + }, + {Cmd: []string{"grep", "Chain AZURE-NPM"}}, + }, + expectedErr: true, + }, + { + name: "cleanup nft", + startWithNft: false, + calls: []testutils.TestCmd{ + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}}, // deprecated rule existed + { + Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + ExitCode: 1, + }, + }, + expectedErr: false, + }, + { + name: "cleanup nft error", + startWithNft: false, + calls: []testutils.TestCmd{ + { + Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: grepOutputTwoAzureChains, + }, + { + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + { + Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, + ExitCode: 1, + }, + {Cmd: []string{"iptables-nft", "-w", "60", "-F", "AZURE-NPM"}, ExitCode: 1}, + }, + expectedErr: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + ioshim := common.NewMockIOShim(tt.calls) + defer ioshim.VerifyCalls(t, tt.calls) + pMgr := NewPolicyManager(ioshim, ipsetConfig) + + if tt.startWithNft { + util.SetIptablesToNft() + } else { + util.SetIptablesToLegacy() + // set back to default + defer util.SetIptablesToNft() + } + + err := pMgr.cleanupOtherIptables() + if tt.expectedErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + if tt.startWithNft { + require.Equal(t, util.IptablesNft, util.Iptables) + } else { + require.Equal(t, util.IptablesLegacy, util.Iptables) + } + }) + } +} + +func TestCreatorForCleanup(t *testing.T) { + chains := []string{ + "AZURE-NPM", + "AZURE-NPM-INGRESS", + "AZURE-NPM-EGRESS", + "AZURE-NPM-ACCEPT", + } + + expectedLines := []string{ + "*filter", + "-F AZURE-NPM", + "-F AZURE-NPM-INGRESS", + "-F AZURE-NPM-EGRESS", + "-F AZURE-NPM-ACCEPT", + "COMMIT", + "", + } + + ioshim := common.NewMockIOShim(nil) + defer ioshim.VerifyCalls(t, nil) + pMgr := NewPolicyManager(ioshim, ipsetConfig) + creator := pMgr.creatorForCleanup(chains) + actualLines := strings.Split(creator.ToString(), "\n") + sortedActualLines := sortFlushes(actualLines) + sortedExpectedLines := sortFlushes(expectedLines) + dptestutils.AssertEqualLines(t, sortedExpectedLines, sortedActualLines) + assertStaleChainsContain(t, pMgr.staleChains, []string{}...) +} diff --git a/npm/pkg/dataplane/policies/policymanager_test.go b/npm/pkg/dataplane/policies/policymanager_test.go index 793dcda3f0..2ce9b27285 100644 --- a/npm/pkg/dataplane/policies/policymanager_test.go +++ b/npm/pkg/dataplane/policies/policymanager_test.go @@ -101,15 +101,21 @@ func (p promVals) testPrometheusMetrics(t *testing.T) { // see chain-management_linux_test.go for testing when an error occurs func TestBootup(t *testing.T) { metrics.ReinitializeAll() - calls := GetBootupTestCalls(false) + calls := GetBootupTestCalls() ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) pMgr := NewPolicyManager(ioshim, ipsetConfig) + // verify that the iptables is explicitly set to nft during bootup + util.SetIptablesToLegacy() + // set back to default + defer util.SetIptablesToNft() + metrics.IncNumACLRules() metrics.IncNumACLRules() require.NoError(t, pMgr.Bootup(epIDs)) + require.Equal(t, util.IptablesNft, util.Iptables) expectedNumACLs := 11 if util.IsWindowsDP() { @@ -126,7 +132,7 @@ func TestAddPolicy(t *testing.T) { ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) pMgr := NewPolicyManager(ioshim, ipsetConfig) - + util.SetIptablesToNft() require.NoError(t, pMgr.AddPolicies([]*NPMNetworkPolicy{testNetPol}, epList)) _, ok := pMgr.GetPolicy(testNetPol.PolicyKey) require.True(t, ok) @@ -144,6 +150,7 @@ func TestAddEmptyPolicy(t *testing.T) { testNetPol := testNetworkPolicy() ioshim := common.NewMockIOShim(nil) pMgr := NewPolicyManager(ioshim, ipsetConfig) + util.SetIptablesToNft() require.NoError(t, pMgr.AddPolicies([]*NPMNetworkPolicy{ { Namespace: "x", @@ -173,7 +180,7 @@ func TestGetPolicy(t *testing.T) { ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) pMgr := NewPolicyManager(ioshim, ipsetConfig) - + util.SetIptablesToNft() require.NoError(t, pMgr.AddPolicies([]*NPMNetworkPolicy{netpol}, epList)) require.True(t, pMgr.PolicyExists("x/test-netpol")) @@ -190,6 +197,7 @@ func TestRemovePolicy(t *testing.T) { ioshim := common.NewMockIOShim(calls) defer ioshim.VerifyCalls(t, calls) pMgr := NewPolicyManager(ioshim, ipsetConfig) + util.SetIptablesToNft() require.NoError(t, pMgr.AddPolicies([]*NPMNetworkPolicy{testNetPol}, epList)) require.NoError(t, pMgr.RemovePolicy(testNetPol.PolicyKey)) _, ok := pMgr.GetPolicy(testNetPol.PolicyKey) @@ -202,6 +210,7 @@ func TestRemoveNonexistentPolicy(t *testing.T) { metrics.ReinitializeAll() ioshim := common.NewMockIOShim(nil) pMgr := NewPolicyManager(ioshim, ipsetConfig) + util.SetIptablesToNft() require.NoError(t, pMgr.RemovePolicy("wrong-policy-key")) promVals{0, 0}.testPrometheusMetrics(t) } diff --git a/npm/pkg/dataplane/policies/testutils_linux.go b/npm/pkg/dataplane/policies/testutils_linux.go index caf8f51ddc..112d020e47 100644 --- a/npm/pkg/dataplane/policies/testutils_linux.go +++ b/npm/pkg/dataplane/policies/testutils_linux.go @@ -8,11 +8,10 @@ import ( ) var ( - fakeIPTablesRestoreCommand = testutils.TestCmd{Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}} - fakeIPTablesRestoreFailureCommand = testutils.TestCmd{Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 1} + fakeIPTablesRestoreCommand = testutils.TestCmd{Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}} + fakeIPTablesRestoreFailureCommand = testutils.TestCmd{Cmd: []string{"iptables-nft-restore", "-w", "60", "-T", "filter", "--noflush"}, ExitCode: 1} - listLineNumbersCommandStrings = []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L", "FORWARD", "--line-numbers"} - listAllCommandStrings = []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"} + listLineNumbersCommandStrings = []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L", "FORWARD", "--line-numbers"} ) func GetAddPolicyTestCalls(_ *NPMNetworkPolicy) []testutils.TestCmd { @@ -27,12 +26,12 @@ func GetRemovePolicyTestCalls(policy *NPMNetworkPolicy) []testutils.TestCmd { calls := []testutils.TestCmd{} hasIngress, hasEgress := policy.hasIngressAndEgress() if hasIngress { - deleteIngressJumpSpecs := []string{"iptables", "-w", "60", "-D", util.IptablesAzureIngressChain} + deleteIngressJumpSpecs := []string{"iptables-nft", "-w", "60", "-D", util.IptablesAzureIngressChain} deleteIngressJumpSpecs = append(deleteIngressJumpSpecs, ingressJumpSpecs(policy)...) calls = append(calls, testutils.TestCmd{Cmd: deleteIngressJumpSpecs}) } if hasEgress { - deleteEgressJumpSpecs := []string{"iptables", "-w", "60", "-D", util.IptablesAzureEgressChain} + deleteEgressJumpSpecs := []string{"iptables-nft", "-w", "60", "-D", util.IptablesAzureEgressChain} deleteEgressJumpSpecs = append(deleteEgressJumpSpecs, egressJumpSpecs(policy)...) calls = append(calls, testutils.TestCmd{Cmd: deleteEgressJumpSpecs}) } @@ -50,24 +49,28 @@ func GetRemovePolicyFailureTestCalls(policy *NPMNetworkPolicy) []testutils.TestC return append(calls, fakeIPTablesRestoreFailureCommand) } -func GetBootupTestCalls(addDetectCalls bool) []testutils.TestCmd { - detectIptable := []testutils.TestCmd{ - {Cmd: []string{"iptables-nft-save", "-t", "mangle"}, Stdout: ""}, //nolint // AZURE-NPM chain didn't exist - {Cmd: []string{"iptables-save", "-t", "mangle"}, Stdout: `# Generated by iptables-save v1.8.7 on Wed May 3 01:35:24 2023 - *mangle - :PREROUTING ACCEPT [0:0] - :INPUT ACCEPT [0:0] - :FORWARD ACCEPT [0:0] - :OUTPUT ACCEPT [0:0] - :POSTROUTING ACCEPT [0:0] - :KUBE-IPTABLES-HINT - [0:0] - :KUBE-KUBELET-CANARY - [0:0] - :KUBE-PROXY-CANARY - [0:0] - COMMIT`}, //nolint // AZURE-NPM chain didn't exist - } +func GetBootupTestCalls() []testutils.TestCmd { bootUp := []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, //nolint // AZURE-NPM chain didn't exist - {Cmd: listAllCommandStrings, PipedToCommand: true}, + // detect iptables version to be nft + { + Cmd: []string{"iptables-nft", "-w", "60", "-L", "KUBE-IPTABLES-HINT", "-t", "mangle", "-n"}, + ExitCode: 0, + }, + // legacy clean up + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, //nolint // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}, ExitCode: 2}, //nolint // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + { + // 1 AZURE-NPM chain + Cmd: []string{"grep", "Chain AZURE-NPM"}, + Stdout: `Chain AZURE-NPM (0 references) +`, + }, + {Cmd: []string{"iptables-restore", "-w", "60", "-T", "filter", "--noflush"}}, + {Cmd: []string{"iptables", "-w", "60", "-X", "AZURE-NPM"}}, + // nft bootup + {Cmd: []string{"iptables-nft", "-w", "60", "-D", "FORWARD", "-j", "AZURE-NPM"}, ExitCode: 2}, //nolint // AZURE-NPM chain didn't exist + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, ExitCode: 1, @@ -75,17 +78,13 @@ func GetBootupTestCalls(addDetectCalls bool) []testutils.TestCmd { fakeIPTablesRestoreCommand, {Cmd: listLineNumbersCommandStrings, PipedToCommand: true}, {Cmd: []string{"grep", "AZURE-NPM"}, ExitCode: 1}, - {Cmd: []string{"iptables", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, - } - - if addDetectCalls { - return append(detectIptable, bootUp...) + {Cmd: []string{"iptables-nft", "-w", "60", "-I", "FORWARD", "-j", "AZURE-NPM", "-m", "conntrack", "--ctstate", "NEW"}}, } return bootUp } func getFakeDeleteJumpCommand(chainName, jumpRule string) testutils.TestCmd { - args := []string{"iptables", "-w", "60", "-D", chainName} + args := []string{"iptables-nft", "-w", "60", "-D", chainName} args = append(args, strings.Split(jumpRule, " ")...) return testutils.TestCmd{Cmd: args} } diff --git a/npm/pkg/dataplane/policies/testutils_windows.go b/npm/pkg/dataplane/policies/testutils_windows.go index 914ea76986..dfc25dab94 100644 --- a/npm/pkg/dataplane/policies/testutils_windows.go +++ b/npm/pkg/dataplane/policies/testutils_windows.go @@ -10,6 +10,6 @@ func GetRemovePolicyTestCalls(_ *NPMNetworkPolicy) []testutils.TestCmd { return []testutils.TestCmd{} } -func GetBootupTestCalls(_ bool) []testutils.TestCmd { +func GetBootupTestCalls() []testutils.TestCmd { return []testutils.TestCmd{} } diff --git a/npm/util/const.go b/npm/util/const.go index 38885935be..25a954a354 100644 --- a/npm/util/const.go +++ b/npm/util/const.go @@ -2,13 +2,7 @@ // MIT License package util -import ( - "bytes" - "fmt" - "strings" - - "github.com/Azure/azure-container-networking/common" -) +import "k8s.io/klog" // kubernetes related constants. const ( @@ -28,10 +22,10 @@ const ( ) var ( - Iptables = IptablesLegacy + Iptables = IptablesNft Ip6tables = Ip6tablesLegacy //nolint (avoid warning to capitalize this p) - IptablesSave = IptablesSaveLegacy - IptablesRestore = IptablesRestoreLegacy + IptablesSave = IptablesSaveNft + IptablesRestore = IptablesRestoreNft ) // iptables related constants. @@ -91,6 +85,7 @@ const ( IptablesEstablishedState string = "ESTABLISHED" IptablesNewState string = "NEW" IptablesFilterTable string = "filter" + IptablesMangleTable string = "mangle" IptablesCommentModuleFlag string = "comment" IptablesCommentFlag string = "--comment" IptablesAddCommentFlag @@ -272,70 +267,16 @@ const ( FanOutServerID // for v2 ) -func DetectIptablesVersion(ioShim *common.IOShim) { - cmd := ioShim.Exec.Command(IptablesSaveNft, "-t", "mangle") - - output, err := cmd.CombinedOutput() - if err != nil { - fmt.Printf("Error running iptables-nft-save: %s", err) - return - } - - if strings.Contains(string(output), "KUBE-IPTABLES-HINT") || strings.Contains(string(output), "KUBE-KUBELET-CANARY") { - Iptables = IptablesNft - IptablesSave = IptablesSaveNft - IptablesRestore = IptablesRestoreNft - } else { - lCmd := ioShim.Exec.Command(IptablesSaveLegacy, "-t", "mangle") - - loutput, err := lCmd.CombinedOutput() - if err != nil { - fmt.Printf("Error running iptables-legacy-save: %s", err) - return - } - - if strings.Contains(string(loutput), "KUBE-IPTABLES-HINT") || strings.Contains(string(loutput), "KUBE-KUBELET-CANARY") { - Iptables = IptablesLegacy - IptablesSave = IptablesSaveLegacy - IptablesRestore = IptablesRestoreLegacy - } else { - lsavecmd := ioShim.Exec.Command(IptablesSaveNft) - lsaveoutput, err := lsavecmd.CombinedOutput() - if err != nil { - fmt.Printf("Error running iptables-nft-save: %s", err) - return - } - - lcount := countLines(lsaveoutput) - - savecmd := ioShim.Exec.Command(IptablesSaveLegacy) - saveoutput, err := savecmd.CombinedOutput() - if err != nil { - fmt.Printf("Error running iptables-legacy-save: %s", err) - return - } - - count := countLines(saveoutput) - - if lcount > count { - Iptables = IptablesLegacy - IptablesSave = IptablesSaveLegacy - IptablesRestore = IptablesRestoreLegacy - } else { - Iptables = IptablesNft - IptablesSave = IptablesSaveNft - IptablesRestore = IptablesRestoreNft - } - } - } +func SetIptablesToNft() { + klog.Info("setting iptables to nft") + Iptables = IptablesNft + IptablesSave = IptablesSaveNft + IptablesRestore = IptablesRestoreNft } -func countLines(output []byte) int { - count := 0 - for _, x := range bytes.Split(output, []byte("\n")) { - if len(x) >= 1 && x[0] == '-' { - count++ - } - } - return count +func SetIptablesToLegacy() { + klog.Info("setting iptables to legacy") + Iptables = IptablesLegacy + IptablesSave = IptablesSaveLegacy + IptablesRestore = IptablesRestoreLegacy } diff --git a/npm/util/ioutil/current-azure-chains_test.go b/npm/util/ioutil/current-azure-chains_test.go index f2640d7884..107311394b 100644 --- a/npm/util/ioutil/current-azure-chains_test.go +++ b/npm/util/ioutil/current-azure-chains_test.go @@ -17,7 +17,7 @@ Chain AZURE-NPM-INGRESS-123456 (1 references) Chain AZURE-NPM-INGRESS-ALLOW-MARK (1 references) ` -var listAllCommandStrings = []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"} +var listAllCommandStrings = []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"} func TestAllCurrentAzureChains(t *testing.T) { tests := []struct { @@ -41,7 +41,7 @@ func TestAllCurrentAzureChains(t *testing.T) { { name: "ignore missing newline at end of grep result", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: `Chain AZURE-NPM (1 references) @@ -54,7 +54,7 @@ Chain AZURE-NPM-INGRESS (1 references)`, { name: "ignore unexpected grep line (chain name too short)", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: `Chain AZURE-NPM (1 references) @@ -69,7 +69,7 @@ Chain AZURE-NPM-INGRESS (1 references) { name: "ignore unexpected grep line (no space)", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: `Chain AZURE-NPM (1 references) @@ -83,7 +83,7 @@ Chain AZURE-NPM-INGRESS (1 references) { name: "success with no chains", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, {Cmd: []string{"grep", "Chain AZURE-NPM"}, ExitCode: 1}, }, expectedChains: nil, @@ -92,7 +92,7 @@ Chain AZURE-NPM-INGRESS (1 references) { name: "grep failure", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true, HasStartError: true, ExitCode: 1}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true, HasStartError: true, ExitCode: 1}, {Cmd: []string{"grep", "Chain AZURE-NPM"}}, }, expectedChains: nil, @@ -101,7 +101,7 @@ Chain AZURE-NPM-INGRESS (1 references) { name: "invalid grep result", calls: []testutils.TestCmd{ - {Cmd: []string{"iptables", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, + {Cmd: []string{"iptables-nft", "-w", "60", "-t", "filter", "-n", "-L"}, PipedToCommand: true}, { Cmd: []string{"grep", "Chain AZURE-NPM"}, Stdout: "",