Skip to content

Commit 73fa63a

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#92035 from danwinship/unmark-before-masq
kubelet, kube-proxy: unmark packets before masquerading them
2 parents b224790 + c12534d commit 73fa63a

File tree

5 files changed

+56
-19
lines changed

5 files changed

+56
-19
lines changed

‎pkg/kubelet/kubelet_network_linux.go‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ func (kl *Kubelet) syncNetworkUtil() {
6262
klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubeMarkDropChain, err)
6363
return
6464
}
65-
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--set-xmark", dropMark); err != nil {
65+
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil {
6666
klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkDropChain, err)
6767
return
6868
}
@@ -72,7 +72,7 @@ func (kl *Kubelet) syncNetworkUtil() {
7272
}
7373
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
7474
"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
75-
"-m", "mark", "--mark", dropMark,
75+
"-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark),
7676
"-j", "DROP"); err != nil {
7777
klog.Errorf("Failed to ensure rule to drop packet marked by %v in %v chain %v: %v", KubeMarkDropChain, utiliptables.TableFilter, KubeFirewallChain, err)
7878
return
@@ -112,7 +112,7 @@ func (kl *Kubelet) syncNetworkUtil() {
112112
klog.Errorf("Failed to ensure that %s chain %s exists: %v", utiliptables.TableNAT, KubePostroutingChain, err)
113113
return
114114
}
115-
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--set-xmark", masqueradeMark); err != nil {
115+
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil {
116116
klog.Errorf("Failed to ensure marking rule for %v: %v", KubeMarkMasqChain, err)
117117
return
118118
}
@@ -121,12 +121,26 @@ func (kl *Kubelet) syncNetworkUtil() {
121121
klog.Errorf("Failed to ensure that %s chain %s jumps to %s: %v", utiliptables.TableNAT, utiliptables.ChainPostrouting, KubePostroutingChain, err)
122122
return
123123
}
124-
// Establish the masquerading rule.
124+
125+
// Set up KUBE-POSTROUTING to unmark and masquerade marked packets
125126
// NB: THIS MUST MATCH the corresponding code in the iptables and ipvs
126127
// modes of kube-proxy
128+
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
129+
"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark),
130+
"-j", "RETURN"); err != nil {
131+
klog.Errorf("Failed to ensure filtering rule for %v: %v", KubePostroutingChain, err)
132+
return
133+
}
134+
// Clear the mark to avoid re-masquerading if the packet re-traverses the network stack.
135+
// We know the mark bit is currently set so we can use --xor-mark to clear it (without needing
136+
// to Sprintf another bitmask).
137+
if _, err := kl.iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
138+
"-j", "MARK", "--xor-mark", masqueradeMark); err != nil {
139+
klog.Errorf("Failed to ensure unmarking rule for %v: %v", KubePostroutingChain, err)
140+
return
141+
}
127142
masqRule := []string{
128143
"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
129-
"-m", "mark", "--mark", masqueradeMark,
130144
"-j", "MASQUERADE",
131145
}
132146
if kl.iptClient.HasRandomFully() {
@@ -141,5 +155,5 @@ func (kl *Kubelet) syncNetworkUtil() {
141155
// getIPTablesMark returns the fwmark given the bit
142156
func getIPTablesMark(bit int) string {
143157
value := 1 << uint(bit)
144-
return fmt.Sprintf("%#08x/%#08x", value, value)
158+
return fmt.Sprintf("%#08x", value)
145159
}

‎pkg/kubelet/kubelet_network_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,11 @@ func TestGetIPTablesMark(t *testing.T) {
3131
}{
3232
{
3333
14,
34-
"0x00004000/0x00004000",
34+
"0x00004000",
3535
},
3636
{
3737
15,
38-
"0x00008000/0x00008000",
38+
"0x00008000",
3939
},
4040
}
4141
for _, tc := range tests {

‎pkg/proxy/iptables/proxier.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ func NewProxier(ipt utiliptables.Interface,
282282

283283
// Generate the masquerade mark to use for SNAT rules.
284284
masqueradeValue := 1 << uint(masqueradeBit)
285-
masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue)
285+
masqueradeMark := fmt.Sprintf("%#08x", masqueradeValue)
286286
klog.V(2).Infof("iptables(%s) masquerade mark: %s", ipt.Protocol(), masqueradeMark)
287287

288288
endpointSlicesEnabled := utilfeature.DefaultFeatureGate.Enabled(features.EndpointSliceProxying)
@@ -925,10 +925,20 @@ func (proxier *Proxier) syncProxyRules() {
925925
// this so that it is easier to flush and change, for example if the mark
926926
// value should ever change.
927927
// NB: THIS MUST MATCH the corresponding code in the kubelet
928+
writeLine(proxier.natRules, []string{
929+
"-A", string(kubePostroutingChain),
930+
"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", proxier.masqueradeMark, proxier.masqueradeMark),
931+
"-j", "RETURN",
932+
}...)
933+
// Clear the mark to avoid re-masquerading if the packet re-traverses the network stack.
934+
writeLine(proxier.natRules, []string{
935+
"-A", string(kubePostroutingChain),
936+
// XOR proxier.masqueradeMark to unset it
937+
"-j", "MARK", "--xor-mark", proxier.masqueradeMark,
938+
}...)
928939
masqRule := []string{
929940
"-A", string(kubePostroutingChain),
930941
"-m", "comment", "--comment", `"kubernetes service traffic requiring SNAT"`,
931-
"-m", "mark", "--mark", proxier.masqueradeMark,
932942
"-j", "MASQUERADE",
933943
}
934944
if proxier.iptables.HasRandomFully() {
@@ -941,7 +951,7 @@ func (proxier *Proxier) syncProxyRules() {
941951
// value should ever change.
942952
writeLine(proxier.natRules, []string{
943953
"-A", string(KubeMarkMasqChain),
944-
"-j", "MARK", "--set-xmark", proxier.masqueradeMark,
954+
"-j", "MARK", "--or-mark", proxier.masqueradeMark,
945955
}...)
946956

947957
// Accumulate NAT chains to keep.
@@ -1520,7 +1530,7 @@ func (proxier *Proxier) syncProxyRules() {
15201530
writeLine(proxier.filterRules,
15211531
"-A", string(kubeForwardChain),
15221532
"-m", "comment", "--comment", `"kubernetes forwarding rules"`,
1523-
"-m", "mark", "--mark", proxier.masqueradeMark,
1533+
"-m", "mark", "--mark", fmt.Sprintf("%s/%s", proxier.masqueradeMark, proxier.masqueradeMark),
15241534
"-j", "ACCEPT",
15251535
)
15261536

‎pkg/proxy/iptables/proxier_test.go‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,7 @@ func NewFakeProxier(ipt utiliptables.Interface, endpointSlicesEnabled bool) *Pro
357357
endpointsMap: make(proxy.EndpointsMap),
358358
endpointsChanges: proxy.NewEndpointChangeTracker(testHostname, newEndpointInfo, nil, nil, endpointSlicesEnabled),
359359
iptables: ipt,
360+
masqueradeMark: "0x4000",
360361
localDetector: detectLocal,
361362
hostname: testHostname,
362363
portsMap: make(map[utilproxy.LocalPort]utilproxy.Closeable),
@@ -2418,7 +2419,7 @@ func TestEndpointSliceE2E(t *testing.T) {
24182419
:KUBE-EXTERNAL-SERVICES - [0:0]
24192420
:KUBE-FORWARD - [0:0]
24202421
-A KUBE-FORWARD -m conntrack --ctstate INVALID -j DROP
2421-
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark -j ACCEPT
2422+
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding rules" -m mark --mark 0x4000/0x4000 -j ACCEPT
24222423
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod source rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
24232424
-A KUBE-FORWARD -m comment --comment "kubernetes forwarding conntrack pod destination rule" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
24242425
COMMIT
@@ -2431,8 +2432,10 @@ COMMIT
24312432
:KUBE-SEP-3JOIVZTXZZRGORX4 - [0:0]
24322433
:KUBE-SEP-IO5XOSKPAXIFQXAJ - [0:0]
24332434
:KUBE-SEP-XGJFVO3L2O5SRFNT - [0:0]
2434-
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -m mark --mark -j MASQUERADE
2435-
-A KUBE-MARK-MASQ -j MARK --set-xmark
2435+
-A KUBE-POSTROUTING -m mark ! --mark 0x4000/0x4000 -j RETURN
2436+
-A KUBE-POSTROUTING -j MARK --xor-mark 0x4000
2437+
-A KUBE-POSTROUTING -m comment --comment "kubernetes service traffic requiring SNAT" -j MASQUERADE
2438+
-A KUBE-MARK-MASQ -j MARK --or-mark 0x4000
24362439
-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 0 ! -s 10.0.0.0/24 -j KUBE-MARK-MASQ
24372440
-A KUBE-SERVICES -m comment --comment "ns1/svc1 cluster IP" -m tcp -p tcp -d 172.20.1.1/32 --dport 0 -j KUBE-SVC-AQI2S6QIMU7PVVRP
24382441
-A KUBE-SVC-AQI2S6QIMU7PVVRP -m comment --comment ns1/svc1 -m statistic --mode random --probability 0.3333333333 -j KUBE-SEP-3JOIVZTXZZRGORX4

‎pkg/proxy/ipvs/proxier.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -418,7 +418,7 @@ func NewProxier(ipt utiliptables.Interface,
418418

419419
// Generate the masquerade mark to use for SNAT rules.
420420
masqueradeValue := 1 << uint(masqueradeBit)
421-
masqueradeMark := fmt.Sprintf("%#08x/%#08x", masqueradeValue, masqueradeValue)
421+
masqueradeMark := fmt.Sprintf("%#08x", masqueradeValue)
422422

423423
isIPv6 := utilnet.IsIPv6(nodeIP)
424424

@@ -1765,7 +1765,7 @@ func (proxier *Proxier) writeIptablesRules() {
17651765
writeLine(proxier.filterRules,
17661766
"-A", string(KubeForwardChain),
17671767
"-m", "comment", "--comment", `"kubernetes forwarding rules"`,
1768-
"-m", "mark", "--mark", proxier.masqueradeMark,
1768+
"-m", "mark", "--mark", fmt.Sprintf("%s/%s", proxier.masqueradeMark, proxier.masqueradeMark),
17691769
"-j", "ACCEPT",
17701770
)
17711771

@@ -1849,10 +1849,20 @@ func (proxier *Proxier) createAndLinkeKubeChain() {
18491849
// this so that it is easier to flush and change, for example if the mark
18501850
// value should ever change.
18511851
// NB: THIS MUST MATCH the corresponding code in the kubelet
1852+
writeLine(proxier.natRules, []string{
1853+
"-A", string(kubePostroutingChain),
1854+
"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", proxier.masqueradeMark, proxier.masqueradeMark),
1855+
"-j", "RETURN",
1856+
}...)
1857+
// Clear the mark to avoid re-masquerading if the packet re-traverses the network stack.
1858+
writeLine(proxier.natRules, []string{
1859+
"-A", string(kubePostroutingChain),
1860+
// XOR proxier.masqueradeMark to unset it
1861+
"-j", "MARK", "--xor-mark", proxier.masqueradeMark,
1862+
}...)
18521863
masqRule := []string{
18531864
"-A", string(kubePostroutingChain),
18541865
"-m", "comment", "--comment", `"kubernetes service traffic requiring SNAT"`,
1855-
"-m", "mark", "--mark", proxier.masqueradeMark,
18561866
"-j", "MASQUERADE",
18571867
}
18581868
if proxier.iptables.HasRandomFully() {
@@ -1865,7 +1875,7 @@ func (proxier *Proxier) createAndLinkeKubeChain() {
18651875
// value should ever change.
18661876
writeLine(proxier.natRules, []string{
18671877
"-A", string(KubeMarkMasqChain),
1868-
"-j", "MARK", "--set-xmark", proxier.masqueradeMark,
1878+
"-j", "MARK", "--or-mark", proxier.masqueradeMark,
18691879
}...)
18701880
}
18711881

0 commit comments

Comments
 (0)