Skip to content

Commit f0907b4

Browse files
matmerrmatmerr
authored
refactor: Move CNI bridge/transparent routes to common (#694)
* fix: pass host gateway to CNI with Swift to enable Swift+Transparent
1 parent d68c75c commit f0907b4

File tree

7 files changed

+166
-155
lines changed

7 files changed

+166
-155
lines changed

‎acncli/deployment/manager.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,17 @@ spec:
1818
hostNetwork: true
1919
containers:
2020
- name: azure-cni-installer
21-
image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.1.9-alphav1
21+
image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63
2222
imagePullPolicy: Always
2323
env:
2424
- name: AZURE_CNI_OS
2525
value: linux
2626
- name: AZURE_CNI_TENANCY
2727
value: singletenancy
2828
- name: AZURE_CNI_MODE
29-
value: bridge
29+
value: transparent
3030
- name: AZURE_CNI_IPAM
31-
value: azure-cns
31+
value: azure-vnet-ipam
3232
- name: AZURE_CNI_EXEMPT
3333
value: azure-vnet-telemetry,azure-vnet-telemetry.config
3434
volumeMounts:

‎cni/network/invoker_cns.go‎

Lines changed: 63 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"github.com/Azure/azure-container-networking/cni"
1010
"github.com/Azure/azure-container-networking/cns"
1111
"github.com/Azure/azure-container-networking/cns/cnsclient"
12+
"github.com/Azure/azure-container-networking/iptables"
1213
"github.com/Azure/azure-container-networking/log"
1314
"github.com/Azure/azure-container-networking/network"
1415
cniTypes "github.com/containernetworking/cni/pkg/types"
@@ -48,7 +49,7 @@ func NewCNSInvoker(podName, namespace string) (*CNSIPAMInvoker, error) {
4849
}
4950

5051
//Add uses the requestipconfig API in cns, and returns ipv4 and a nil ipv6 as CNS doesn't support IPv6 yet
51-
func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
52+
func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, options map[string]interface{}) (*cniTypesCurr.Result, *cniTypesCurr.Result, error) {
5253

5354
// Parse Pod arguments.
5455
podInfo := cns.KubernetesPodInfo{PodName: invoker.podName, PodNamespace: invoker.podNamespace}
@@ -61,7 +62,7 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
6162
return nil, nil, err
6263
}
6364

64-
resultIPv4 := IPv4ResultInfo{
65+
info := IPv4ResultInfo{
6566
podIPAddress: response.PodIpInfo.PodIPConfig.IPAddress,
6667
ncSubnetPrefix: response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.PrefixLength,
6768
ncPrimaryIP: response.PodIpInfo.NetworkContainerPrimaryIPConfig.IPSubnet.IPAddress,
@@ -71,20 +72,46 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
7172
hostGateway: response.PodIpInfo.HostPrimaryIPInfo.Gateway,
7273
}
7374

74-
ncgw := net.ParseIP(resultIPv4.ncGatewayIPAddress)
75+
// set the NC Primary IP in options
76+
options[network.SNATIPKey] = info.ncPrimaryIP
77+
78+
log.Printf("[cni-invoker-cns] Received info %v for pod %v", info, podInfo)
79+
80+
ncgw := net.ParseIP(info.ncGatewayIPAddress)
7581
if ncgw == nil {
76-
return nil, nil, fmt.Errorf("Gateway address %v from response is invalid", resultIPv4.ncGatewayIPAddress)
82+
return nil, nil, fmt.Errorf("Gateway address %v from response is invalid", info.ncGatewayIPAddress)
7783
}
7884

79-
// set the NC Primary IP in options
80-
options[network.SNATIPKey] = resultIPv4.ncPrimaryIP
85+
// set result ipconfig from CNS Response Body
86+
ip, ncipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix))
87+
if ip == nil {
88+
return nil, nil, fmt.Errorf("Unable to parse IP from response: %v with err %v", info.podIPAddress, err)
89+
}
8190

82-
// set host gateway in options
83-
options[network.HostGWKey] = resultIPv4.hostGateway
91+
// construct ipnet for result
92+
resultIPnet := net.IPNet{
93+
IP: ip,
94+
Mask: ncipnet.Mask,
95+
}
8496

85-
log.Printf("Received result %+v for pod %v", resultIPv4, podInfo)
97+
result := &cniTypesCurr.Result{
98+
IPs: []*cniTypesCurr.IPConfig{
99+
{
100+
Version: "4",
101+
Address: resultIPnet,
102+
Gateway: ncgw,
103+
},
104+
},
105+
Routes: []*cniTypes.Route{
106+
{
107+
Dst: network.Ipv4DefaultRouteDstPrefix,
108+
GW: ncgw,
109+
},
110+
},
111+
}
86112

87-
result, err := getCNIIPv4Result(resultIPv4, subnetPrefix)
113+
// set subnet prefix for host vm
114+
err = setHostOptions(nwCfg, hostSubnetPrefix, ncipnet, options, info)
88115
if err != nil {
89116
return nil, nil, err
90117
}
@@ -93,54 +120,44 @@ func (invoker *CNSIPAMInvoker) Add(nwCfg *cni.NetworkConfig, subnetPrefix *net.I
93120
return result, nil, nil
94121
}
95122

96-
func getCNIIPv4Result(info IPv4ResultInfo, subnetPrefix *net.IPNet) (*cniTypesCurr.Result, error) {
97-
98-
gw := net.ParseIP(info.ncGatewayIPAddress)
99-
if gw == nil {
100-
return nil, fmt.Errorf("Gateway address %v from response is invalid", gw)
123+
func setHostOptions(nwCfg *cni.NetworkConfig, hostSubnetPrefix *net.IPNet, ncSubnetPrefix *net.IPNet, options map[string]interface{}, info IPv4ResultInfo) error {
124+
// get the name of the primary IP address
125+
_, hostIPNet, err := net.ParseCIDR(info.hostSubnet)
126+
if err != nil {
127+
return err
101128
}
102129

130+
*hostSubnetPrefix = *hostIPNet
131+
132+
// get the host ip
103133
hostIP := net.ParseIP(info.hostPrimaryIP)
104134
if hostIP == nil {
105-
return nil, fmt.Errorf("Host IP address %v from response is invalid", hostIP)
135+
return fmt.Errorf("Host IP address %v from response is invalid", info.hostPrimaryIP)
106136
}
107137

108-
// set result ipconfig from CNS Response Body
109-
ip, ipnet, err := net.ParseCIDR(info.podIPAddress + "/" + fmt.Sprint(info.ncSubnetPrefix))
110-
if ip == nil {
111-
return nil, fmt.Errorf("Unable to parse IP from response: %v", info.podIPAddress)
138+
// get host gateway
139+
hostGateway := net.ParseIP(info.hostGateway)
140+
if hostGateway == nil {
141+
return fmt.Errorf("Host Gateway %v from response is invalid", info.hostGateway)
112142
}
113143

114-
// get the name of the primary IP address
115-
_, hostIPNet, err := net.ParseCIDR(info.hostSubnet)
116-
if err != nil {
117-
return nil, err
144+
// this route is needed when the vm on subnet A needs to send traffic to a pod in subnet B on a different vm
145+
options[network.RoutesKey] = []network.RouteInfo{
146+
{
147+
Dst: *ncSubnetPrefix,
148+
Gw: hostGateway,
149+
},
118150
}
119151

120-
// set subnet prefix for host vm
121-
*subnetPrefix = *hostIPNet
122-
123-
// construct ipnet for result
124-
resultIPnet := net.IPNet{
125-
IP: ip,
126-
Mask: ipnet.Mask,
152+
azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", ncSubnetPrefix.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
153+
snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, info.ncPrimaryIP)
154+
options[network.IPTablesKey] = []iptables.IPTableEntry{
155+
iptables.GetCreateChainCmd(iptables.V4, iptables.Nat, iptables.Swift),
156+
iptables.GetAppendIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift),
157+
iptables.GetInsertIptableRuleCmd(iptables.V4, iptables.Nat, iptables.Swift, azureDNSMatch, snatPrimaryIPJump),
127158
}
128159

129-
return &cniTypesCurr.Result{
130-
IPs: []*cniTypesCurr.IPConfig{
131-
{
132-
Version: "4",
133-
Address: resultIPnet,
134-
Gateway: gw,
135-
},
136-
},
137-
Routes: []*cniTypes.Route{
138-
{
139-
Dst: network.Ipv4DefaultRouteDstPrefix,
140-
GW: gw,
141-
},
142-
},
143-
}, nil
160+
return nil
144161
}
145162

146163
// Delete calls into the releaseipconfiguration API in CNS

‎iptables/iptables.go‎

Lines changed: 35 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,13 @@ var (
8585
DisableIPTableLock bool
8686
)
8787

88+
type IPTableEntry struct {
89+
Version string
90+
Params string
91+
}
92+
8893
// Run iptables command
89-
func runCmd(version, params string) error {
94+
func RunCmd(version, params string) error {
9095
var cmd string
9196

9297
iptCmd := iptables
@@ -110,20 +115,26 @@ func runCmd(version, params string) error {
110115
// check if iptable chain alreay exists
111116
func ChainExists(version, tableName, chainName string) bool {
112117
params := fmt.Sprintf("-t %s -L %s", tableName, chainName)
113-
if err := runCmd(version, params); err != nil {
118+
if err := RunCmd(version, params); err != nil {
114119
return false
115120
}
116121

117122
return true
118123
}
124+
func GetCreateChainCmd(version, tableName, chainName string) IPTableEntry {
125+
return IPTableEntry{
126+
Version: version,
127+
Params: fmt.Sprintf("-t %s -N %s", tableName, chainName),
128+
}
129+
}
119130

120131
// create new iptable chain under specified table name
121132
func CreateChain(version, tableName, chainName string) error {
122133
var err error
123134

124135
if !ChainExists(version, tableName, chainName) {
125-
params := fmt.Sprintf("-t %s -N %s", tableName, chainName)
126-
err = runCmd(version, params)
136+
cmd := GetCreateChainCmd(version, tableName, chainName)
137+
err = RunCmd(version, cmd.Params)
127138
} else {
128139
log.Printf("%s Chain exists in table %s", chainName, tableName)
129140
}
@@ -134,21 +145,35 @@ func CreateChain(version, tableName, chainName string) error {
134145
// check if iptable rule alreay exists
135146
func RuleExists(version, tableName, chainName, match, target string) bool {
136147
params := fmt.Sprintf("-t %s -C %s %s -j %s", tableName, chainName, match, target)
137-
if err := runCmd(version, params); err != nil {
148+
if err := RunCmd(version, params); err != nil {
138149
return false
139150
}
140151
return true
141152
}
142153

154+
func GetInsertIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry {
155+
return IPTableEntry{
156+
Version: version,
157+
Params: fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target),
158+
}
159+
}
160+
143161
// Insert iptable rule at beginning of iptable chain
144162
func InsertIptableRule(version, tableName, chainName, match, target string) error {
145163
if RuleExists(version, tableName, chainName, match, target) {
146164
log.Printf("Rule already exists")
147165
return nil
148166
}
149167

150-
params := fmt.Sprintf("-t %s -I %s 1 %s -j %s", tableName, chainName, match, target)
151-
return runCmd(version, params)
168+
cmd := GetInsertIptableRuleCmd(version, tableName, chainName, match, target)
169+
return RunCmd(version, cmd.Params)
170+
}
171+
172+
func GetAppendIptableRuleCmd(version, tableName, chainName, match, target string) IPTableEntry {
173+
return IPTableEntry{
174+
Version: version,
175+
Params: fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target),
176+
}
152177
}
153178

154179
// Append iptable rule at end of iptable chain
@@ -158,12 +183,12 @@ func AppendIptableRule(version, tableName, chainName, match, target string) erro
158183
return nil
159184
}
160185

161-
params := fmt.Sprintf("-t %s -A %s %s -j %s", tableName, chainName, match, target)
162-
return runCmd(version, params)
186+
cmd := GetAppendIptableRuleCmd(version, tableName, chainName, match, target)
187+
return RunCmd(version, cmd.Params)
163188
}
164189

165190
// Delete matched iptable rule
166191
func DeleteIptableRule(version, tableName, chainName, match, target string) error {
167192
params := fmt.Sprintf("-t %s -D %s %s -j %s", tableName, chainName, match, target)
168-
return runCmd(version, params)
193+
return RunCmd(version, params)
169194
}

‎network/bridge_networkclient_linux.go‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package network
22

33
import (
4-
"fmt"
54
"net"
6-
"strings"
75

86
"github.com/Azure/azure-container-networking/ebtables"
97
"github.com/Azure/azure-container-networking/log"
@@ -48,59 +46,6 @@ func (client *LinuxBridgeClient) CreateBridge() error {
4846
return epcommon.DisableRAForInterface(client.bridgeName)
4947
}
5048

51-
func (client *LinuxBridgeClient) AddRoutes(nwInfo *NetworkInfo, interfaceName string) error {
52-
if client.nwInfo.IPAMType == AzureCNS {
53-
54-
// fetch the host gateway IP from options
55-
gwIP := client.nwInfo.Options[HostGWKey]
56-
if gwIP == nil {
57-
return fmt.Errorf("Host gateway IP in Options not set")
58-
}
59-
60-
gatewayIP := net.ParseIP(gwIP.(string))
61-
if gatewayIP == nil {
62-
return fmt.Errorf("Invalid host gateway IP: %+v", gwIP)
63-
}
64-
65-
// add host gateway as the default gateway for pod IP's
66-
devIf, _ := net.InterfaceByName(interfaceName)
67-
ifIndex := devIf.Index
68-
family := netlink.GetIpAddressFamily(gatewayIP)
69-
70-
nlRoute := &netlink.Route{
71-
Family: family,
72-
Dst: &client.nwInfo.PodSubnet.Prefix,
73-
Gw: gatewayIP,
74-
LinkIndex: ifIndex,
75-
}
76-
77-
log.Printf("Adding Swift route %+v", nlRoute)
78-
79-
if err := netlink.AddIpRoute(nlRoute); err != nil {
80-
if !strings.Contains(strings.ToLower(err.Error()), "file exists") {
81-
return fmt.Errorf("Failed to add route to host interface with error: %v", err)
82-
}
83-
log.Printf("[cni-cns-net] route already exists: dst %+v, gw %+v, interfaceName %v", nlRoute.Dst, nlRoute.Gw, interfaceName)
84-
}
85-
86-
// Add snat Rules
87-
snatIP := client.nwInfo.Options[SNATIPKey]
88-
if snatIP == nil {
89-
return fmt.Errorf("snatIP in Options not set %v", snatIP)
90-
}
91-
92-
ncPrimaryIP := net.ParseIP(fmt.Sprintf("%v", snatIP))
93-
if ncPrimaryIP == nil {
94-
return fmt.Errorf("Failed to parse SNAT IP from options %v", client.nwInfo.Options)
95-
}
96-
97-
log.Printf("Adding SNAT rule with snat IP %+v for subnet %s", ncPrimaryIP, client.nwInfo.PodSubnet.Prefix)
98-
return epcommon.SNATfromSubnetToDNSWithNCPrimaryIP(ncPrimaryIP, client.nwInfo.PodSubnet.Prefix)
99-
100-
}
101-
return nil
102-
}
103-
10449
func (client *LinuxBridgeClient) DeleteBridge() error {
10550
// Disconnect external interface from its bridge.
10651
err := netlink.SetLinkMaster(client.hostInterfaceName, "")

‎network/epcommon/endpoint_common.go‎

Lines changed: 0 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -240,39 +240,6 @@ func AddSnatRule(match string, ip net.IP) error {
240240
return iptables.InsertIptableRule(version, iptables.Nat, iptables.Postrouting, match, target)
241241
}
242242

243-
// SNATfromSubnetToDNSWithNCPrimaryIP snat's the snattedAddressSpace with the ipForSnat IP
244-
func SNATfromSubnetToDNSWithNCPrimaryIP(ipForSNAT net.IP, snattedAddressSpace net.IPNet) (err error) {
245-
246-
// Create SWIFT chain, this checks if the chain already exists
247-
// Check if theres a primary IP
248-
if ipForSNAT != nil {
249-
// Create SWIFT chain, this checks if the chain already exists
250-
log.Printf("Creating SWIFT chain...")
251-
err := iptables.CreateChain(iptables.V4, iptables.Nat, iptables.Swift)
252-
if err != nil {
253-
return err
254-
}
255-
256-
log.Printf("Creating SWIFT chain jump from POSTROUTING")
257-
// add jump to SWIFT chain from POSTROUTING
258-
err = iptables.AppendIptableRule(iptables.V4, iptables.Nat, iptables.Postrouting, "", iptables.Swift)
259-
if err != nil {
260-
return err
261-
}
262-
263-
log.Printf("Adding rule to SNAT subnet %v DNS requests with ip %v", snattedAddressSpace, ipForSNAT)
264-
// SNAT requests to Azure DNS
265-
azureDNSMatch := fmt.Sprintf(" -m addrtype ! --dst-type local -s %s -d %s -p %s --dport %d", snattedAddressSpace.String(), iptables.AzureDNS, iptables.UDP, iptables.DNSPort)
266-
snatPrimaryIPJump := fmt.Sprintf("%s --to %s", iptables.Snat, ipForSNAT)
267-
err = iptables.InsertIptableRule(iptables.V4, iptables.Nat, iptables.Swift, azureDNSMatch, snatPrimaryIPJump)
268-
if err != nil {
269-
return err
270-
}
271-
}
272-
273-
return nil
274-
}
275-
276243
func DisableRAForInterface(ifName string) error {
277244
raFilePath := fmt.Sprintf(acceptRAV6File, ifName)
278245
exist, err := common.CheckIfFileExists(raFilePath)

0 commit comments

Comments
 (0)