Skip to content

Commit b7f6742

Browse files
tamilmani1989sharmasushant
authored andcommitted
CNI to support transparent mode (#279)
* added changes in azure cni to support transparent mode * cni for calico policy controller * removed unused parameter * minor fix * addressed review comments * addressed review comments * modified vethname generation and the hostbveth prefix * removed setting arp for default gw * minor fix
1 parent f816f8e commit b7f6742

File tree

7 files changed

+201
-14
lines changed

7 files changed

+201
-14
lines changed

cni/network/network.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -167,6 +167,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
167167
result *cniTypesCurr.Result
168168
azIpamResult *cniTypesCurr.Result
169169
err error
170+
vethName string
170171
nwCfg *cni.NetworkConfig
171172
epInfo *network.EndpointInfo
172173
iface *cniTypesCurr.Interface
@@ -388,17 +389,16 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
388389
// Network already exists.
389390
subnetPrefix := nwInfo.Subnets[0].Prefix.String()
390391
log.Printf("[cni-net] Found network %v with subnet %v.", networkId, subnetPrefix)
392+
nwCfg.Ipam.Subnet = subnetPrefix
391393

392394
// Call into IPAM plugin to allocate an address for the endpoint.
393-
nwCfg.Ipam.Subnet = subnetPrefix
394395
result, err = plugin.DelegateAdd(nwCfg.Ipam.Type, nwCfg)
395396
if err != nil {
396397
err = plugin.Errorf("Failed to allocate address: %v", err)
397398
return err
398399
}
399400

400401
ipconfig := result.IPs[0]
401-
402402
iface := &cniTypesCurr.Interface{Name: args.IfName}
403403
result.Interfaces = append(result.Interfaces, iface)
404404

@@ -456,7 +456,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
456456

457457
// A runtime must not call ADD twice (without a corresponding DEL) for the same
458458
// (network name, container id, name of the interface inside the container)
459-
vethName := fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName)
459+
vethName = fmt.Sprintf("%s%s%s", networkId, k8sContainerID, k8sIfName)
460460
setEndpointOptions(cnsNetworkConfig, epInfo, vethName)
461461

462462
// Create the endpoint.

netlink/ip.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,17 @@ import (
1111
"golang.org/x/sys/unix"
1212
)
1313

14+
const (
15+
RT_SCOPE_UNIVERSE = 0
16+
RT_SCOPE_SITE = 200
17+
RT_SCOPE_LINK = 253
18+
RT_SCOPE_HOST = 254
19+
RT_SCOPE_NOWHERE = 255
20+
)
21+
const (
22+
RTPROT_KERNEL = 2
23+
)
24+
1425
// GetIpAddressFamily returns the address family of an IP address.
1526
func GetIpAddressFamily(ip net.IP) int {
1627
if len(ip) <= net.IPv4len {

network/endpoint.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,12 @@ type EndpointInfo struct {
6464

6565
// RouteInfo contains information about an IP route.
6666
type RouteInfo struct {
67-
Dst net.IPNet
68-
Gw net.IP
69-
DevName string
67+
Dst net.IPNet
68+
Src net.IP
69+
Gw net.IP
70+
Protocol int
71+
DevName string
72+
Scope int
7073
}
7174

7275
// NewEndpoint creates a new endpoint in the network.

network/endpoint_linux.go

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,14 +81,19 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
8181
}
8282

8383
if vlanid != 0 {
84+
log.Printf("OVS client")
8485
epClient = NewOVSEndpointClient(
8586
nw.extIf,
8687
epInfo,
8788
hostIfName,
8889
contIfName,
8990
vlanid)
90-
} else {
91+
} else if nw.Mode != opModeTransparent {
92+
log.Printf("Bridge client")
9193
epClient = NewLinuxBridgeEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode)
94+
} else {
95+
log.Printf("Transparent client")
96+
epClient = NewTransparentEndpointClient(nw.extIf, hostIfName, contIfName, nw.Mode)
9297
}
9398

9499
// Cleanup on failure.
@@ -207,8 +212,10 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error {
207212
if ep.VlanID != 0 {
208213
epInfo := ep.getInfo()
209214
epClient = NewOVSEndpointClient(nw.extIf, epInfo, ep.HostIfName, "", ep.VlanID)
210-
} else {
215+
} else if nw.Mode != opModeTransparent {
211216
epClient = NewLinuxBridgeEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode)
217+
} else {
218+
epClient = NewTransparentEndpointClient(nw.extIf, ep.HostIfName, "", nw.Mode)
212219
}
213220

214221
epClient.DeleteEndpointRules(ep)
@@ -246,7 +253,7 @@ func addRoutes(interfaceName string, routes []RouteInfo) error {
246253
if !strings.Contains(strings.ToLower(err.Error()), "file exists") {
247254
return err
248255
} else {
249-
log.Printf("route already exists")
256+
log.Printf("[net] route already exists")
250257
}
251258
}
252259
}
@@ -259,7 +266,7 @@ func deleteRoutes(interfaceName string, routes []RouteInfo) error {
259266
interfaceIf, _ := net.InterfaceByName(interfaceName)
260267

261268
for _, route := range routes {
262-
log.Printf("[ovs] Deleting IP route %+v from link %v.", route, interfaceName)
269+
log.Printf("[net] Deleting IP route %+v from link %v.", route, interfaceName)
263270

264271
if route.DevName != "" {
265272
devIf, _ := net.InterfaceByName(route.DevName)
@@ -416,3 +423,14 @@ func updateRoutes(existingEp *EndpointInfo, targetEp *EndpointInfo) error {
416423

417424
return nil
418425
}
426+
427+
func getDefaultGateway(routes []RouteInfo) net.IP {
428+
_, defDstIP, _ := net.ParseCIDR("0.0.0.0/0")
429+
for _, route := range routes {
430+
if route.Dst.String() == defDstIP.String() {
431+
return route.Gw
432+
}
433+
}
434+
435+
return nil
436+
}

network/network.go

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,10 @@ import (
1414

1515
const (
1616
// Operational modes.
17-
opModeBridge = "bridge"
18-
opModeTunnel = "tunnel"
19-
opModeDefault = opModeTunnel
17+
opModeBridge = "bridge"
18+
opModeTunnel = "tunnel"
19+
opModeTransparent = "transparent"
20+
opModeDefault = opModeTunnel
2021
)
2122

2223
// ExternalInterface is a host network interface that bridges containers to external networks.

network/network_linux.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInt
5252
if opt != nil && opt[VlanIDKey] != nil {
5353
vlanid, _ = strconv.Atoi(opt[VlanIDKey].(string))
5454
}
55-
55+
case opModeTransparent:
56+
break
5657
default:
5758
return nil, errNetworkModeInvalid
5859
}
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
package network
2+
3+
import (
4+
"fmt"
5+
"net"
6+
7+
"github.com/Azure/azure-container-networking/log"
8+
"github.com/Azure/azure-container-networking/netlink"
9+
"github.com/Azure/azure-container-networking/network/epcommon"
10+
"github.com/Azure/azure-container-networking/platform"
11+
)
12+
13+
const (
14+
FAKE_GW_IP = "169.254.1.1/32"
15+
DEFAULT_GW = "0.0.0.0/0"
16+
)
17+
18+
type TransparentEndpointClient struct {
19+
bridgeName string
20+
hostPrimaryIfName string
21+
hostVethName string
22+
containerVethName string
23+
hostPrimaryMac net.HardwareAddr
24+
containerMac net.HardwareAddr
25+
hostVethMac net.HardwareAddr
26+
mode string
27+
}
28+
29+
func NewTransparentEndpointClient(
30+
extIf *externalInterface,
31+
hostVethName string,
32+
containerVethName string,
33+
mode string,
34+
) *TransparentEndpointClient {
35+
36+
client := &TransparentEndpointClient{
37+
bridgeName: extIf.BridgeName,
38+
hostPrimaryIfName: extIf.Name,
39+
hostVethName: hostVethName,
40+
containerVethName: containerVethName,
41+
hostPrimaryMac: extIf.MacAddress,
42+
mode: mode,
43+
}
44+
45+
return client
46+
}
47+
48+
func setArpProxy(ifName string) error {
49+
cmd := fmt.Sprintf("echo 1 > /proc/sys/net/ipv4/conf/%v/proxy_arp", ifName)
50+
_, err := platform.ExecuteCommand(cmd)
51+
return err
52+
}
53+
54+
func (client *TransparentEndpointClient) AddEndpoints(epInfo *EndpointInfo) error {
55+
if err := epcommon.CreateEndpoint(client.hostVethName, client.containerVethName); err != nil {
56+
return err
57+
}
58+
59+
containerIf, err := net.InterfaceByName(client.containerVethName)
60+
if err != nil {
61+
return err
62+
}
63+
64+
client.containerMac = containerIf.HardwareAddr
65+
66+
hostVethIf, err := net.InterfaceByName(client.hostVethName)
67+
if err != nil {
68+
return err
69+
}
70+
71+
client.hostVethMac = hostVethIf.HardwareAddr
72+
73+
return nil
74+
}
75+
76+
func (client *TransparentEndpointClient) AddEndpointRules(epInfo *EndpointInfo) error {
77+
var routeInfoList []RouteInfo
78+
79+
// ip route add <podip> dev <hostveth>
80+
// This route is needed for incoming packets to pod to route via hostveth
81+
for _, ipAddr := range epInfo.IPAddresses {
82+
var routeInfo RouteInfo
83+
ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)}
84+
log.Printf("[net] Adding route for the ip %v", ipNet.String())
85+
routeInfo.Dst = ipNet
86+
routeInfoList = append(routeInfoList, routeInfo)
87+
if err := addRoutes(client.hostVethName, routeInfoList); err != nil {
88+
return err
89+
}
90+
}
91+
92+
log.Printf("calling setArpProxy for %v", client.hostVethName)
93+
if err := setArpProxy(client.hostVethName); err != nil {
94+
log.Printf("setArpProxy failed with: %v", err)
95+
return err
96+
}
97+
98+
return nil
99+
}
100+
101+
func (client *TransparentEndpointClient) DeleteEndpointRules(ep *endpoint) {
102+
var routeInfoList []RouteInfo
103+
104+
// ip route del <podip> dev <hostveth>
105+
// Deleting the route set up for routing the incoming packets to pod
106+
for _, ipAddr := range ep.IPAddresses {
107+
var routeInfo RouteInfo
108+
ipNet := net.IPNet{IP: ipAddr.IP, Mask: net.CIDRMask(32, 32)}
109+
log.Printf("[net] Deleting route for the ip %v", ipNet.String())
110+
routeInfo.Dst = ipNet
111+
routeInfoList = append(routeInfoList, routeInfo)
112+
deleteRoutes(client.hostVethName, routeInfoList)
113+
}
114+
}
115+
116+
func (client *TransparentEndpointClient) MoveEndpointsToContainerNS(epInfo *EndpointInfo, nsID uintptr) error {
117+
// Move the container interface to container's network namespace.
118+
log.Printf("[net] Setting link %v netns %v.", client.containerVethName, epInfo.NetNsPath)
119+
if err := netlink.SetLinkNetNs(client.containerVethName, nsID); err != nil {
120+
return err
121+
}
122+
123+
return nil
124+
}
125+
126+
func (client *TransparentEndpointClient) SetupContainerInterfaces(epInfo *EndpointInfo) error {
127+
if err := epcommon.SetupContainerInterface(client.containerVethName, epInfo.IfName); err != nil {
128+
return err
129+
}
130+
131+
client.containerVethName = epInfo.IfName
132+
133+
return nil
134+
}
135+
136+
func (client *TransparentEndpointClient) ConfigureContainerInterfacesAndRoutes(epInfo *EndpointInfo) error {
137+
if err := epcommon.AssignIPToInterface(client.containerVethName, epInfo.IPAddresses); err != nil {
138+
return err
139+
}
140+
141+
return addRoutes(client.containerVethName, epInfo.Routes)
142+
}
143+
144+
func (client *TransparentEndpointClient) DeleteEndpoints(ep *endpoint) error {
145+
log.Printf("[net] Deleting veth pair %v %v.", ep.HostIfName, ep.IfName)
146+
err := netlink.DeleteLink(ep.HostIfName)
147+
if err != nil {
148+
log.Printf("[net] Failed to delete veth pair %v: %v.", ep.HostIfName, err)
149+
return err
150+
}
151+
152+
return nil
153+
}

0 commit comments

Comments
 (0)