Added support for Linux l2tunnel mode

ofiliz · ofiliz · commit 278ce7587bac · 2017-03-06T16:24:20.000-08:00
diff --git a/cni/netconfig.go b/cni/netconfig.go
@@ -12,6 +12,7 @@ type NetworkConfig struct {
 	CniVersion string `json:"cniVersion"`
 	Name       string `json:"name"`
 	Type       string `json:"type"`
+	Mode       string `json:"mode"`
 	Master     string `json:"master"`
 	Bridge     string `json:"bridge,omitempty"`
 	LogLevel   string `json:"logLevel,omitempty"`
diff --git a/cni/network/network.go b/cni/network/network.go
@@ -135,6 +135,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		// Create the network.
 		nwInfo := network.NetworkInfo{
 			Id:         networkId,
+			Mode:       nwCfg.Mode,
 			Subnets:    []string{subnet.String()},
 			BridgeName: nwCfg.Bridge,
 		}
diff --git a/cnm/api.go b/cnm/api.go
@@ -9,8 +9,13 @@ const (
 
 	// Libnetwork remote plugin paths
 	activatePath = "/Plugin.Activate"
+
+	// Libnetwork labels
+	genericData = "com.docker.network.generic"
 )
 
+type OptionMap map[string]interface{}
+
 //
 // Libnetwork remote plugin API
 //
diff --git a/cnm/network/api.go b/cnm/network/api.go
@@ -16,6 +16,9 @@ const (
 	joinPath             = "/NetworkDriver.Join"
 	leavePath            = "/NetworkDriver.Leave"
 	endpointOperInfoPath = "/NetworkDriver.EndpointOperInfo"
+
+	// Libnetwork network plugin options
+	modeOption = "com.microsoft.azure.network.mode"
 )
 
 // Request sent by libnetwork when querying plugin capabilities.
diff --git a/cnm/network/network.go b/cnm/network/network.go
@@ -132,6 +132,12 @@ func (plugin *netPlugin) createNetwork(w http.ResponseWriter, r *http.Request) {
 		Options: req.Options,
 	}
 
+	// Parse network options.
+	options := plugin.ParseOptions(req.Options)
+	if options != nil {
+		nwInfo.Mode, _ = options[modeOption].(string)
+	}
+
 	// Assume single pool per address family.
 	if len(req.IPv4Data) > 0 {
 		nwInfo.Subnets = append(nwInfo.Subnets, req.IPv4Data[0].Pool)
diff --git a/cnm/plugin.go b/cnm/plugin.go
@@ -78,6 +78,12 @@ func (plugin *Plugin) Uninitialize() {
 	plugin.Plugin.Uninitialize()
 }
 
+// ParseOptions returns generic options from a libnetwork request.
+func (plugin *Plugin) ParseOptions(options OptionMap) OptionMap {
+	opt, _ := options[genericData].(OptionMap)
+	return opt
+}
+
 //
 // Libnetwork remote plugin API
 //
diff --git a/ebtables/ebtables.go b/ebtables/ebtables.go
@@ -36,26 +36,44 @@ func installEbtables() {
 // SetSnatForInterface sets a MAC SNAT rule for an interface.
 func SetSnatForInterface(interfaceName string, macAddress net.HardwareAddr, action string) error {
 	command := fmt.Sprintf(
-		"ebtables -t nat %s POSTROUTING -o %s -j snat --to-src %s --snat-arp",
+		"ebtables -t nat %s POSTROUTING -s unicast -o %s -j snat --to-src %s --snat-arp --snat-target ACCEPT",
 		action, interfaceName, macAddress.String())
 
 	return executeShellCommand(command)
 }
 
+// SetArpReply sets an ARP reply rule for the given target IP address and MAC address.
+func SetArpReply(ipAddress net.IP, macAddress net.HardwareAddr, action string) error {
+	command := fmt.Sprintf(
+		"ebtables -t nat %s PREROUTING -p ARP --arp-ip-dst %s -j arpreply --arpreply-mac %s --arpreply-target DROP",
+		action, ipAddress, macAddress.String())
+
+	return executeShellCommand(command)
+}
+
 // SetDnatForArpReplies sets a MAC DNAT rule for ARP replies received on an interface.
 func SetDnatForArpReplies(interfaceName string, action string) error {
 	command := fmt.Sprintf(
-		"ebtables -t nat %s PREROUTING -p ARP -i %s -j dnat --to-dst ff:ff:ff:ff:ff:ff",
+		"ebtables -t nat %s PREROUTING -p ARP -i %s -j dnat --to-dst ff:ff:ff:ff:ff:ff --dnat-target ACCEPT",
 		action, interfaceName)
 
 	return executeShellCommand(command)
 }
 
+// SetVepaMode sets the VEPA mode for an interface.
+func SetVepaMode(upstreamIfName string, upstreamMacAddress string, action string) error {
+	command := fmt.Sprintf(
+		"ebtables -t nat %s PREROUTING -i ! %s -j dnat --to-dst %s --dnat-target ACCEPT",
+		action, upstreamIfName, upstreamMacAddress)
+
+	return executeShellCommand(command)
+}
+
 // SetDnatForIPAddress sets a MAC DNAT rule for an IP address.
-func SetDnatForIPAddress(ipAddress net.IP, macAddress net.HardwareAddr, action string) error {
+func SetDnatForIPAddress(interfaceName string, ipAddress net.IP, macAddress net.HardwareAddr, action string) error {
 	command := fmt.Sprintf(
-		"ebtables -t nat %s PREROUTING -p IPv4 --ip-dst %s -j dnat --to-dst %s",
-		action, ipAddress.String(), macAddress.String())
+		"ebtables -t nat %s PREROUTING -p IPv4 -i %s --ip-dst %s -j dnat --to-dst %s --dnat-target ACCEPT",
+		action, interfaceName, ipAddress.String(), macAddress.String())
 
 	return executeShellCommand(command)
 }
diff --git a/network/api.go b/network/api.go
@@ -10,7 +10,7 @@ import (
 var (
 	// Error responses returned by NetworkManager.
 	errSubnetNotFound     = fmt.Errorf("Subnet not found")
-	errNetworkTypeInvalid = fmt.Errorf("Network type is invalid")
+	errNetworkModeInvalid = fmt.Errorf("Network mode is invalid")
 	errNetworkExists      = fmt.Errorf("Network already exists")
 	errNetworkNotFound    = fmt.Errorf("Network not found")
 	errEndpointExists     = fmt.Errorf("Endpoint already exists")
diff --git a/network/endpoint_linux.go b/network/endpoint_linux.go
@@ -80,7 +80,7 @@ func (nw *network) newEndpointImpl(epInfo *EndpointInfo) (*endpoint, error) {
 	// Setup MAC address translation rules for container interface.
 	log.Printf("[net] Setting up MAC address translation rules for endpoint %v.", contIfName)
 	for _, ipAddr := range epInfo.IPAddresses {
-		err = ebtables.SetDnatForIPAddress(ipAddr.IP, containerIf.HardwareAddr, ebtables.Append)
+		err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, containerIf.HardwareAddr, ebtables.Append)
 		if err != nil {
 			goto cleanup
 		}
@@ -205,7 +205,7 @@ func (nw *network) deleteEndpointImpl(ep *endpoint) error {
 	// Delete MAC address translation rule.
 	log.Printf("[net] Deleting MAC address translation rules for endpoint %v.", ep.Id)
 	for _, ipAddr := range ep.IPAddresses {
-		err = ebtables.SetDnatForIPAddress(ipAddr.IP, ep.MacAddress, ebtables.Delete)
+		err = ebtables.SetDnatForIPAddress(nw.extIf.Name, ipAddr.IP, ep.MacAddress, ebtables.Delete)
 		if err != nil {
 			goto cleanup
 		}
diff --git a/network/network.go b/network/network.go
@@ -11,7 +11,9 @@ import (
 
 const (
 	// Operational modes.
-	OpModeBridge = "bridge"
+	opModeBridge  = "bridge"
+	opModeTunnel  = "tunnel"
+	opModeDefault = opModeTunnel
 )
 
 // ExternalInterface is a host network interface that bridges containers to external networks.
@@ -104,6 +106,11 @@ func (nm *networkManager) newNetwork(nwInfo *NetworkInfo) (*network, error) {
 
 	log.Printf("[net] Creating network %+v.", nwInfo)
 
+	// Set defaults.
+	if nwInfo.Mode == "" {
+		nwInfo.Mode = opModeDefault
+	}
+
 	// Find the external interface for this subnet.
 	extIf := nm.findExternalInterfaceBySubnet(nwInfo.Subnets[0])
 	if extIf == nil {
diff --git a/network/network_linux.go b/network/network_linux.go
@@ -18,26 +18,27 @@ import (
 const (
 	// Prefix for bridge names.
 	bridgePrefix = "azure"
+
+	// Virtual MAC address used by Azure VNET.
+	virtualMacAddress = "12:34:56:78:9a:bc"
 )
 
 // Linux implementation of route.
 type route netlink.Route
 
 // NewNetworkImpl creates a new container network.
 func (nm *networkManager) newNetworkImpl(nwInfo *NetworkInfo, extIf *externalInterface) (*network, error) {
-	if nwInfo.Mode == "" {
-		nwInfo.Mode = OpModeBridge
-	}
-
 	// Connect the external interface.
 	switch nwInfo.Mode {
-	case OpModeBridge:
-		err := nm.connectExternalInterface(extIf, nwInfo.BridgeName)
+	case opModeTunnel:
+		fallthrough
+	case opModeBridge:
+		err := nm.connectExternalInterface(extIf, nwInfo)
 		if err != nil {
 			return nil, err
 		}
 	default:
-		return nil, errNetworkTypeInvalid
+		return nil, errNetworkModeInvalid
 	}
 
 	// Create the network object.
@@ -139,8 +140,54 @@ func (nm *networkManager) applyIPConfig(extIf *externalInterface, targetIf *net.
 	return nil
 }
 
+// AddBridgeRules adds bridge frame table rules for container traffic.
+func (nm *networkManager) addBridgeRules(extIf *externalInterface, hostIf *net.Interface, opMode string) error {
+	// Add SNAT rule to translate container egress traffic.
+	log.Printf("[net] Adding SNAT rule for egress traffic on %v.", hostIf.Name)
+	err := ebtables.SetSnatForInterface(hostIf.Name, hostIf.HardwareAddr, ebtables.Append)
+	if err != nil {
+		return err
+	}
+
+	// Add ARP reply rule for host primary IP address.
+	// ARP requests for all IP addresses are forwarded to the SDN fabric, but fabric
+	// doesn't respond to ARP requests from the VM for its own primary IP address.
+	primary := extIf.IPAddresses[0].IP
+	log.Printf("[net] Adding ARP reply rule for primary IP address %v.", primary)
+	err = ebtables.SetArpReply(primary, hostIf.HardwareAddr, ebtables.Append)
+	if err != nil {
+		return err
+	}
+
+	// Add DNAT rule to forward ARP replies to container interfaces.
+	log.Printf("[net] Adding DNAT rule for ingress ARP traffic on interface %v.", hostIf.Name)
+	err = ebtables.SetDnatForArpReplies(hostIf.Name, ebtables.Append)
+	if err != nil {
+		return err
+	}
+
+	// Enable VEPA for host policy enforcement if necessary.
+	if opMode == opModeTunnel {
+		log.Printf("[net] Enabling VEPA mode for %v.", hostIf.Name)
+		err = ebtables.SetVepaMode(hostIf.Name, virtualMacAddress, ebtables.Append)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// DeleteBridgeRules deletes bridge rules for container traffic.
+func (nm *networkManager) deleteBridgeRules(extIf *externalInterface) {
+	ebtables.SetVepaMode(extIf.Name, virtualMacAddress, ebtables.Delete)
+	ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete)
+	ebtables.SetArpReply(extIf.IPAddresses[0].IP, extIf.MacAddress, ebtables.Delete)
+	ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete)
+}
+
 // ConnectExternalInterface connects the given host interface to a bridge.
-func (nm *networkManager) connectExternalInterface(extIf *externalInterface, bridgeName string) error {
+func (nm *networkManager) connectExternalInterface(extIf *externalInterface, nwInfo *NetworkInfo) error {
 	log.Printf("[net] Connecting interface %v.", extIf.Name)
 
 	// Check whether this interface is already connected.
@@ -156,6 +203,7 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, bri
 	}
 
 	// If a bridge name is not specified, generate one based on the external interface index.
+	bridgeName := nwInfo.BridgeName
 	if bridgeName == "" {
 		bridgeName = fmt.Sprintf("%s%d", bridgePrefix, hostIf.Index)
 	}
@@ -193,14 +241,8 @@ func (nm *networkManager) connectExternalInterface(extIf *externalInterface, bri
 		log.Printf("[net] Failed to save IP configuration for interface %v: %v.", hostIf.Name, err)
 	}
 
-	// Setup MAC address translation rules for external interface.
-	log.Printf("[net] Setting up MAC address translation rules for %v.", hostIf.Name)
-	err = ebtables.SetSnatForInterface(hostIf.Name, hostIf.HardwareAddr, ebtables.Append)
-	if err != nil {
-		goto cleanup
-	}
-
-	err = ebtables.SetDnatForArpReplies(hostIf.Name, ebtables.Append)
+	// Add the bridge rules.
+	err = nm.addBridgeRules(extIf, hostIf, nwInfo.Mode)
 	if err != nil {
 		goto cleanup
 	}
@@ -249,9 +291,7 @@ cleanup:
 	log.Printf("[net] Connecting interface %v failed, err:%v.", extIf.Name, err)
 
 	// Roll back the changes for the network.
-	ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete)
-	ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete)
-
+	nm.deleteBridgeRules(extIf)
 	netlink.DeleteLink(bridgeName)
 
 	return err
@@ -261,9 +301,8 @@ cleanup:
 func (nm *networkManager) disconnectExternalInterface(extIf *externalInterface) error {
 	log.Printf("[net] Disconnecting interface %v.", extIf.Name)
 
-	// Cleanup MAC address translation rules.
-	ebtables.SetDnatForArpReplies(extIf.Name, ebtables.Delete)
-	ebtables.SetSnatForInterface(extIf.Name, extIf.MacAddress, ebtables.Delete)
+	// Delete bridge rules set on the external interface.
+	nm.deleteBridgeRules(extIf)
 
 	// Disconnect external interface from its bridge.
 	err := netlink.SetLinkMaster(extIf.Name, "")

Original file line number	Diff line number	Diff line change
`@@ -135,6 +135,7 @@ func (plugin netPlugin) Add(args cniSkel.CmdArgs) error {`
`135`	`135`	`// Create the network.`
`136`	`136`	`nwInfo := network.NetworkInfo{`
`137`	`137`	`Id: networkId,`
	`138`	`+ Mode: nwCfg.Mode,`
`138`	`139`	`Subnets: []string{subnet.String()},`
`139`	`140`	`BridgeName: nwCfg.Bridge,`
`140`	`141`	`}`
Original file line number	Diff line number	Diff line change
`@@ -16,6 +16,9 @@ const (`
`16`	`16`	`joinPath = "/NetworkDriver.Join"`
`17`	`17`	`leavePath = "/NetworkDriver.Leave"`
`18`	`18`	`endpointOperInfoPath = "/NetworkDriver.EndpointOperInfo"`
	`19`	`+`
	`20`	`+ // Libnetwork network plugin options`
	`21`	`+ modeOption = "com.microsoft.azure.network.mode"`
`19`	`22`	`)`
`20`	`23`
`21`	`24`	`// Request sent by libnetwork when querying plugin capabilities.`