Setup SNAT Configuration Based on Azure Host Support (#401)

* Save enable snat on host settings after querying NMagent version

* Adding changes to exclude outbound snat for win cni if new NMAgent is running

* try to acquire lock file when writing to disableSnatOnHost.json

* addressed some of Tamilmani's comments

* Adding snat for DNS if current NMAgent does not support it yet

* Adding DNS NAT changes for Windows CNI

* vendoring HCSShim changes that support destination based SNATing

* Reverting k8s.io/api dependencies from master branch to last working version

* Addressing Tamilmani's comments

* syncing with an older version of k8s.io dependencies

* verify valid windows version before Dns NAT.

* only remove snat on windows when host has full support

* addressing Tamilmani's comments

* addressing Tamilmani's comments

* rebased and re-depped

Author: jaer-tsun

Gopkg.lock

@@ -27,7 +27,7 @@
 
 [[constraint]]
   name = "github.com/Microsoft/go-winio"
-  version = "0.4.12"
+  version = "0.4.14"
 
 [[constraint]]
   name = "github.com/docker/libnetwork"
@@ -42,20 +42,20 @@
   name = "golang.org/x/sys"
 
 [[constraint]]
-  branch = "master"
   name = "k8s.io/api"
+  version = "kubernetes-1.13.6"
 
 [[constraint]]
   name = "k8s.io/apimachinery"
-  revision = "d7deff9243b165ee192f5551710ea4285dcfd615"
+  revision = "kubernetes-1.13.6"
 
 [[constraint]]
   name = "k8s.io/client-go"
-  version = "11.0.0"
+  version = "kubernetes-1.13.6"
 
 [[constraint]]
   name = "github.com/Microsoft/hcsshim"
-  version = "0.8.6"
+  revision = "2a08d6fcd23883573a39ba32ab7ed2b197a9a9eb"
 
 [[constraint]]
   name = "github.com/containernetworking/cni"

Gopkg.toml

@@ -26,7 +26,7 @@ func SetupRoutingForMultitenancy(
 	result *cniTypesCurr.Result) {
 	// Adding default gateway
 	if nwCfg.MultiTenancy {
-		// if snat enabled, add 169.254.0.1 as default gateway
+		// if snat enabled, add 169.254.128.1 as default gateway
 		if nwCfg.EnableSnatOnHost {
 			log.Printf("add default route for multitenancy.snat on host enabled")
 			addDefaultRoute(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result)
@@ -36,6 +36,11 @@ func SetupRoutingForMultitenancy(
 			gwIP := net.ParseIP(cnsNetworkConfig.IPConfiguration.GatewayIPAddress)
 			epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: dstIP, Gw: gwIP})
 			result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP})
+
+			if epInfo.EnableSnatForDns {
+				log.Printf("add SNAT for DNS enabled")
+				addSnatForDNS(cnsNetworkConfig.LocalIPConfiguration.GatewayIPAddress, epInfo, result)
+			}
 		}
 
 		setupInfraVnetRoutingForMultitenancy(nwCfg, azIpamResult, epInfo, result)

cni/network/multitenancy.go

@@ -6,7 +6,12 @@ package network
 import (
 	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"net"
+	"net/http"
+	"os"
+	"strings"
+	"time"
 
 	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/cns"
@@ -35,13 +40,34 @@ const (
 	CNI_UPDATE = "UPDATE"
 )
 
+const (
+	// URL to query NMAgent version and determine whether we snat on host
+	nmAgentSupportedApisURL = "http://168.63.129.16/machine/plugins/?comp=nmagent&type=GetSupportedApis"
+	// Only SNAT support (no DNS support)
+	nmAgentSnatSupportAPI = "NetworkManagementSnatSupport"
+	// SNAT and DNS are both supported
+	nmAgentSnatAndDnsSupportAPI = "NetworkManagementDNSSupport"
+)
+
+// temporary consts related func determineSnat() which is to be deleted after
+// a baking period with newest NMAgent changes
+const (
+	jsonFileExtension = ".json"
+)
+
 // NetPlugin represents the CNI network plugin.
 type netPlugin struct {
 	*cni.Plugin
 	nm     network.NetworkManager
 	report *telemetry.CNIReport
 }
 
+// snatConfiguration contains a bool that determines whether CNI enables snat on host and snat for dns
+type snatConfiguration struct {
+	EnableSnatOnHost bool
+	EnableSnatForDns bool
+}
+
 // NewPlugin creates a new netPlugin object.
 func NewPlugin(name string, config *common.PluginConfig) (*netPlugin, error) {
 	// Setup base plugin.
@@ -190,6 +216,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		subnetPrefix     net.IPNet
 		cnsNetworkConfig *cns.GetNetworkContainerResponse
 		enableInfraVnet  bool
+		enableSnatForDns bool
 		nwDNSInfo        network.DNSInfo
 	)
 
@@ -205,6 +232,13 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 
 	log.Printf("[cni-net] Read network configuration %+v.", nwCfg)
 
+	// Temporary if block to determing whether we disable SNAT on host (for multi-tenant scenario only)
+	if nwCfg.MultiTenancy {
+		if enableSnatForDns, nwCfg.EnableSnatOnHost, err = determineSnat(); err != nil {
+			return err
+		}
+	}
+
 	plugin.setCNIReportDetails(nwCfg, CNI_ADD, "")
 
 	defer func() {
@@ -453,6 +487,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		EnableSnatOnHost:   nwCfg.EnableSnatOnHost,
 		EnableMultiTenancy: nwCfg.MultiTenancy,
 		EnableInfraVnet:    enableInfraVnet,
+		EnableSnatForDns:   enableSnatForDns,
 		PODName:            k8sPodName,
 		PODNameSpace:       k8sNamespace,
 		SkipHotAttachEp:    false, // Hot attach at the time of endpoint creation
@@ -855,3 +890,68 @@ func (plugin *netPlugin) Update(args *cniSkel.CmdArgs) error {
 
 	return nil
 }
+
+// Temporary function to determine whether we need to disable SNAT due to NMAgent support
+func determineSnat() (bool, bool, error) {
+	var (
+		snatConfig            snatConfiguration
+		retrieveSnatConfigErr error
+		jsonFile              *os.File
+		httpClient            = &http.Client{Timeout: time.Second * 5}
+		snatConfigFile        = snatConfigFileName + jsonFileExtension
+	)
+
+	// Check if we've already retrieved NMAgent version and determined whether to disable snat on host
+	if jsonFile, retrieveSnatConfigErr = os.Open(snatConfigFile); retrieveSnatConfigErr == nil {
+		bytes, _ := ioutil.ReadAll(jsonFile)
+		jsonFile.Close()
+		if retrieveSnatConfigErr = json.Unmarshal(bytes, &snatConfig); retrieveSnatConfigErr != nil {
+			log.Errorf("[cni-net] failed to unmarshal to snatConfig with error %v",
+				retrieveSnatConfigErr)
+		}
+
+	}
+
+	// If we weren't able to retrieve snatConfiguration, query NMAgent
+	if retrieveSnatConfigErr != nil {
+		var resp *http.Response
+		resp, retrieveSnatConfigErr = httpClient.Get(nmAgentSupportedApisURL)
+		if retrieveSnatConfigErr == nil {
+			defer resp.Body.Close()
+
+			if resp.StatusCode == http.StatusOK {
+				var bodyBytes []byte
+				// if the list of APIs (strings) contains the nmAgentSnatSupportAPI we will disable snat on host
+				if bodyBytes, retrieveSnatConfigErr = ioutil.ReadAll(resp.Body); retrieveSnatConfigErr == nil {
+					bodyStr := string(bodyBytes)
+					if !strings.Contains(bodyStr, nmAgentSnatAndDnsSupportAPI) {
+						snatConfig.EnableSnatForDns = true
+						snatConfig.EnableSnatOnHost = !strings.Contains(bodyStr, nmAgentSnatSupportAPI)
+					}
+
+					jsonStr, _ := json.Marshal(snatConfig)
+					fp, err := os.OpenFile(snatConfigFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(0664))
+					if err == nil {
+						fp.Write(jsonStr)
+						fp.Close()
+					} else {
+						log.Printf("[cni-net] failed to save snatConfig")
+					}
+				}
+			} else {
+				retrieveSnatConfigErr = fmt.Errorf("nmagent request status code %d", resp.StatusCode)
+			}
+		}
+	}
+
+	// Log and return the error when we fail acquire snat configuration for host and dns
+	if retrieveSnatConfigErr != nil {
+		log.Errorf("[cni-net] failed to acquire SNAT configuration with error %v",
+			retrieveSnatConfigErr)
+		return snatConfig.EnableSnatForDns, snatConfig.EnableSnatOnHost, retrieveSnatConfigErr
+	}
+
+	log.Printf("[cni-net] EnableSnatOnHost set to %t; EnableSnatForDns set to %t", snatConfig.EnableSnatOnHost, snatConfig.EnableSnatForDns)
+
+	return snatConfig.EnableSnatForDns, snatConfig.EnableSnatOnHost, nil
+}

cni/network/network.go

@@ -20,6 +20,10 @@ const (
 	infraInterface = "eth2"
 )
 
+const (
+	snatConfigFileName = "/tmp/snatConfig"
+)
+
 // handleConsecutiveAdd is a dummy function for Linux platform.
 func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *network.NetworkInfo, nwCfg *cni.NetworkConfig) (*cniTypesCurr.Result, error) {
 	return nil, nil
@@ -33,6 +37,13 @@ func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cn
 	result.Routes = append(result.Routes, &cniTypes.Route{Dst: dstIP, GW: gwIP})
 }
 
+func addSnatForDNS(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) {
+	_, dnsIPNet, _ := net.ParseCIDR("168.63.129.16/32")
+	gwIP := net.ParseIP(gwIPString)
+	epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: *dnsIPNet, Gw: gwIP, DevName: snatInterface})
+	result.Routes = append(result.Routes, &cniTypes.Route{Dst: *dnsIPNet, GW: gwIP})
+}
+
 func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Result, epInfo *network.EndpointInfo) {
 	for _, route := range azIpamResult.Routes {
 		epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW, DevName: infraInterface})

cni/network/network_linux.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"net"
+	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 
@@ -13,12 +15,19 @@ import (
 	"github.com/Azure/azure-container-networking/network"
 	"github.com/Azure/azure-container-networking/network/policy"
 	"github.com/Microsoft/hcsshim"
+	"golang.org/x/sys/windows/registry"
 
 	cniSkel "github.com/containernetworking/cni/pkg/skel"
 	cniTypes "github.com/containernetworking/cni/pkg/types"
 	cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
 )
 
+var (
+	snatConfigFileName = filepath.FromSlash(os.Getenv("TEMP")) + "\\snatConfig"
+	// windows build for version 1903
+	win1903Version = 18362
+)
+
 /* handleConsecutiveAdd handles consecutive add calls for infrastructure containers on Windows platform.
  * This is a temporary work around for issue #57253 of Kubernetes.
  * We can delete this if statement once they fix it.
@@ -73,6 +82,9 @@ func handleConsecutiveAdd(args *cniSkel.CmdArgs, endpointId string, nwInfo *netw
 func addDefaultRoute(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) {
 }
 
+func addSnatForDNS(gwIPString string, epInfo *network.EndpointInfo, result *cniTypesCurr.Result) {
+}
+
 func addInfraRoutes(azIpamResult *cniTypesCurr.Result, result *cniTypesCurr.Result, epInfo *network.EndpointInfo) {
 }
 
@@ -125,6 +137,7 @@ func getNetworkName(podName, podNs, ifName string, nwCfg *cni.NetworkConfig) (ne
 	networkName = nwCfg.Name
 	err = nil
 	if nwCfg.MultiTenancy {
+		determineWinVer()
 		if len(strings.TrimSpace(podName)) == 0 || len(strings.TrimSpace(podNs)) == 0 {
 			err = fmt.Errorf("POD info cannot be empty. PodName: %s, PodNamespace: %s", podName, podNs)
 			return
@@ -242,3 +255,22 @@ func getCustomDNS(nwCfg *cni.NetworkConfig) network.DNSInfo {
 		Options: nwCfg.RuntimeConfig.DNS.Options,
 	}
 }
+
+func determineWinVer() {
+	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
+	if err == nil {
+		defer k.Close()
+
+		cb, _, err := k.GetStringValue("CurrentBuild")
+		if err == nil {
+			winVer, err := strconv.Atoi(cb)
+			if err == nil {
+				policy.ValidWinVerForDnsNat = winVer >= win1903Version
+			}
+		}
+	}
+
+	if err != nil {
+		log.Errorf(err.Error())
+	}
+}

cni/network/network_windows.go

@@ -62,6 +62,7 @@ type EndpointInfo struct {
 	EnableSnatOnHost         bool
 	EnableInfraVnet          bool
 	EnableMultiTenancy       bool
+	EnableSnatForDns         bool
 	AllowInboundFromHostToNC bool
 	AllowInboundFromNCToHost bool
 	NetworkContainerID       string

network/endpoint.go

@@ -87,7 +87,7 @@ func (nw *network) newEndpointImplHnsV1(epInfo *EndpointInfo) (*endpoint, error)
 		VirtualNetwork: nw.HnsId,
 		DNSSuffix:      epInfo.DNS.Suffix,
 		DNSServerList:  strings.Join(epInfo.DNS.Servers, ","),
-		Policies:       policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data),
+		Policies:       policy.SerializePolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy),
 	}
 
 	// HNS currently supports only one IP address per endpoint.
@@ -175,7 +175,7 @@ func (nw *network) configureHcnEndpoint(epInfo *EndpointInfo) (*hcn.HostComputeE
 		MacAddress: epInfo.MacAddress.String(),
 	}
 
-	if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data); err == nil {
+	if endpointPolicies, err := policy.GetHcnEndpointPolicies(policy.EndpointPolicy, epInfo.Policies, epInfo.Data, epInfo.EnableSnatForDns, epInfo.EnableMultiTenancy); err == nil {
 		for _, epPolicy := range endpointPolicies {
 			hcnEndpoint.Policies = append(hcnEndpoint.Policies, epPolicy)
 		}

network/endpoint_windows.go

@@ -61,7 +61,7 @@ func (nm *networkManager) newNetworkImplHnsV1(nwInfo *NetworkInfo, extIf *extern
 		Name:               nwInfo.Id,
 		NetworkAdapterName: networkAdapterName,
 		DNSServerList:      strings.Join(nwInfo.DNS.Servers, ","),
-		Policies:           policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies, nil),
+		Policies:           policy.SerializePolicies(policy.NetworkPolicy, nwInfo.Policies, nil, false, false),
 	}
 
 	// Set the VLAN and OutboundNAT policies

network/network_windows.go

@@ -8,8 +8,7 @@ import (
 )
 
 func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP string, epInfo *EndpointInfo) {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC {
-
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		hostIfName := fmt.Sprintf("%s%s", snatVethInterfacePrefix, epInfo.Id[:7])
 		contIfName := fmt.Sprintf("%s%s-2", snatVethInterfacePrefix, epInfo.Id[:7])
 
@@ -18,7 +17,7 @@ func NewSnatClient(client *OVSEndpointClient, snatBridgeIP string, localIP strin
 }
 
 func AddSnatEndpoint(client *OVSEndpointClient) error {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost {
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		if err := client.snatClient.CreateSnatEndpoint(client.bridgeName); err != nil {
 			return err
 		}
@@ -28,7 +27,7 @@ func AddSnatEndpoint(client *OVSEndpointClient) error {
 }
 
 func AddSnatEndpointRules(client *OVSEndpointClient) error {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost {
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		// Allow specific Private IPs via Snat Bridge
 		if err := client.snatClient.AllowIPAddressesOnSnatBrdige(); err != nil {
 			return err
@@ -63,31 +62,31 @@ func AddSnatEndpointRules(client *OVSEndpointClient) error {
 }
 
 func MoveSnatEndpointToContainerNS(client *OVSEndpointClient, netnsPath string, nsID uintptr) error {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost {
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		return client.snatClient.MoveSnatEndpointToContainerNS(netnsPath, nsID)
 	}
 
 	return nil
 }
 
 func SetupSnatContainerInterface(client *OVSEndpointClient) error {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost {
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		return client.snatClient.SetupSnatContainerInterface()
 	}
 
 	return nil
 }
 
 func ConfigureSnatContainerInterface(client *OVSEndpointClient) error {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost {
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		return client.snatClient.ConfigureSnatContainerInterface()
 	}
 
 	return nil
 }
 
 func DeleteSnatEndpoint(client *OVSEndpointClient) error {
-	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost {
+	if client.enableSnatOnHost || client.allowInboundFromHostToNC || client.allowInboundFromNCToHost || client.enableSnatForDns {
 		return client.snatClient.DeleteSnatEndpoint()
 	}