Support k8s DNS & endpoint policy (#127)

* configure dnssuffix & dnsServerList

* add dns info to conflist

* change case for dns info

* remove hardcoding

* remove hardcoding

* parse k8s pod info

* add AdditionalArgs and Dns info to conflist

* serialize policies

* program route info

* start addressign ipam ip leak

* fix 1) ip inconsistency in k8s & pod. 2) IP leak in IPAM.

* remove comments

* separate windows & linux

* remove dnsServers

* remove comments

* address comments

* add dns verification

* abstract linux policy struct

* remove setPolicies

Author: Yongli Chen

cni/azure-linux.conflist

@@ -1,18 +1,21 @@
 {
-  "cniVersion": "0.3.0",
-  "name": "azure",
-  "plugins": [{
-                "type": "azure-vnet",
-                "mode": "bridge",
-                "bridge": "azure0",
-                "ipam": {
-                  "type": "azure-vnet-ipam"
-                }
-            },
-            {
-                "type": "portmap",
-                "capabilities": {"portMappings": true},
-                "snat": true
-            }
-  ]
-}
+   "cniVersion":"0.3.0",
+   "name":"azure",
+   "plugins":[
+      {
+         "type":"azure-vnet",
+         "mode":"bridge",
+         "bridge":"azure0",
+         "ipam":{
+            "type":"azure-vnet-ipam"
+         }
+      },
+      {
+         "type":"portmap",
+         "capabilities":{
+            "portMappings":true
+         },
+         "snat":true
+      }
+   ]
+}

cni/azure-windows.conflist

@@ -1,13 +1,43 @@
 {
-  "cniVersion": "0.3.0",
-  "name": "azure",
-  "plugins": [{
-                "type": "azure-vnet",
-                "mode": "bridge",
-                "bridge": "azure0",
-                "ipam": {
-                  "type": "azure-vnet-ipam"
-                }
+   "cniVersion":"0.3.0",
+   "name":"azure",
+   "plugins":[
+      {
+         "type":"azure-vnet",
+         "mode":"bridge",
+         "bridge":"azure0",
+         "ipam":{
+            "type":"azure-vnet-ipam"
+         },
+         "dns":{
+            "Nameservers":[
+               "168.63.129.16",
+               "10.0.0.10"
+            ],
+            "Search":[
+               "svc.cluster.local"
+            ]
+         },
+         "AdditionalArgs":[
+            {
+               "Name":"EndpointPolicy",
+               "Value":{
+                  "Type":"OutBoundNAT",
+                  "ExceptionList":[
+                     "10.240.0.0/16",
+                     "10.0.0.0/8"
+                  ]
+               }
+            },
+            {
+               "Name":"EndpointPolicy",
+               "Value":{
+                  "Type":"ROUTE",
+                  "DestinationPrefix":"10.0.0.0/8",
+                  "NeedEncap":true
+               }
             }
-  ]
+         ]
+      }
+   ]
 }

cni/netconfig.go

@@ -5,8 +5,23 @@ package cni
 
 import (
 	"encoding/json"
+	"strings"
+
+	"github.com/Azure/azure-container-networking/network/policy"
+
+	cniTypes "github.com/containernetworking/cni/pkg/types"
 )
 
+const (
+	PolicyStr string = "Policy"
+)
+
+// KVPair represents a K-V pair of a json object.
+type KVPair struct {
+	Name  string          `json:"name"`
+	Value json.RawMessage `json:"value"`
+}
+
 // NetworkConfig represents Azure CNI plugin network configuration.
 type NetworkConfig struct {
 	CNIVersion string `json:"cniVersion"`
@@ -25,6 +40,26 @@ type NetworkConfig struct {
 		Address       string `json:"ipAddress,omitempty"`
 		QueryInterval string `json:"queryInterval,omitempty"`
 	}
+	DNS            cniTypes.DNS `json:"dns"`
+	AdditionalArgs []KVPair
+}
+
+type K8SPodEnvArgs struct {
+	cniTypes.CommonArgs
+	K8S_POD_NAMESPACE          cniTypes.UnmarshallableString `json:"K8S_POD_NAMESPACE,omitempty"`
+	K8S_POD_NAME               cniTypes.UnmarshallableString `json:"K8S_POD_NAME,omitempty"`
+	K8S_POD_INFRA_CONTAINER_ID cniTypes.UnmarshallableString `json:"K8S_POD_INFRA_CONTAINER_ID,omitempty"`
+}
+
+// ParseCniArgs unmarshals cni arguments.
+func ParseCniArgs(args string) (*K8SPodEnvArgs, error) {
+	podCfg := K8SPodEnvArgs{}
+	err := cniTypes.LoadArgs(args, &podCfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &podCfg, nil
 }
 
 // ParseNetworkConfig unmarshals network configuration from bytes.
@@ -43,6 +78,22 @@ func ParseNetworkConfig(b []byte) (*NetworkConfig, error) {
 	return &nwCfg, nil
 }
 
+// GetPoliciesFromNwCfg returns network policies from network config.
+func GetPoliciesFromNwCfg(kvp []KVPair) []policy.Policy {
+	var policies []policy.Policy
+	for _, pair := range kvp {
+		if strings.Contains(pair.Name, PolicyStr) {
+			policy := policy.Policy{
+				Type: policy.CNIPolicyType(pair.Name),
+				Data: pair.Value,
+			}
+			policies = append(policies, policy)
+		}
+	}
+
+	return policies
+}
+
 // Serialize marshals a network configuration to bytes.
 func (nwcfg *NetworkConfig) Serialize() []byte {
 	bytes, _ := json.Marshal(nwcfg)

cni/network/network.go

@@ -5,6 +5,7 @@ package network
 
 import (
 	"net"
+	"strings"
 
 	"github.com/Azure/azure-container-networking/cni"
 	"github.com/Azure/azure-container-networking/common"
@@ -123,16 +124,46 @@ func (plugin *netPlugin) findMasterInterface(nwCfg *cni.NetworkConfig, subnetPre
 
 // Add handles CNI add commands.
 func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
-	var result *cniTypesCurr.Result
-	var err error
+	var (
+		result   *cniTypesCurr.Result
+		err      error
+		nwCfg    *cni.NetworkConfig
+		ipconfig *cniTypesCurr.IPConfig
+		epInfo   *network.EndpointInfo
+		iface    *cniTypesCurr.Interface
+	)
 
 	log.Printf("[cni-net] Processing ADD command with args {ContainerID:%v Netns:%v IfName:%v Args:%v Path:%v}.",
 		args.ContainerID, args.Netns, args.IfName, args.Args, args.Path)
 
-	defer func() { log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err) }()
+	defer func() {
+		// Add Interfaces to result.
+		iface = &cniTypesCurr.Interface{
+			Name: args.IfName,
+		}
+		result.Interfaces = append(result.Interfaces, iface)
+
+		// Convert result to the requested CNI version.
+		res, err := result.GetAsVersion(nwCfg.CNIVersion)
+		if err != nil {
+			err = plugin.Error(err)
+		}
+
+		// Output the result to stdout.
+		res.Print()
+		log.Printf("[cni-net] ADD command completed with result:%+v err:%v.", result, err)
+	}()
+
+	// Parse Pod arguments.
+	podCfg, err := cni.ParseCniArgs(args.Args)
+	k8sNamespace := string(podCfg.K8S_POD_NAMESPACE)
+	if len(k8sNamespace) == 0 {
+		err = plugin.Errorf("No k8s pod namespace provided.")
+		return err
+	}
 
 	// Parse network configuration from stdin.
-	nwCfg, err := cni.ParseNetworkConfig(args.StdinData)
+	nwCfg, err = cni.ParseNetworkConfig(args.StdinData)
 	if err != nil {
 		err = plugin.Errorf("Failed to parse network configuration: %v.", err)
 		return err
@@ -142,11 +173,31 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 
 	// Initialize values from network config.
 	networkId := nwCfg.Name
-	endpointId := plugin.GetEndpointID(args)
+	endpointId := network.GetEndpointID(args)
+
+	nwInfo, nwInfoErr := plugin.nm.GetNetworkInfo(networkId)
+
+	/* Handle consecutive ADD calls for infrastructure containers.
+	 * This is a temporary work around for issue #57253 of Kubernetes.
+	 * We can delete this if statement once they fix it.
+	 * Issue link: https://github.com/kubernetes/kubernetes/issues/57253
+	 */
+	epInfo, _ = plugin.nm.GetEndpointInfo(networkId, endpointId)
+	if epInfo != nil {
+		result, err = handleConsecutiveAdd(args.ContainerID, endpointId, nwInfo, nwCfg)
+		if err != nil {
+			return err
+		}
+
+		if result != nil {
+			return nil
+		}
+	}
+
+	policies := cni.GetPoliciesFromNwCfg(nwCfg.AdditionalArgs)
 
 	// Check whether the network already exists.
-	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)
-	if err != nil {
+	if nwInfoErr != nil {
 		// Network does not exist.
 		log.Printf("[cni-net] Creating network %v.", networkId)
 
@@ -158,7 +209,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		}
 
 		// Derive the subnet prefix from allocated IP address.
-		ipconfig := result.IPs[0]
+		ipconfig = result.IPs[0]
 		subnetPrefix := ipconfig.Address
 		subnetPrefix.IP = subnetPrefix.IP.Mask(subnetPrefix.Mask)
 
@@ -201,6 +252,11 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 				},
 			},
 			BridgeName: nwCfg.Bridge,
+			DNS: network.DNSInfo{
+				Servers: nwCfg.DNS.Nameservers,
+				Suffix:  strings.Join(nwCfg.DNS.Search, ","),
+			},
+			Policies: policies,
 		}
 
 		err = plugin.nm.CreateNetwork(&nwInfo)
@@ -223,7 +279,7 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 			return err
 		}
 
-		ipconfig := result.IPs[0]
+		ipconfig = result.IPs[0]
 
 		// On failure, call into IPAM plugin to release the address.
 		defer func() {
@@ -235,11 +291,31 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 	}
 
 	// Initialize endpoint info.
-	epInfo := &network.EndpointInfo{
+	var dns network.DNSInfo
+	if (len(nwCfg.DNS.Search) == 0) != (len(nwCfg.DNS.Nameservers) == 0) {
+		err = plugin.Errorf("Wrong DNS configuration: %+v", nwCfg.DNS)
+		return err
+	}
+
+	if len(nwCfg.DNS.Search) > 0 {
+		dns = network.DNSInfo{
+			Servers: nwCfg.DNS.Nameservers,
+			Suffix:  strings.Join(nwCfg.DNS.Search, ","),
+		}
+	} else {
+		dns = network.DNSInfo{
+			Suffix:  result.DNS.Domain,
+			Servers: result.DNS.Nameservers,
+		}
+	}
+
+	epInfo = &network.EndpointInfo{
 		Id:          endpointId,
 		ContainerID: args.ContainerID,
 		NetNsPath:   args.Netns,
 		IfName:      args.IfName,
+		DNS:         dns,
+		Policies:    policies,
 	}
 
 	// Populate addresses.
@@ -252,10 +328,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		epInfo.Routes = append(epInfo.Routes, network.RouteInfo{Dst: route.Dst, Gw: route.GW})
 	}
 
-	// Populate DNS info.
-	epInfo.DNS.Suffix = result.DNS.Domain
-	epInfo.DNS.Servers = result.DNS.Nameservers
-
 	// Create the endpoint.
 	log.Printf("[cni-net] Creating endpoint %v.", epInfo.Id)
 	err = plugin.nm.CreateEndpoint(networkId, epInfo)
@@ -264,22 +336,6 @@ func (plugin *netPlugin) Add(args *cniSkel.CmdArgs) error {
 		return err
 	}
 
-	// Add Interfaces to result.
-	iface := &cniTypesCurr.Interface{
-		Name: epInfo.IfName,
-	}
-	result.Interfaces = append(result.Interfaces, iface)
-
-	// Convert result to the requested CNI version.
-	res, err := result.GetAsVersion(nwCfg.CNIVersion)
-	if err != nil {
-		err = plugin.Error(err)
-		return err
-	}
-
-	// Output the result to stdout.
-	res.Print()
-
 	return nil
 }
 
@@ -303,7 +359,7 @@ func (plugin *netPlugin) Delete(args *cniSkel.CmdArgs) error {
 
 	// Initialize values from network config.
 	networkId := nwCfg.Name
-	endpointId := plugin.GetEndpointID(args)
+	endpointId := network.GetEndpointID(args)
 
 	// Query the network.
 	nwInfo, err := plugin.nm.GetNetworkInfo(networkId)

cni/network/network_linux.go

@@ -0,0 +1,10 @@
+package network
+
+import (
+	cniTypesCurr "github.com/containernetworking/cni/pkg/types/current"
+)
+
+// handleConsecutiveAdd is a dummy function for Linux platform.
+func handleConsecutiveAdd(containerId, endpointId string, nwInfo *NetworkInfo, nwCfg *NetworkConfig) (*cniTypesCurr.Result, error) {
+	return nil, nil
+}