[WIP] support for Hostport (#335)

* Support for hostport
- remove the hardcode cni conf file path
- move it to env variable
- keep it backward compatible even if env variable is not present
- support both .conf and .conflist
- daemonset with hostport support

Fixes: #168

* add unit test for InsertPodCidrInCniSpec for .conflist

* add documentation for hostport

Author: murali-reddy

Documentation/README.md

@@ -264,6 +264,41 @@ For destination hashing scheduling use:
 kubectl annotate service my-service "kube-router.io/service.scheduler=dh"
 ```
 
+### HostPort support
+
+If you would like to use `HostPort` functionality below changes are required in the manifest.
+
+- By default kube-router assumes CNI conf file to be `/etc/cni/net.d/10-kuberouter.conf`. Add an environment variable `KUBE_ROUTER_CNI_CONF_FILE` to kube-router manifest and set it to `/etc/cni/net.d/10-kuberouter.conflist`
+- Modify `kube-router-cfg` ConfigMap with CNI config that supports `portmap` as additional plug-in
+```
+    {
+       "cniVersion":"0.3.0",
+       "name":"mynet",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "capabilities":{
+                "snat":true,
+                "portMappings":true
+             }
+          }
+       ]
+    }
+```
+- Update init container command to create `/etc/cni/net.d/10-kuberouter.conflist` file
+- Restart the container runtime
+
+For an e.g manifest please look at [manifest](../daemonset/kubeadm-kuberouter-all-features-hostport.yaml) with necessary changes required for `HostPort` functionality.
+
 ## BGP configuration
 
 [Configuring BGP Peers](bgp.md)

app/controllers/network_routes_controller.go

@@ -68,6 +68,7 @@ type NetworkRoutingController struct {
 	bgpRRClient          bool
 	bgpRRServer          bool
 	bgpClusterId         uint32
+	cniConfFile          string
 }
 
 var (
@@ -89,7 +90,7 @@ const (
 
 // Run runs forever until we are notified on stop channel
 func (nrc *NetworkRoutingController) Run(healthChan chan<- *ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
-	cidr, err := utils.GetPodCidrFromCniSpec("/etc/cni/net.d/10-kuberouter.conf")
+	cidr, err := utils.GetPodCidrFromCniSpec(nrc.cniConfFile)
 	if err != nil {
 		glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err.Error())
 	}
@@ -102,7 +103,7 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *ControllerHeartbeat,
 	}
 
 	if len(cidr.IP) == 0 || strings.Compare(oldCidr, currentCidr) != 0 {
-		err = utils.InsertPodCidrInCniSpec("/etc/cni/net.d/10-kuberouter.conf", currentCidr)
+		err = utils.InsertPodCidrInCniSpec(nrc.cniConfFile, currentCidr)
 		if err != nil {
 			glog.Errorf("Failed to insert pod CIDR into CNI conf file: %s", err.Error())
 		}
@@ -1563,6 +1564,14 @@ func NewNetworkRoutingController(clientset *kubernetes.Clientset,
 	nrc.bgpRRServer = false
 	nrc.bgpServerStarted = false
 
+	nrc.cniConfFile = os.Getenv("KUBE_ROUTER_CNI_CONF_FILE")
+	if nrc.cniConfFile == "" {
+		nrc.cniConfFile = "/etc/cni/net.d/10-kuberouter.conf"
+	}
+	if _, err := os.Stat(nrc.cniConfFile); os.IsNotExist(err) {
+		return nil, errors.New("CNI conf file " + nrc.cniConfFile + " does not exist.")
+	}
+
 	nrc.ipSetHandler, err = utils.NewIPSet()
 	if err != nil {
 		return nil, err

daemonset/kubeadm-kuberouter-all-features-hostport.yaml

@@ -0,0 +1,184 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kube-router-cfg
+  namespace: kube-system
+  labels:
+    tier: node
+    k8s-app: kube-router
+data:
+  cni-conf.json: |
+    {
+       "cniVersion":"0.3.0",
+       "name":"mynet",
+       "plugins":[
+          {
+             "name":"kubernetes",
+             "type":"bridge",
+             "bridge":"kube-bridge",
+             "isDefaultGateway":true,
+             "ipam":{
+                "type":"host-local"
+             }
+          },
+          {
+             "type":"portmap",
+             "capabilities":{
+                "snat":true,
+                "portMappings":true
+             }
+          }
+       ]
+    }
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      serviceAccountName: kube-router
+      serviceAccount: kube-router
+      containers:
+      - name: kube-router
+        image: cloudnativelabs/kube-router
+        imagePullPolicy: Always
+        args:
+        - --run-router=true
+        - --run-firewall=true
+        - --run-service-proxy=true
+        - --kubeconfig=/var/lib/kube-router/kubeconfig
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: KUBE_ROUTER_CNI_CONF_FILE
+          value: /etc/cni/net.d/10-kuberouter.conflist
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: lib-modules
+          mountPath: /lib/modules
+          readOnly: true
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kubeconfig
+          mountPath: /var/lib/kube-router
+          readOnly: true
+      initContainers:
+      - name: install-cni
+        image: busybox
+        imagePullPolicy: Always
+        command:
+        - /bin/sh
+        - -c
+        - set -e -x;
+          if [ ! -f /etc/cni/net.d/10-kuberouter.conflist ]; then
+            TMP=/etc/cni/net.d/.tmp-kuberouter-cfg;
+            cp /etc/kube-router/cni-conf.json ${TMP};
+            mv ${TMP} /etc/cni/net.d/10-kuberouter.conflist;
+          fi
+        volumeMounts:
+        - name: cni-conf-dir
+          mountPath: /etc/cni/net.d
+        - name: kube-router-cfg
+          mountPath: /etc/kube-router
+      hostNetwork: true
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Exists
+      volumes:
+      - name: lib-modules
+        hostPath:
+          path: /lib/modules
+      - name: cni-conf-dir
+        hostPath:
+          path: /etc/cni/net.d
+      - name: kube-router-cfg
+        configMap:
+          name: kube-router-cfg
+      - name: kubeconfig
+        configMap:
+          name: kube-proxy
+          items:
+          - key: kubeconfig.conf
+            path: kubeconfig
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+  - apiGroups:
+    - ""
+    resources:
+      - namespaces
+      - pods
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - "networking.k8s.io"
+    resources:
+      - networkpolicies
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - extensions
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

utils/pod_cidr.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"net"
 	"reflect"
+	"strings"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containernetworking/cni/plugins/ipam/host-local/backend/allocator"
@@ -15,22 +16,39 @@ import (
 
 // GetPodCidrFromCniSpec gets pod CIDR allocated to the node from CNI spec file and returns it
 func GetPodCidrFromCniSpec(cniConfFilePath string) (net.IPNet, error) {
-	netconfig, err := libcni.ConfFromFile(cniConfFilePath)
-	if err != nil {
-		return net.IPNet{}, fmt.Errorf("Failed to load CNI conf file: %s", err.Error())
-	}
-
+	var podCidr net.IPNet
+	var err error
 	var ipamConfig *allocator.IPAMConfig
-	ipamConfig, _, err = allocator.LoadIPAMConfig(netconfig.Bytes, "")
-	if err != nil {
-		return net.IPNet{}, fmt.Errorf("Failed to get IPAM details from the CNI conf file: %s", err.Error())
-	}
 
-	podCidr := net.IPNet(ipamConfig.Subnet)
+	if strings.HasSuffix(cniConfFilePath, ".conflist") {
+		var confList *libcni.NetworkConfigList
+		confList, err = libcni.ConfListFromFile(cniConfFilePath)
+		if err != nil {
+			return net.IPNet{}, fmt.Errorf("Failed to load CNI config list file: %s", err.Error())
+		}
+		for _, conf := range confList.Plugins {
+			if conf.Network.IPAM.Type != "" {
+				ipamConfig, _, err = allocator.LoadIPAMConfig(conf.Bytes, "")
+				if err != nil {
+					return net.IPNet{}, fmt.Errorf("Failed to get IPAM details from the CNI conf file: %s", err.Error())
+				}
+				break
+			}
+		}
+	} else {
+		netconfig, err := libcni.ConfFromFile(cniConfFilePath)
+		if err != nil {
+			return net.IPNet{}, fmt.Errorf("Failed to load CNI conf file: %s", err.Error())
+		}
+		ipamConfig, _, err = allocator.LoadIPAMConfig(netconfig.Bytes, "")
+		if err != nil {
+			return net.IPNet{}, fmt.Errorf("Failed to get IPAM details from the CNI conf file: %s", err.Error())
+		}
+	}
+	podCidr = net.IPNet(ipamConfig.Subnet)
 	if reflect.DeepEqual(podCidr, net.IPNet{}) {
 		return net.IPNet{}, errors.New("subnet missing from CNI IPAM")
 	}
-
 	return podCidr, nil
 }
 
@@ -41,15 +59,46 @@ func InsertPodCidrInCniSpec(cniConfFilePath string, cidr string) error {
 	if err != nil {
 		return fmt.Errorf("Failed to load CNI conf file: %s", err.Error())
 	}
-	config := make(map[string]interface{})
-	err = json.Unmarshal(file, &config)
-	if err != nil {
-		return fmt.Errorf("Failed to parse JSON from CNI conf file: %s", err.Error())
-	}
+	var config interface{}
+	if strings.HasSuffix(cniConfFilePath, ".conflist") {
+		err = json.Unmarshal(file, &config)
+		if err != nil {
+			return fmt.Errorf("Failed to parse JSON from CNI conf file: %s", err.Error())
+		}
+		updatedCidr := false
+		configMap := config.(map[string]interface{})
+		for key := range configMap {
+			if key != "plugins" {
+				continue
+			}
+			// .conflist file has array of plug-in config. Find the one with ipam key
+			// and insert the CIDR for the node
+			pluginConfigs := configMap["plugins"].([]interface{})
+			for _, pluginConfig := range pluginConfigs {
+				pluginConfigMap := pluginConfig.(map[string]interface{})
+				if val, ok := pluginConfigMap["ipam"]; ok {
+					valObj := val.(map[string]interface{})
+					valObj["subnet"] = cidr
+					updatedCidr = true
+					break
+				}
+			}
+		}
 
-	config["ipam"].(map[string]interface{})["subnet"] = cidr
-	configJson, _ := json.Marshal(config)
-	err = ioutil.WriteFile(cniConfFilePath, configJson, 0644)
+		if !updatedCidr {
+			return fmt.Errorf("Failed to insert subnet cidr into CNI conf file: %s as CNI file is invalid.", cniConfFilePath)
+		}
+
+	} else {
+		err = json.Unmarshal(file, &config)
+		if err != nil {
+			return fmt.Errorf("Failed to parse JSON from CNI conf file: %s", err.Error())
+		}
+		pluginConfig := config.(map[string]interface{})
+		pluginConfig["ipam"].(map[string]interface{})["subnet"] = cidr
+	}
+	configJSON, _ := json.Marshal(config)
+	err = ioutil.WriteFile(cniConfFilePath, configJSON, 0644)
 	if err != nil {
 		return fmt.Errorf("Failed to insert subnet cidr into CNI conf file: %s", err.Error())
 	}