Allow CNI plugin to be disabled (#443)

Author: andrewsykim

daemonset/generic-kuberouter-only-advertise-routes.yaml

@@ -0,0 +1,117 @@
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    k8s-app: kube-router
+    tier: node
+  name: kube-router
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-router
+        tier: node
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      serviceAccountName: kube-router
+      containers:
+      - name: kube-router
+        image: cloudnativelabs/kube-router
+        imagePullPolicy: Always
+        args:
+        - "--run-router=true"
+        - "--run-firewall=false"
+        - "--run-service-proxy=false"
+        - "--enable-cni=false"
+        - "--enable-ibgp=false"
+        - "--enable-overlay=false"
+        - "--peer-router-ips=<CHANGE ME>"
+        - "--peer-router-asns=<CHANGE ME>"
+        - "--cluster-asn=<CHANGE ME>"
+        - "--advertise-cluster-ip=true"
+        - "--advertise-external-ip=true"
+        - "--advertise-loadbalancer-ip=true"
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 20244
+          initialDelaySeconds: 10
+          periodSeconds: 3
+        resources:
+          requests:
+            cpu: 250m
+            memory: 250Mi
+        securityContext:
+          privileged: true
+      hostNetwork: true
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        key: node-role.kubernetes.io/master
+        operator: Exists
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-router
+  namespace: kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+  namespace: kube-system
+rules:
+  - apiGroups:
+    - ""
+    resources:
+      - namespaces
+      - pods
+      - services
+      - nodes
+      - endpoints
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - "networking.k8s.io"
+    resources:
+      - networkpolicies
+    verbs:
+      - list
+      - get
+      - watch
+  - apiGroups:
+    - extensions
+    resources:
+      - networkpolicies
+    verbs:
+      - get
+      - list
+      - watch
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kube-router
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: kube-router
+subjects:
+- kind: ServiceAccount
+  name: kube-router
+  namespace: kube-system

docs/user-guide.md

@@ -34,6 +34,7 @@ Usage of kube-router:
       --cleanup-config                   Cleanup iptables rules, ipvs, ipset configuration and exit.
       --cluster-asn uint                 ASN number under which cluster nodes will run iBGP.
       --cluster-cidr string              CIDR range of pods in the cluster. It is used to identify traffic originating from and destinated to pods.
+      --enable-cni                       Enable CNI plugin. Disable if you want to use kube-router features alongside another CNI plugin. (default true)
       --enable-ibgp                      Enables peering with nodes with the same ASN, if disabled will only peer with external BGP peers (default true)
       --enable-overlay                   When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
       --enable-pod-egress                SNAT traffic from Pods to destinations outside the cluster. (default true)

pkg/controllers/routing/network_routes_controller.go

@@ -72,6 +72,7 @@ type NetworkRoutingController struct {
 	nodeAsnNumber           uint32
 	globalPeerRouters       []*config.NeighborConfig
 	nodePeerRouters         []string
+	enableCNI               bool
 	bgpFullMeshMode         bool
 	bgpEnableInternal       bool
 	bgpGracefulRestart      bool
@@ -98,26 +99,9 @@ type NetworkRoutingController struct {
 
 // Run runs forever until we are notified on stop channel
 func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.ControllerHeartbeat, stopCh <-chan struct{}, wg *sync.WaitGroup) {
-	cidr, err := utils.GetPodCidrFromCniSpec(nrc.cniConfFile)
-	if err != nil {
-		glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err.Error())
-	}
-	if reflect.DeepEqual(cidr, net.IPNet{}) {
-		glog.Infof("`subnet` in CNI conf file is empty so populating `subnet` in CNI conf file with pod CIDR assigned to the node obtained from node spec.")
-	}
-	cidrlen, _ := cidr.Mask.Size()
-	oldCidr := cidr.IP.String() + "/" + strconv.Itoa(cidrlen)
-
-	currentCidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset, nrc.hostnameOverride)
-	if err != nil {
-		glog.Fatalf("Failed to get pod CIDR from node spec. kube-router relies on kube-controller-manager to allocate pod CIDR for the node. Error: %v", err.Error())
-	}
-
-	if len(cidr.IP) == 0 || strings.Compare(oldCidr, currentCidr) != 0 {
-		err = utils.InsertPodCidrInCniSpec(nrc.cniConfFile, currentCidr)
-		if err != nil {
-			glog.Fatalf("Failed to insert `subnet`(pod CIDR) into CNI conf file: %s", err.Error())
-		}
+	var err error
+	if nrc.enableCNI {
+		nrc.updateCNIConfig()
 	}
 
 	glog.V(1).Info("Populating ipsets.")
@@ -284,6 +268,31 @@ func (nrc *NetworkRoutingController) Run(healthChan chan<- *healthcheck.Controll
 	}
 }
 
+func (nrc *NetworkRoutingController) updateCNIConfig() {
+	cidr, err := utils.GetPodCidrFromCniSpec(nrc.cniConfFile)
+	if err != nil {
+		glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err)
+	}
+
+	if reflect.DeepEqual(cidr, net.IPNet{}) {
+		glog.Infof("`subnet` in CNI conf file is empty so populating `subnet` in CNI conf file with pod CIDR assigned to the node obtained from node spec.")
+	}
+	cidrlen, _ := cidr.Mask.Size()
+	oldCidr := cidr.IP.String() + "/" + strconv.Itoa(cidrlen)
+
+	currentCidr, err := utils.GetPodCidrFromNodeSpec(nrc.clientset, nrc.hostnameOverride)
+	if err != nil {
+		glog.Fatalf("Failed to get pod CIDR from node spec. kube-router relies on kube-controller-manager to allocate pod CIDR for the node or an annotation `kube-router.io/pod-cidr`. Error: %v", err)
+	}
+
+	if len(cidr.IP) == 0 || strings.Compare(oldCidr, currentCidr) != 0 {
+		err = utils.InsertPodCidrInCniSpec(nrc.cniConfFile, currentCidr)
+		if err != nil {
+			glog.Fatalf("Failed to insert `subnet`(pod CIDR) into CNI conf file: %s", err.Error())
+		}
+	}
+}
+
 func (nrc *NetworkRoutingController) watchBgpUpdates() {
 	watcher := nrc.bgpServer.Watch(gobgp.WatchBestPath(false))
 	for {
@@ -704,6 +713,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
 	}
 
 	nrc.bgpFullMeshMode = kubeRouterConfig.FullMeshMode
+	nrc.enableCNI = kubeRouterConfig.EnableCNI
 	nrc.bgpEnableInternal = kubeRouterConfig.EnableiBGP
 	nrc.bgpGracefulRestart = kubeRouterConfig.BGPGracefulRestart
 	nrc.peerMultihopTTL = kubeRouterConfig.PeerMultihopTtl
@@ -719,12 +729,14 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
 	// lets start with assumption we hace necessary IAM creds to access EC2 api
 	nrc.ec2IamAuthorized = true
 
-	nrc.cniConfFile = os.Getenv("KUBE_ROUTER_CNI_CONF_FILE")
-	if nrc.cniConfFile == "" {
-		nrc.cniConfFile = "/etc/cni/net.d/10-kuberouter.conf"
-	}
-	if _, err := os.Stat(nrc.cniConfFile); os.IsNotExist(err) {
-		return nil, errors.New("CNI conf file " + nrc.cniConfFile + " does not exist.")
+	if nrc.enableCNI {
+		nrc.cniConfFile = os.Getenv("KUBE_ROUTER_CNI_CONF_FILE")
+		if nrc.cniConfFile == "" {
+			nrc.cniConfFile = "/etc/cni/net.d/10-kuberouter.conf"
+		}
+		if _, err := os.Stat(nrc.cniConfFile); os.IsNotExist(err) {
+			return nil, errors.New("CNI conf file " + nrc.cniConfFile + " does not exist.")
+		}
 	}
 
 	nrc.ipSetHandler, err = utils.NewIPSet()

pkg/options/options.go

@@ -16,6 +16,7 @@ type KubeRouterConfig struct {
 	CleanupConfig           bool
 	ClusterAsn              uint
 	ClusterCIDR             string
+	EnableCNI               bool
 	EnableiBGP              bool
 	EnableOverlay           bool
 	EnablePodEgress         bool
@@ -105,6 +106,8 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
 		"Each node in the cluster will setup BGP peering with rest of the nodes.")
 	fs.BoolVar(&s.BGPGracefulRestart, "bgp-graceful-restart", false,
 		"Enables the BGP Graceful Restart capability so that routes are preserved on unexpected restarts")
+	fs.BoolVar(&s.EnableCNI, "enable-cni", true,
+		"Enable CNI plugin. Disable if you want to use kube-router features alongside another CNI plugin.")
 	fs.BoolVar(&s.EnableiBGP, "enable-ibgp", true,
 		"Enables peering with nodes with the same ASN, if disabled will only peer with external BGP peers")
 	fs.StringVar(&s.HostnameOverride, "hostname-override", s.HostnameOverride,