Introduces the option --overlay-type={subnet,full}, to be able to always generate IPIP tunnels regardless of node subnets (#666)

* Introduces the option --full-overlay, to always generate IPIP tunnels regardless of node subnets

* Use --overlay-type={subnet,full} instead of --full-overlay={true,false}

Author: lucasmundim

docs/user-guide.md

@@ -49,7 +49,7 @@ Usage of kube-router:
       --disable-source-dest-check        Disable the source-dest-check attribute for AWS EC2 instances. When this option is false, it must be set some other way. (default true)
       --enable-cni                       Enable CNI plugin. Disable if you want to use kube-router features alongside another CNI plugin. (default true)
       --enable-ibgp                      Enables peering with nodes with the same ASN, if disabled will only peer with external BGP peers (default true)
-      --enable-overlay                   When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
+      --enable-overlay                   When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets (default true)
       --enable-pod-egress                SNAT traffic from Pods to destinations outside the cluster. (default true)
       --enable-pprof                     Enables pprof for debugging performance and memory leak issues.
       --hairpin-mode                     Add iptables rules for every Service Endpoint to support hairpin traffic.
@@ -65,6 +65,7 @@ Usage of kube-router:
       --metrics-port uint16              Prometheus metrics port, (Default 0, Disabled)
       --nodeport-bindon-all-ip           For service of NodePort type create IPVS service that listens on all IP's of the node.
       --nodes-full-mesh                  Each node in the cluster will setup BGP peering with rest of the nodes. (default true)
+      --overlay-type string              Possible values: subnet,full - When set to "subnet", the default, default "--enable-overlay=true" behavior is used. When set to "full", it changes "--enable-overlay=true" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in. (default "subnet")
       --override-nexthop                 Override the next-hop in bgp routes sent to peers with the local ip.
       --peer-router-asns uints           ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default [])
       --peer-router-ips ipSlice          The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. (default [])

pkg/controllers/routing/network_routes_controller.go

@@ -91,6 +91,7 @@ type NetworkRoutingController struct {
 	bgpGracefulRestart      bool
 	ipSetHandler            *utils.IPSet
 	enableOverlays          bool
+	overlayType             string
 	peerMultihopTTL         uint8
 	MetricsEnabled          bool
 	bgpServerStarted        bool
@@ -410,50 +411,41 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error {
 	dst, _ := netlink.ParseIPNet(nlri.String())
 	var route *netlink.Route
 
-	// check if the neighbour is in same subnet. If node is not in same subnet and --override-nexthop=false
-	// only then create IPIP tunnels
-	if !nrc.nodeSubnet.Contains(nexthop) && !nrc.overrideNextHop {
-		tunnelName := generateTunnelName(nexthop.String())
-		glog.Infof("Found node: " + nexthop.String() + " to be in different subnet.")
-
-		// if overlay is not enabled then skip creating tunnels and adding route
-		if !nrc.enableOverlays {
-			glog.Infof("Found node: " + nexthop.String() + " to be in different subnet but overlays are " +
-				"disabled so not creating any tunnel and injecting route for the node's pod CIDR.")
-
-			glog.Infof("Cleaning up old routes if there are any")
-			routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{
-				Dst: dst, Protocol: 0x11,
-			}, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL)
-			if err != nil {
-				glog.Errorf("Failed to get routes from netlink")
-			}
-			for i, r := range routes {
-				glog.V(2).Infof("Found route to remove: %s", r.String())
-				err := netlink.RouteDel(&routes[i])
-				if err != nil {
-					glog.Errorf("Failed to remove route due to " + err.Error())
-				}
-			}
+	tunnelName := generateTunnelName(nexthop.String())
+	sameSubnet := nrc.nodeSubnet.Contains(nexthop)
 
-			glog.Infof("Cleaning up if there is any existing tunnel interface for the node")
-			link, err := netlink.LinkByName(tunnelName)
-			if err != nil {
-				return nil
+	// cleanup route and tunnel if overlay is disabled or node is in same subnet and overlay-type is set to 'subnet'
+	if !nrc.enableOverlays || (sameSubnet && nrc.overlayType == "subnet") {
+		glog.Infof("Cleaning up old routes if there are any")
+		routes, err := netlink.RouteListFiltered(nl.FAMILY_ALL, &netlink.Route{
+			Dst: dst, Protocol: 0x11,
+		}, netlink.RT_FILTER_DST|netlink.RT_FILTER_PROTOCOL)
+		if err != nil {
+			glog.Errorf("Failed to get routes from netlink")
+		}
+		for i, r := range routes {
+			glog.V(2).Infof("Found route to remove: %s", r.String())
+			if err := netlink.RouteDel(&routes[i]); err != nil {
+				glog.Errorf("Failed to remove route due to " + err.Error())
 			}
-			err = netlink.LinkDel(link)
-			if err != nil {
+		}
+
+		glog.Infof("Cleaning up if there is any existing tunnel interface for the node")
+		if link, err := netlink.LinkByName(tunnelName); err == nil {
+			if err = netlink.LinkDel(link); err != nil {
 				glog.Errorf("Failed to delete tunnel link for the node due to " + err.Error())
 			}
-			return nil
 		}
+	}
 
+	// create IPIP tunnels only when node is not in same subnet or overlay-type is set to 'full'
+	// prevent creation when --override-nexthop=true as well
+	if (!sameSubnet || nrc.overlayType == "full") && !nrc.overrideNextHop {
 		// create ip-in-ip tunnel and inject route as overlay is enabled
 		var link netlink.Link
 		var err error
 		link, err = netlink.LinkByName(tunnelName)
 		if err != nil {
-			glog.Infof("Found node: " + nexthop.String() + " to be in different subnet. Creating tunnel: " + tunnelName)
 			out, err := exec.Command("ip", "tunnel", "add", tunnelName, "mode", "ipip", "local", nrc.nodeIP.String(),
 				"remote", nexthop.String(), "dev", nrc.nodeInterface).CombinedOutput()
 			if err != nil {
@@ -938,6 +930,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
 	nrc.advertiseLoadBalancerIP = kubeRouterConfig.AdvertiseLoadBalancerIp
 	nrc.advertisePodCidr = kubeRouterConfig.AdvertiseNodePodCidr
 	nrc.enableOverlays = kubeRouterConfig.EnableOverlay
+	nrc.overlayType = kubeRouterConfig.OverlayType
 
 	nrc.bgpPort = kubeRouterConfig.BGPPort
 

pkg/options/options.go

@@ -4,8 +4,9 @@ import (
 	"net"
 	"time"
 
-	"github.com/spf13/pflag"
 	"strconv"
+
+	"github.com/spf13/pflag"
 )
 
 const DEFAULT_BGP_PORT = 179
@@ -28,6 +29,7 @@ type KubeRouterConfig struct {
 	EnablePodEgress         bool
 	EnablePprof             bool
 	FullMeshMode            bool
+	OverlayType             string
 	GlobalHairpinMode       bool
 	HealthPort              uint16
 	HelpRequested           bool
@@ -64,6 +66,7 @@ func NewKubeRouterConfig() *KubeRouterConfig {
 		IPTablesSyncPeriod: 5 * time.Minute,
 		RoutesSyncPeriod:   5 * time.Minute,
 		EnableOverlay:      true,
+		OverlayType:        "subnet",
 	}
 }
 
@@ -134,8 +137,12 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.NodePortBindOnAllIp, "nodeport-bindon-all-ip", false,
 		"For service of NodePort type create IPVS service that listens on all IP's of the node.")
 	fs.BoolVar(&s.EnableOverlay, "enable-overlay", true,
-		"When enable-overlay set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+
-			"When set to false no tunneling is used and routing infrastrcture is expected to route traffic for pod-to-pod networking across nodes in different subnets")
+		"When enable-overlay is set to true, IP-in-IP tunneling is used for pod-to-pod networking across nodes in different subnets. "+
+			"When set to false no tunneling is used and routing infrastructure is expected to route traffic for pod-to-pod networking across nodes in different subnets")
+	fs.StringVar(&s.OverlayType, "overlay-type", s.OverlayType,
+		"Possible values: subnet,full - "+
+			"When set to \"subnet\", the default, default \"--enable-overlay=true\" behavior is used. "+
+			"When set to \"full\", it changes \"--enable-overlay=true\" default behavior so that IP-in-IP tunneling is used for pod-to-pod networking across nodes regardless of the subnet the nodes are in.")
 	fs.StringSliceVar(&s.PeerPasswords, "peer-router-passwords", s.PeerPasswords,
 		"Password for authenticating against the BGP peer defined with \"--peer-router-ips\".")
 	fs.BoolVar(&s.EnablePprof, "enable-pprof", false,