Introduces the option --override-nexthop, to override the next hop used in advertised routes (#502)

* Introduces the option --override-nexthop, setting it to true will make
advertised next hop for the routers to the peers will be automatically
selected to be appropriate reachable local IP. This will be overrider
any next-hop set for the routes in the RIB. Kube-router by defauly set
the next-hop to `node IP` which is not correct in case of nodes with
multiple interfaces and use differnt interaces for differect external
peers.

Fixes #480

* add next-hop-self documentation

Author: murali-reddy

docs/bgp.md

@@ -152,3 +152,7 @@ Here is sample example to make GoBGP server to listen on multiple IP address
 kubectl annotate node ip-172-20-46-87.us-west-2.compute.internal "kube-router.io/bgp-local-addresses=172.20.56.25,192.168.1.99"
 ```
 
+## Overriding the next hop
+
+By default kube-router populates GoBGP RIB with node IP as next hop for the advertised pod CIDR's and service VIP. While this works for most cases, overriding the next hop for the advertised rotues is necessary when node has multiple interfaces over which external peers are reached. Next hop need to be as per the interface local IP over which external peer can be reached. `--override-nexthop` let you override the next hop for the advertised route. Setting `--override-nexthop` to true leverages BGP next-hop-self functionality implemented in GoBGP. Next hop will automatically selected appropriately when advertising routes irrespective of the next hop in the RIB. 
+

docs/user-guide.md

@@ -53,6 +53,7 @@ Usage of kube-router:
       --metrics-port uint16              Prometheus metrics port, (Default 0, Disabled)
       --nodeport-bindon-all-ip           For service of NodePort type create IPVS service that listens on all IP's of the node.
       --nodes-full-mesh                  Each node in the cluster will setup BGP peering with rest of the nodes. (default true)
+      --override-nexthop                 Override the next-hop in bgp routes sent to peers with the local ip.
       --peer-router-asns uints           ASN numbers of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr. (default [])
       --peer-router-ips ipSlice          The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's. (default [])
       --peer-router-multihop-ttl uint8   Enable eBGP multihop supports -- sets multihop-ttl. (Relevant only if ttl >= 2)

pkg/controllers/routing/export_policies.go

@@ -23,6 +23,7 @@ import (
 //   BGP peers
 // - each node is NOT allowed to advertise service VIP's (cluster ip, load balancer ip, external IP) to
 //   iBGP peers
+// - an option to allow overriding the next-hop-address with the outgoing ip for external bgp peers
 func (nrc *NetworkRoutingController) addExportPolicies() error {
 
 	// we are rr server do not add export policies
@@ -96,6 +97,12 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 		if err != nil {
 			nrc.bgpServer.AddDefinedSet(iBGPPeerNS)
 		}
+		actions := config.Actions{
+			RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE,
+		}
+		if nrc.overrideNextHop {
+			actions.BgpActions.SetNextHop = "self"
+		}
 		// statement to represent the export policy to permit advertising node's pod CIDR
 		statements = append(statements,
 			config.Statement{
@@ -107,9 +114,7 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 						NeighborSet: "iBGPpeerset",
 					},
 				},
-				Actions: config.Actions{
-					RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE,
-				},
+				Actions: actions,
 			})
 	}
 
@@ -133,6 +138,9 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 		if err != nil {
 			nrc.bgpServer.AddDefinedSet(ns)
 		}
+		if nrc.overrideNextHop {
+			bgpActions.SetNextHop = "self"
+		}
 		// statement to represent the export policy to permit advertising cluster IP's
 		// only to the global BGP peer or node specific BGP peer
 		statements = append(statements, config.Statement{
@@ -150,6 +158,12 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 			},
 		})
 		if nrc.advertisePodCidr {
+			actions := config.Actions{
+				RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE,
+			}
+			if nrc.overrideNextHop {
+				actions.BgpActions.SetNextHop = "self"
+			}
 			statements = append(statements, config.Statement{
 				Conditions: config.Conditions{
 					MatchPrefixSet: config.MatchPrefixSet{
@@ -159,9 +173,7 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 						NeighborSet: "externalpeerset",
 					},
 				},
-				Actions: config.Actions{
-					RouteDisposition: config.ROUTE_DISPOSITION_ACCEPT_ROUTE,
-				},
+				Actions: actions,
 			})
 		}
 	}

pkg/controllers/routing/network_routes_controller.go

@@ -94,6 +94,7 @@ type NetworkRoutingController struct {
 	pathPrependCount        uint8
 	pathPrepend             bool
 	localAddressList        []string
+	overrideNextHop         bool
 
 	nodeLister cache.Indexer
 	svcLister  cache.Indexer
@@ -763,6 +764,7 @@ func NewNetworkRoutingController(clientset kubernetes.Interface,
 	nrc.peerMultihopTTL = kubeRouterConfig.PeerMultihopTtl
 	nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress
 	nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod
+	nrc.overrideNextHop = kubeRouterConfig.OverrideNextHop
 	nrc.clientset = clientset
 	nrc.activeNodes = make(map[string]bool)
 	nrc.bgpRRClient = false

pkg/options/options.go

@@ -36,6 +36,7 @@ type KubeRouterConfig struct {
 	MetricsPath             string
 	MetricsPort             uint16
 	NodePortBindOnAllIp     bool
+	OverrideNextHop         bool
 	PeerASNs                []uint
 	PeerMultihopTtl         uint8
 	PeerPasswords           []string
@@ -133,4 +134,5 @@ func (s *KubeRouterConfig) AddFlags(fs *pflag.FlagSet) {
 	// 	"Password that cluster-node BGP servers will use to authenticate one another when \"--nodes-full-mesh\" is set.")
 	fs.StringVarP(&s.VLevel, "v", "v", "0", "log level for V logs")
 	fs.Uint16Var(&s.HealthPort, "health-port", 20244, "Health check port, 0 = Disabled")
+	fs.BoolVar(&s.OverrideNextHop, "override-nexthop", false, "Override the next-hop in bgp routes sent to peers with the local ip.")
 }