Enable pod egress masquerading by default (#111)

* Enable pod egress masquerading by default
- Adds flag "--enable-pod-egress" (default: true)
- Removes previously created iptables rule if option is changed to false
* Use an ipset to match Pod egress traffic to be masqueraded
* Set --cluster-cidr as depreciated flag
If set to anything, normal dynamic Pod egress masquerading is turned on.
* Use Replace else Add logic for updating export policy
Fixes errors logged due to existing statement in poliy.

Author: bzub

Documentation/README.md

@@ -69,25 +69,27 @@ Also you can choose to run kube-router as agent running on each cluster node. Al
 ### command line options
 
 ```
-  --run-firewall                    If false, kube-router won't setup iptables to provide ingress firewall for pods. true by default.
-  --run-router                      If true each node advertise routes the rest of the nodes and learn the routes for the pods. false by default
-  --run-service-proxy               If false, kube-router won't setup IPVS for services proxy. true by default.
-  --cleanup-config                  If true cleanup iptables rules, ipvs, ipset configuration and exit.
-  --masquerade-all                  SNAT all traffic to cluster IP/node port. False by default
-  --cluster-cidr                    CIDR range of pods in the cluster. If specified external traffic from the pods will be masquraded
-  --config-sync-period duration     How often configuration from the apiserver is refreshed. Must be greater than 0. (default 1m0s)
-  --iptables-sync-period duration   The maximum interval of how often iptables rules are refreshed (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s)
-  --ipvs-sync-period duration       The maximum interval of how often ipvs config is refreshed (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
-  --kubeconfig string               Path to kubeconfig file with authorization information (the master location is set by the master flag).
-  --master string                   The address of the Kubernetes API server (overrides any value in kubeconfig)
-  --routes-sync-period duration     The maximum interval of how often routes are advertised and learned (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
-  --advertise-cluster-ip            If true then cluster IP will be added into the RIB and will be advertised to the peers. False by default.
-  --cluster-asn                     ASN number under which cluster nodes will run iBGP
-  --peer-asn                        ASN number of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr
-  --peer-router                     The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's
-  --nodes-full-mesh                 When enabled each node in the cluster will setup BGP peer with rest of the nodes. True by default
-  --hostname-override               If non-empty, this string will be used as identification of node name instead of the actual hostname.
-  --hairpin-mode                    Adds iptable rules for every ClusterIP Service Endpoint to support hairpin traffic. False by default
+Usage of ./kube-router:
+    --advertise-cluster-ip            Add Cluster IP to the RIB and advertise to peers.
+    --cleanup-config                  Cleanup iptables rules, ipvs, ipset configuration and exit.
+    --cluster-asn string              ASN number under which cluster nodes will run iBGP.
+    --config-sync-period duration     The delay between apiserver configuration synchronizations (e.g. '5s', '1m').  Must be greater than 0. (default 1m0s)
+    --enable-pod-egress               SNAT traffic from Pods to destinations outside the cluster. (default true)
+    --hairpin-mode                    Add iptable rules for every Service Endpoint to support hairpin traffic.
+-h, --help                            Print usage information.
+    --hostname-override string        Overrides the NodeName of the node. Set this if kube-router is unable to determine your NodeName automatically.
+    --iptables-sync-period duration   The delay between iptables rule synchronizations (e.g. '5s', '1m'). Must be greater than 0. (default 1m0s)
+    --ipvs-sync-period duration       The delay between ipvs config synchronizations (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
+    --kubeconfig string               Path to kubeconfig file with authorization information (the master location is set by the master flag).
+    --masquerade-all                  SNAT all traffic to cluster IP/node port.
+    --master string                   The address of the Kubernetes API server (overrides any value in kubeconfig).
+    --nodes-full-mesh                 Each node in the cluster will setup BGP peering with rest of the nodes. (default true)
+    --peer-asn string                 ASN number of the BGP peer to which cluster nodes will advertise cluster ip and node's pod cidr.
+    --peer-router string              The ip address of the external router to which all nodes will peer and advertise the cluster ip and pod cidr's.
+    --routes-sync-period duration     The delay between route updates and advertisements (e.g. '5s', '1m', '2h22m'). Must be greater than 0. (default 1m0s)
+    --run-firewall                    Enables Network Policy -- sets up iptables to provide ingress firewall for pods. (default true)
+    --run-router                      Enables Pod Networking -- Advertises and learns the routes to Pods via iBGP. (default true)
+    --run-service-proxy               Enables Service Proxy -- sets up IPVS for Kubernetes Services. (default true)
 ```
 
 ### requirements

app/controllers/network_routes_controller.go

@@ -20,6 +20,7 @@ import (
 	"github.com/cloudnativelabs/kube-router/utils"
 	"github.com/coreos/go-iptables/iptables"
 	"github.com/golang/glog"
+	"github.com/janeczku/go-ipset/ipset"
 	bgpapi "github.com/osrg/gobgp/api"
 	"github.com/osrg/gobgp/config"
 	"github.com/osrg/gobgp/packet/bgp"
@@ -40,6 +41,7 @@ type NetworkRoutingController struct {
 	bgpServer            *gobgp.BgpServer
 	syncPeriod           time.Duration
 	clusterCIDR          string
+	enablePodEgress      bool
 	hostnameOverride     string
 	advertiseClusterIp   bool
 	defaultNodeAsnNumber uint32
@@ -48,18 +50,22 @@ type NetworkRoutingController struct {
 	nodePeerRouters      []string
 	globalPeerAsnNumber  uint32
 	bgpFullMeshMode      bool
+	podSubnetsIpSet      *ipset.IPSet
 }
 
 var (
-	activeNodes = make(map[string]bool)
+	activeNodes   = make(map[string]bool)
+	podEgressArgs = []string{"-m", "set", "--match-set", podSubnetIpSetName, "src",
+		"-m", "set", "!", "--match-set", podSubnetIpSetName, "dst",
+		"-j", "MASQUERADE"}
 )
 
 const (
 	clustetNieghboursSet = "clusterneighboursset"
+	podSubnetIpSetName   = "kube-router-pod-subnets"
 )
 
 func (nrc *NetworkRoutingController) Run(stopCh <-chan struct{}, wg *sync.WaitGroup) {
-
 	cidr, err := utils.GetPodCidrFromCniSpec("/etc/cni/net.d/10-kuberouter.conf")
 	if err != nil {
 		glog.Errorf("Failed to get pod CIDR from CNI conf file: %s", err.Error())
@@ -88,15 +94,24 @@ func (nrc *NetworkRoutingController) Run(stopCh <-chan struct{}, wg *sync.WaitGr
 
 	glog.Infof("Starting network route controller")
 
-	if len(nrc.clusterCIDR) != 0 {
-		args := []string{"-s", nrc.clusterCIDR, "!", "-d", nrc.clusterCIDR, "-j", "MASQUERADE"}
-		iptablesCmdHandler, err := iptables.New()
+	// Handle Pod egress masquerading configuration
+	if nrc.enablePodEgress {
+		glog.Infoln("Enabling Pod egress.")
+		err = createPodEgressRule()
 		if err != nil {
-			glog.Errorf("Failed to add iptable rule to masqurade outbound traffic from pods due to %s. External connectivity will not work.", err.Error())
+			glog.Errorf("Error enabling Pod egress: %s", err.Error())
 		}
-		err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", args...)
+	} else {
+		glog.Infoln("Disabling Pod egress.")
+		err = deletePodEgressRule()
+		// TODO: Don't error if removing non-existant Pod egress rules/ipsets.
+		if err != nil {
+			glog.Infof("Error disabling Pod egress: %s", err.Error())
+		}
+
+		err = deletePodSubnetIpSet()
 		if err != nil {
-			glog.Errorf("Failed to add iptable rule to masqurade outbound traffic from pods due to %s. External connectivity will not work.", err.Error())
+			glog.Infof("Error disabling Pod egress: %s", err.Error())
 		}
 	}
 
@@ -127,6 +142,14 @@ func (nrc *NetworkRoutingController) Run(stopCh <-chan struct{}, wg *sync.WaitGr
 		default:
 		}
 
+		// Update Pod subnet ipset entries
+		if nrc.enablePodEgress {
+			err := nrc.syncPodSubnetIpSet()
+			if err != nil {
+				glog.Errorf("Error synchronizing Pod subnet ipset: %s", err.Error())
+			}
+		}
+
 		// add the current set of nodes (excluding self) as BGP peers. Nodes form full mesh
 		nrc.syncPeers()
 
@@ -148,14 +171,14 @@ func (nrc *NetworkRoutingController) Run(stopCh <-chan struct{}, wg *sync.WaitGr
 		}
 
 		glog.Infof("Performing periodic syn of the routes")
-		err := nrc.advertiseRoute()
+		err = nrc.advertiseRoute()
 		if err != nil {
-			glog.Errorf("Failed to advertise route: %s", err.Error())
+			glog.Errorf("Error advertising route: %s", err.Error())
 		}
 
 		err = nrc.addExportPolicies()
 		if err != nil {
-			glog.Errorf("Failed to add BGP export policies due to %s", err.Error())
+			glog.Errorf("Error adding BGP export policies: %s", err.Error())
 		}
 
 		select {
@@ -167,6 +190,43 @@ func (nrc *NetworkRoutingController) Run(stopCh <-chan struct{}, wg *sync.WaitGr
 	}
 }
 
+func createPodEgressRule() error {
+	iptablesCmdHandler, err := iptables.New()
+	if err != nil {
+		return errors.New("Failed create iptables handler:" + err.Error())
+	}
+
+	err = iptablesCmdHandler.AppendUnique("nat", "POSTROUTING", podEgressArgs...)
+	if err != nil {
+		return errors.New("Failed to add iptable rule to masqurade outbound traffic from pods: " +
+			err.Error() + "External connectivity will not work.")
+
+	}
+	glog.Infof("Added iptables rule to masqurade outbound traffic from pods.")
+	return nil
+}
+
+func deletePodEgressRule() error {
+	iptablesCmdHandler, err := iptables.New()
+	if err != nil {
+		return errors.New("Failed create iptables handler:" + err.Error())
+	}
+
+	exists, err := iptablesCmdHandler.Exists("nat", "POSTROUTING", podEgressArgs...)
+	if err != nil {
+		return errors.New("Failed to lookup iptable rule to masqurade outbound traffic from pods: " + err.Error())
+	}
+	if exists {
+		err = iptablesCmdHandler.Delete("nat", "POSTROUTING", podEgressArgs...)
+		if err != nil {
+			return errors.New("Failed to delete iptable rule to masqurade outbound traffic from pods: " +
+				err.Error() + ". Pod egress might still work...")
+		}
+		glog.Infof("Deleted iptables rule to masqurade outbound traffic from pods.")
+	}
+	return nil
+}
+
 func (nrc *NetworkRoutingController) watchBgpUpdates() {
 	watcher := nrc.bgpServer.Watch(gobgp.WatchBestPath(false))
 	for {
@@ -339,23 +399,32 @@ func (nrc *NetworkRoutingController) addExportPolicies() error {
 
 	policy, err := table.NewPolicy(definition)
 	if err != nil {
-		return err
+		return errors.New("Failed to create new policy: " + err.Error())
 	}
-	if err = nrc.bgpServer.AddPolicy(policy, false); err != nil {
-		return err
+
+	err = nrc.bgpServer.ReplacePolicy(policy, false, false)
+	if err != nil {
+		err = nrc.bgpServer.AddPolicy(policy, false)
+		if err != nil {
+			return errors.New("Failed to add policy: " + err.Error())
+		}
 	}
+
 	err = nrc.bgpServer.AddPolicyAssignment("",
 		table.POLICY_DIRECTION_EXPORT,
 		[]*config.PolicyDefinition{&definition},
 		table.ROUTE_TYPE_ACCEPT)
 	if err != nil {
-		return err
+		return errors.New("Failed to add policy assignment: " + err.Error())
 	}
 
 	// configure default BGP export policy to reject
 	pd := make([]*config.PolicyDefinition, 0)
 	pd = append(pd, &definition)
-	nrc.bgpServer.ReplacePolicyAssignment("", table.POLICY_DIRECTION_EXPORT, pd, table.ROUTE_TYPE_REJECT)
+	err = nrc.bgpServer.ReplacePolicyAssignment("", table.POLICY_DIRECTION_EXPORT, pd, table.ROUTE_TYPE_REJECT)
+	if err != nil {
+		return errors.New("Failed to replace policy assignment: " + err.Error())
+	}
 
 	return nil
 }
@@ -419,10 +488,33 @@ func (nrc *NetworkRoutingController) injectRoute(path *table.Path) error {
 }
 
 func (nrc *NetworkRoutingController) Cleanup() {
+	err := deletePodEgressRule()
+	if err != nil {
+		glog.Errorf("Error deleting Pod egress iptable rule: %s", err.Error())
+	}
+
+	err = deletePodSubnetIpSet()
+	if err != nil {
+		glog.Errorf("Error deleting Pod subnet ipset: %s", err.Error())
+	}
 }
 
-func (nrc *NetworkRoutingController) disableSourceDestinationCheck() {
+func deletePodSubnetIpSet() error {
+	_, err := exec.LookPath("ipset")
+	if err != nil {
+		return errors.New("Ensure ipset package is installed: " + err.Error())
+	}
+
+	podSubnetIpSet := ipset.IPSet{Name: podSubnetIpSetName, HashType: "bitmap:ip"}
+	err = podSubnetIpSet.Destroy()
+	if err != nil {
+		return errors.New("Failure deleting Pod egress ipset: " + err.Error())
+	}
 
+	return nil
+}
+
+func (nrc *NetworkRoutingController) disableSourceDestinationCheck() {
 	nodes, err := nrc.clientset.Core().Nodes().List(metav1.ListOptions{})
 	if err != nil {
 		glog.Errorf("Failed to list nodes from API server due to: %s. Can not perform BGP peer sync", err.Error())
@@ -463,10 +555,33 @@ func (nrc *NetworkRoutingController) disableSourceDestinationCheck() {
 	}
 }
 
+func (nrc *NetworkRoutingController) syncPodSubnetIpSet() error {
+	glog.Infof("Syncing Pod subnet ipset entries.")
+
+	// get the current list of the nodes from API server
+	nodes, err := nrc.clientset.Core().Nodes().List(metav1.ListOptions{})
+	if err != nil {
+		return errors.New("Failed to list nodes from API server: " + err.Error())
+	}
+
+	// Collect active PodCIDR(s) from nodes
+	currentPodCidrs := make([]string, 0)
+	for _, node := range nodes.Items {
+		currentPodCidrs = append(currentPodCidrs, node.Spec.PodCIDR)
+	}
+
+	err = nrc.podSubnetsIpSet.Refresh(currentPodCidrs)
+	if err != nil {
+		return errors.New("Failed to update Pod subnet ipset: " + err.Error())
+	}
+
+	return nil
+}
+
 // Refresh the peer relationship rest of the nodes in the cluster. Node add/remove
 // events should ensure peer relationship with only currently active nodes. In case
 // we miss any events from API server this method which is called periodically
-// ensure peer relationship with removed nodes is deleted.
+// ensure peer relationship with removed nodes is deleted. Also update Pod subnet ipset.
 func (nrc *NetworkRoutingController) syncPeers() {
 
 	glog.Infof("Syncing BGP peers for the node.")
@@ -478,7 +593,7 @@ func (nrc *NetworkRoutingController) syncPeers() {
 		return
 	}
 
-	// establish peer with current set of nodes
+	// establish peer and add Pod CIDRs with current set of nodes
 	currentNodes := make([]string, 0)
 	for _, node := range nodes.Items {
 		nodeIP, _ := getNodeIP(&node)
@@ -518,6 +633,7 @@ func (nrc *NetworkRoutingController) syncPeers() {
 				PeerAs:          nrc.defaultNodeAsnNumber,
 			},
 		}
+
 		// TODO: check if a node is alredy added as nieighbour in a better way than add and catch error
 		if err := nrc.bgpServer.AddNeighbor(n); err != nil {
 			if !strings.Contains(err.Error(), "Can't overwrite the existing peer") {
@@ -723,14 +839,33 @@ func getNodeSubnet(nodeIp net.IP) (net.IPNet, string, error) {
 }
 
 func NewNetworkRoutingController(clientset *kubernetes.Clientset, kubeRouterConfig *options.KubeRouterConfig) (*NetworkRoutingController, error) {
+	// TODO: Remove lookup, ipset.New already does this.
+	_, err := exec.LookPath("ipset")
+	if err != nil {
+		return nil, errors.New("Ensure ipset package is installed: " + err.Error())
+	}
 
 	nrc := NetworkRoutingController{}
 
 	nrc.bgpFullMeshMode = kubeRouterConfig.FullMeshMode
-	nrc.clusterCIDR = kubeRouterConfig.ClusterCIDR
+	nrc.enablePodEgress = kubeRouterConfig.EnablePodEgress
 	nrc.syncPeriod = kubeRouterConfig.RoutesSyncPeriod
 	nrc.clientset = clientset
 
+	if nrc.enablePodEgress || len(nrc.clusterCIDR) != 0 {
+		nrc.enablePodEgress = true
+
+		// TODO: Add bitmap hashtype support to ipset package. It would work well here.
+		podSubnetIpSet, err := ipset.New(podSubnetIpSetName, "hash:net", &ipset.Params{})
+		if err != nil {
+			return nil, fmt.Errorf("failed to create Pod subnet ipset: %s", err.Error())
+		}
+
+		nrc.podSubnetsIpSet = podSubnetIpSet
+	} else {
+		nrc.podSubnetsIpSet = nil
+	}
+
 	if len(kubeRouterConfig.ClusterAsn) != 0 {
 		asn, err := strconv.ParseUint(kubeRouterConfig.ClusterAsn, 0, 32)
 		if err != nil {

app/controllers/network_services_controller.go

@@ -690,6 +690,7 @@ func deleteMasqueradeIptablesRule() error {
 			if err != nil {
 				return errors.New("Failed to run iptables command" + err.Error())
 			}
+			glog.Infof("Deleted iptables masquerade rule: %s", rule)
 			break
 		}
 	}