[RFC] prevent host services from being accessible through service IPs (#618)

* prevent host services from being accessible through service IPs

- on startup create ipsets and firewall rules
- on sync update ipsets
- on cleanup remove firewall rules and ipsets

Fixes #282.

Signed-off-by: Steven Armstrong <[email protected]>

* ensure iptables rules are also available during cleanup

Signed-off-by: Steven Armstrong <[email protected]>

* first check if chain exists

Signed-off-by: Steven Armstrong <[email protected]>

* err not a new variable

Signed-off-by: Steven Armstrong <[email protected]>

* more redeclared vars

Signed-off-by: Steven Armstrong <[email protected]>

* maintain a ipset for local addresses and exclude those from our default deny rule

Signed-off-by: Steven Armstrong <[email protected]>

* copy/paste errors

Signed-off-by: Steven Armstrong <[email protected]>

Author: Steven Armstrong

pkg/controllers/proxy/network_services_controller.go

@@ -56,7 +56,10 @@ const (
 	svcSchedFlagsAnnotation = "kube-router.io/service.schedflags"
 
 	LeaderElectionRecordAnnotationKey = "control-plane.alpha.kubernetes.io/leader"
-	svcIpSetName                      = "KUBE-SVC-ALL"
+	localIPsIPSetName                 = "kube-router-local-ips"
+	ipvsServicesIPSetName             = "kube-router-ipvs-services"
+	serviceIPsIPSetName               = "kube-router-service-ips"
+	ipvsFirewallChainName             = "KUBE-ROUTER-SERVICES"
 )
 
 var (
@@ -214,6 +217,9 @@ type NetworkServicesController struct {
 	ln                  LinuxNetworking
 	readyForUpdates     bool
 
+	// Map of ipsets that we use.
+	ipsetMap map[string]*utils.Set
+
 	svcLister cache.Indexer
 	epLister  cache.Indexer
 	podLister cache.Indexer
@@ -318,6 +324,12 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control
 		return errors.New(sysctlErr.Error())
 	}
 
+	// https://github.com/cloudnativelabs/kube-router/issues/282
+	err = nsc.setupIpvsFirewall()
+	if err != nil {
+		return errors.New("Error setting up ipvs firewall: " + err.Error())
+	}
+
 	// loop forever unitl notified to stop on stopCh
 	for {
 		select {
@@ -375,52 +387,110 @@ func (nsc *NetworkServicesController) sync() error {
 	return nil
 }
 
+func getIpvsFirewallInputChainRule() []string {
+	// The iptables rule for use in {setup,cleanup}IpvsFirewall.
+	return []string{
+		"-m", "comment", "--comment", "handle traffic to IPVS service IPs in custom chain",
+		"-m", "set", "--match-set", serviceIPsIPSetName, "dst",
+		"-j", ipvsFirewallChainName}
+}
+
 func (nsc *NetworkServicesController) setupIpvsFirewall() error {
-	// Add ipset containg all SVCs
+	/*
+	   - create ipsets
+	   - create firewall rules
+	*/
+
+	var err error
+	var ipset *utils.Set
+
 	ipSetHandler, err := utils.NewIPSet(false)
 	if err != nil {
 		return err
 	}
 
-	svcIpSet, err := ipSetHandler.Create(svcIpSetName, utils.TypeHashIPPort, utils.OptionTimeout, "0")
+	// Remember ipsets for use in syncIpvsFirewall
+	nsc.ipsetMap = make(map[string]*utils.Set)
+
+	// Create ipset for local addresses.
+	ipset, err = ipSetHandler.Create(localIPsIPSetName, utils.TypeHashIP, utils.OptionTimeout, "0")
 	if err != nil {
 		return fmt.Errorf("failed to create ipset: %s", err.Error())
 	}
+	nsc.ipsetMap[localIPsIPSetName] = ipset
 
-	ipvsSvcs, err := nsc.ln.ipvsGetServices()
+	// Create 2 ipsets for services. One for 'ip' and one for 'ip,port'
+	ipset, err = ipSetHandler.Create(serviceIPsIPSetName, utils.TypeHashIP, utils.OptionTimeout, "0")
 	if err != nil {
-		return errors.New("Failed to list IPVS services: " + err.Error())
+		return fmt.Errorf("failed to create ipset: %s", err.Error())
 	}
+	nsc.ipsetMap[serviceIPsIPSetName] = ipset
 
-	svcSets := make([]string, 0, len(ipvsSvcs))
-	for _, ipvsSvc := range ipvsSvcs {
-		protocol := "udp"
-		if ipvsSvc.Protocol == syscall.IPPROTO_TCP {
-			protocol = "tcp"
+	ipset, err = ipSetHandler.Create(ipvsServicesIPSetName, utils.TypeHashIPPort, utils.OptionTimeout, "0")
+	if err != nil {
+		return fmt.Errorf("failed to create ipset: %s", err.Error())
+	}
+	nsc.ipsetMap[ipvsServicesIPSetName] = ipset
+
+	// Setup a custom iptables chain to explicitly allow input traffic to
+	// ipvs services only.
+	iptablesCmdHandler, err := iptables.New()
+	if err != nil {
+		return errors.New("Failed to initialize iptables executor" + err.Error())
+	}
+
+	// ClearChain either clears an existing chain or creates a new one.
+	err = iptablesCmdHandler.ClearChain("filter", ipvsFirewallChainName)
+	if err != nil {
+		return fmt.Errorf("Failed to run iptables command: %s", err.Error())
+	}
+
+	var comment string
+	var args []string
+
+	comment = "allow input traffic to ipvs services"
+	args = []string{"-m", "comment", "--comment", comment,
+		"-m", "set", "--match-set", ipvsServicesIPSetName, "dst,dst",
+		"-j", "ACCEPT"}
+	exists, err := iptablesCmdHandler.Exists("filter", ipvsFirewallChainName, args...)
+	if err != nil {
+		return fmt.Errorf("Failed to run iptables command: %s", err.Error())
+	}
+	if !exists {
+		err := iptablesCmdHandler.Insert("filter", ipvsFirewallChainName, 1, args...)
+		if err != nil {
+			return fmt.Errorf("Failed to run iptables command: %s", err.Error())
 		}
-		set := fmt.Sprintf("%s,%s:%d", ipvsSvc.Address.String(), protocol, ipvsSvc.Port)
-		svcSets = append(svcSets, set)
 	}
 
-	err = svcIpSet.Refresh(svcSets, utils.OptionTimeout, "0")
+	comment = "allow icmp echo requests to service IPs"
+	args = []string{"-m", "comment", "--comment", comment,
+		"-p", "icmp", "--icmp-type", "echo-request",
+		"-j", "ACCEPT"}
+	err = iptablesCmdHandler.AppendUnique("filter", ipvsFirewallChainName, args...)
 	if err != nil {
-		return fmt.Errorf("failed to sync ipset: %s", err.Error())
+		return fmt.Errorf("Failed to run iptables command: %s", err.Error())
 	}
 
-	// Add iptables rule to allow input traffic to ipvs services
-	iptablesCmdHandler, err := iptables.New()
+	// We exclude the local addresses here as that would otherwise block all
+	// traffic to local addresses if any NodePort service exists.
+	comment = "reject all unexpected traffic to service IPs"
+	args = []string{"-m", "comment", "--comment", comment,
+		"-m", "set", "!", "--match-set", localIPsIPSetName, "dst",
+		"-j", "REJECT", "--reject-with", "icmp-port-unreachable"}
+	err = iptablesCmdHandler.AppendUnique("filter", ipvsFirewallChainName, args...)
 	if err != nil {
-		return errors.New("Failed to initialize iptables executor" + err.Error())
+		return fmt.Errorf("Failed to run iptables command: %s", err.Error())
 	}
 
-	comment := "allow input traffic to ipvs services"
-	args := []string{"-m", "comment", "--comment", comment, "-m", "set", "--match-set", svcIpSetName, "dst,dst", "-j", "ACCEPT"}
-	exists, err := iptablesCmdHandler.Exists("filter", "INPUT", args...)
+	// Pass incomming traffic into our custom chain.
+	ipvsFirewallInputChainRule := getIpvsFirewallInputChainRule()
+	exists, err = iptablesCmdHandler.Exists("filter", "INPUT", ipvsFirewallInputChainRule...)
 	if err != nil {
 		return fmt.Errorf("Failed to run iptables command: %s", err.Error())
 	}
 	if !exists {
-		err := iptablesCmdHandler.Insert("filter", "INPUT", 1, args...)
+		err = iptablesCmdHandler.Insert("filter", "INPUT", 1, ipvsFirewallInputChainRule...)
 		if err != nil {
 			return fmt.Errorf("Failed to run iptables command: %s", err.Error())
 		}
@@ -429,6 +499,114 @@ func (nsc *NetworkServicesController) setupIpvsFirewall() error {
 	return nil
 }
 
+func (nsc *NetworkServicesController) cleanupIpvsFirewall() {
+	/*
+	   - delete firewall rules
+	   - delete ipsets
+	*/
+	var err error
+
+	// Clear iptables rules.
+	iptablesCmdHandler, err := iptables.New()
+	if err != nil {
+		glog.Errorf("Failed to initialize iptables executor: %s", err.Error())
+	} else {
+		ipvsFirewallInputChainRule := getIpvsFirewallInputChainRule()
+		err = iptablesCmdHandler.Delete("filter", "INPUT", ipvsFirewallInputChainRule...)
+		if err != nil {
+			glog.Errorf("Failed to run iptables command: %s", err.Error())
+		}
+
+		err = iptablesCmdHandler.ClearChain("filter", ipvsFirewallChainName)
+		if err != nil {
+			glog.Errorf("Failed to run iptables command: %s", err.Error())
+		}
+
+		err = iptablesCmdHandler.DeleteChain("filter", ipvsFirewallChainName)
+		if err != nil {
+			glog.Errorf("Failed to run iptables command: %s", err.Error())
+		}
+	}
+
+	// Clear ipsets.
+	ipSetHandler, err := utils.NewIPSet(false)
+	if err != nil {
+		glog.Errorf("Failed to initialize ipset handler: %s", err.Error())
+	} else {
+		err = ipSetHandler.Destroy(localIPsIPSetName)
+		if err != nil {
+			glog.Errorf("failed to destroy ipset: %s", err.Error())
+		}
+
+		err = ipSetHandler.Destroy(serviceIPsIPSetName)
+		if err != nil {
+			glog.Errorf("failed to destroy ipset: %s", err.Error())
+		}
+
+		err = ipSetHandler.Destroy(ipvsServicesIPSetName)
+		if err != nil {
+			glog.Errorf("failed to destroy ipset: %s", err.Error())
+		}
+	}
+}
+
+func (nsc *NetworkServicesController) syncIpvsFirewall() error {
+	/*
+	   - update ipsets based on currently active IPVS services
+	*/
+	var err error
+
+	localIPsIPSet := nsc.ipsetMap[localIPsIPSetName]
+
+	// Populate local addresses ipset.
+	addrs, err := getAllLocalIPs()
+	localIPsSets := make([]string, 0, len(addrs))
+	for _, addr := range addrs {
+		localIPsSets = append(localIPsSets, addr.IP.String())
+	}
+	err = localIPsIPSet.Refresh(localIPsSets, utils.OptionTimeout, "0")
+	if err != nil {
+		return fmt.Errorf("failed to sync ipset: %s", err.Error())
+	}
+
+	// Populate service ipsets.
+	ipvsServices, err := nsc.ln.ipvsGetServices()
+	if err != nil {
+		return errors.New("Failed to list IPVS services: " + err.Error())
+	}
+
+	serviceIPsSets := make([]string, 0, len(ipvsServices))
+	ipvsServicesSets := make([]string, 0, len(ipvsServices))
+
+	for _, ipvsService := range ipvsServices {
+		protocol := "udp"
+		if ipvsService.Protocol == syscall.IPPROTO_TCP {
+			protocol = "tcp"
+		}
+
+		serviceIPsSet := ipvsService.Address.String()
+		serviceIPsSets = append(serviceIPsSets, serviceIPsSet)
+
+		ipvsServicesSet := fmt.Sprintf("%s,%s:%d", ipvsService.Address.String(), protocol, ipvsService.Port)
+		ipvsServicesSets = append(ipvsServicesSets, ipvsServicesSet)
+
+	}
+
+	serviceIPsIPSet := nsc.ipsetMap[serviceIPsIPSetName]
+	err = serviceIPsIPSet.Refresh(serviceIPsSets, utils.OptionTimeout, "0")
+	if err != nil {
+		return fmt.Errorf("failed to sync ipset: %s", err.Error())
+	}
+
+	ipvsServicesIPSet := nsc.ipsetMap[ipvsServicesIPSetName]
+	err = ipvsServicesIPSet.Refresh(ipvsServicesSets, utils.OptionTimeout, "0")
+	if err != nil {
+		return fmt.Errorf("failed to sync ipset: %s", err.Error())
+	}
+
+	return nil
+}
+
 func (nsc *NetworkServicesController) publishMetrics(serviceInfoMap serviceInfoMap) error {
 	start := time.Now()
 	defer func() {
@@ -952,7 +1130,7 @@ func (nsc *NetworkServicesController) syncIpvsServices(serviceInfoMap serviceInf
 		}
 	}
 
-	err = nsc.setupIpvsFirewall()
+	err = nsc.syncIpvsFirewall()
 	if err != nil {
 		glog.Errorf("Error syncing ipvs svc iptable rules: %s", err.Error())
 	}
@@ -2061,6 +2239,8 @@ func (nsc *NetworkServicesController) Cleanup() {
 		return
 	}
 
+	nsc.cleanupIpvsFirewall()
+
 	// delete dummy interface used to assign cluster IP's
 	dummyVipInterface, err := netlink.LinkByName(KUBE_DUMMY_IF)
 	if err != nil {