check ipv4 routes for default network interface, only fall back to eth0 if none can be found (#498)

check ipv4 routes for default network interface, only fall back to eth0 if none can be found

Author: yinyic

README.md

@@ -25,7 +25,7 @@ Additionally, the following environment variable(s) can be used to configure the
 |:----------------|:----------------------------|:------------|:-----------------------|
 | `ECS_SKIP_LOCALHOST_TRAFFIC_FILTER` | <true | false> | By default, the ecs-init service adds an iptable rule to drop non-local packets to localhost if they're not part of an existing forwarded connection or DNAT, and removes the rule upon stop. If `ECS_SKIP_LOCALHOST_TRAFFIC_FILTER` is set to true, this rule will not be added/removed. | false |
 | `ECS_ALLOW_OFFHOST_INTROSPECTION_ACCESS` | <true | false> | By default, the ecs-init service adds an iptable rule to block access to ECS Agent's introspection port from off-host (or containers in awsvpc network mode), and removes the rule upon stop. If `ECS_ALLOW_OFFHOST_INTROSPECTION_ACCESS` is set to true, this rule will not be added/removed. | false |
-| `ECS_OFFHOST_INTROSPECTION_INTERFACE_NAME` | `eth0` | Primary network interface name to be used for blocking offhost agent introspection port access. By default, this value is `eth0` | `eth0` |
+| `ECS_OFFHOST_INTROSPECTION_INTERFACE_NAME` | `eth0` | Primary network interface name to be used for blocking offhost agent introspection port access. By default, this value is the interface that handles the default route (`0.0.0.0/0`) in kernel routing table (`/proc/net/route`). If none could be found, we fall back to `eth0` | - (Resolved at runtime) |
 
 The above environment variable(s) can be used in the following way
 - On Amazon Linux 1, the flag `ECS_SKIP_LOCALHOST_TRAFFIC_FILTER` can be turned on by adding `env ECS_SKIP_LOCALHOST_TRAFFIC_FILTER=true` to /etc/init/ecs.conf.

ecs-init/exec/iptables/iptables.go

@@ -14,7 +14,9 @@
 package iptables
 
 import (
+	"bufio"
 	"fmt"
+	"io"
 	"os"
 	"strconv"
 	"strings"
@@ -47,7 +49,15 @@ const (
 	offhostIntrospectionAccessConfigEnv   = "ECS_ALLOW_OFFHOST_INTROSPECTION_ACCESS"
 	offhostIntrospectonAccessInterfaceEnv = "ECS_OFFHOST_INTROSPECTION_INTERFACE_NAME"
 	agentIntrospectionServerPort          = "51678"
-	defaultOffhostIntrospectionInterface  = "eth0"
+
+	ipv4RouteFile                         = "/proc/net/route"
+	ipv4ZeroAddrInHex                     = "00000000"
+	loopbackInterfaceName                 = "lo"
+	fallbackOffhostIntrospectionInterface = "eth0"
+)
+
+var (
+	defaultOffhostIntrospectionInterface = ""
 )
 
 // NetfilterRoute implements the engine.credentialsProxyRoute interface by
@@ -69,6 +79,14 @@ func NewNetfilterRoute(cmdExec exec.Exec) (*NetfilterRoute, error) {
 		return nil, err
 	}
 
+	defaultOffhostIntrospectionInterface, err = getOffhostIntrospectionInterface()
+	if err != nil {
+		log.Warnf("Error resolving default offhost introspection network interface, will use eth0 as fallback: %+v", err)
+		// fall back to the previous behavior (always use 'eth0') in the rare case that it
+		// might affect some customer with a special routing setup that's previously working.
+		defaultOffhostIntrospectionInterface = fallbackOffhostIntrospectionInterface
+	}
+
 	return &NetfilterRoute{
 		cmdExec: cmdExec,
 	}, nil
@@ -189,18 +207,51 @@ func getBlockIntrospectionOffhostAccessInputChainArgs() []string {
 	return []string{
 		"INPUT",
 		"-p", "tcp",
-		"-i", getOffhostIntrospectionInterface(),
+		"-i", defaultOffhostIntrospectionInterface,
 		"--dport", agentIntrospectionServerPort,
 		"-j", "DROP",
 	}
 }
 
-func getOffhostIntrospectionInterface() string {
+func getOffhostIntrospectionInterface() (string, error) {
 	s := os.Getenv(offhostIntrospectonAccessInterfaceEnv)
 	if s != "" {
-		return s
+		return s, nil
+	}
+	return getDefaultNetworkInterfaceIPv4()
+}
+
+// Parse /proc/net/route file and retrieves a non-loopback default network interface for IPv4 (which maps to default 0.0.0.0/0 destination)
+// Example file content:
+// $ sudo cat /proc/net/route
+// Iface   Destination Gateway     Flags   RefCnt  Use Metric  Mask   MTU Window  IRTT
+// ens5    00000000    01201FAC    0003    0   		0   512 00000000    0   0   	0
+// ...
+//
+// 1st column contains interface name
+// 2nd column contains destination network in hex
+var getDefaultNetworkInterfaceIPv4 = func() (string, error) {
+	input, err := os.Open(ipv4RouteFile)
+	if err != nil {
+		return "", fmt.Errorf("could not get IPv4 route input: %v", err)
+	}
+	defer input.Close()
+	return scanIPv4RoutesForDefaultInterface(input)
+}
+
+func scanIPv4RoutesForDefaultInterface(input io.Reader) (string, error) {
+	scanner := bufio.NewScanner(input)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "Iface") { // skip header line
+			continue
+		}
+		fields := strings.Fields(line)
+		if (fields[1] == ipv4ZeroAddrInHex) && (fields[0] != loopbackInterfaceName) {
+			return fields[0], nil
+		}
 	}
-	return defaultOffhostIntrospectionInterface
+	return "", fmt.Errorf("could not find a default IPv4 route through non-loopback interface")
 }
 
 func getOutputChainArgs() []string {

ecs-init/exec/iptables/iptables_test.go

@@ -16,6 +16,7 @@ package iptables
 import (
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -40,9 +41,10 @@ var (
 		"!", "--ctstate", "RELATED,ESTABLISHED,DNAT",
 		"-j", "DROP",
 	}
+	offhostIntrospectionInterface                 = "ens5"
 	blockIntrospectionOffhostAccessInputRouteArgs = []string{
 		"-p", "tcp",
-		"-i", "eth0",
+		"-i", offhostIntrospectionInterface,
 		"--dport", agentIntrospectionServerPort,
 		"-j", "DROP",
 	}
@@ -59,9 +61,30 @@ var (
 		"-j", "REDIRECT",
 		"--to-ports", localhostCredentialsProxyPort,
 	}
+
+	testIPV4RouteInput = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+ens5	00000000	01201FAC	0003	0	0	0	00000000	0	0	0                                                                               
+ens5	FEA9FEA9	00000000	0005	0	0	0	FFFFFFFF	0	0	0                                                                               
+ens5	00201FAC	00000000	0001	0	0	0	00F0FFFF	0	0	0
+`
 )
 
+func overrideIPRouteInput(ipv4RouteInput string) func() {
+	originalv4 := getDefaultNetworkInterfaceIPv4
+
+	getDefaultNetworkInterfaceIPv4 = func() (string, error) {
+		return scanIPv4RoutesForDefaultInterface(strings.NewReader(ipv4RouteInput))
+	}
+
+	return func() {
+		getDefaultNetworkInterfaceIPv4 = originalv4
+		// in real environment we'll only set it once, for testing we unset it after executing relevant test cases
+		defaultOffhostIntrospectionInterface = ""
+	}
+}
+
 func TestNewNetfilterRouteFailsWhenExecutableNotFound(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -72,7 +95,21 @@ func TestNewNetfilterRouteFailsWhenExecutableNotFound(t *testing.T) {
 	assert.Error(t, err, "Expected error when executable's path lookup fails")
 }
 
+func TestNewNetfilterRouteWithDefaultOffhostIntrospectionInterfaceFallback(t *testing.T) {
+	defer overrideIPRouteInput("")()
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+
+	mockExec := NewMockExec(ctrl)
+	mockExec.EXPECT().LookPath(iptablesExecutable).Return("", nil)
+
+	_, err := NewNetfilterRoute(mockExec)
+	assert.NoError(t, err)
+	assert.Equal(t, defaultOffhostIntrospectionInterface, fallbackOffhostIntrospectionInterface)
+}
+
 func TestCreate(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	testCases := []struct {
 		setOffhostInterface bool
 		inputRouteArgs      []string
@@ -124,6 +161,7 @@ func TestCreate(t *testing.T) {
 }
 
 func TestCreateSkipLocalTrafficFilter(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	os.Setenv("ECS_SKIP_LOCALHOST_TRAFFIC_FILTER", "true")
 	defer os.Unsetenv("ECS_SKIP_LOCALHOST_TRAFFIC_FILTER")
 
@@ -226,6 +264,7 @@ func TestCreateErrorOnInputChainCommandError(t *testing.T) {
 }
 
 func TestCreateErrorOnOutputChainCommandError(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -256,6 +295,7 @@ func TestCreateErrorOnOutputChainCommandError(t *testing.T) {
 }
 
 func TestRemove(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -287,6 +327,7 @@ func TestRemove(t *testing.T) {
 }
 
 func TestRemoveSkipLocalTrafficFilter(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	os.Setenv("ECS_SKIP_LOCALHOST_TRAFFIC_FILTER", "true")
 	defer os.Unsetenv("ECS_SKIP_LOCALHOST_TRAFFIC_FILTER")
 
@@ -316,6 +357,7 @@ func TestRemoveSkipLocalTrafficFilter(t *testing.T) {
 }
 
 func TestRemoveAllowIntrospectionOffhostAccess(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	os.Setenv(offhostIntrospectionAccessConfigEnv, "true")
 	defer os.Unsetenv(offhostIntrospectionAccessConfigEnv)
 
@@ -348,6 +390,7 @@ func TestRemoveAllowIntrospectionOffhostAccess(t *testing.T) {
 }
 
 func TestRemoveErrorOnPreroutingChainCommandError(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -378,6 +421,7 @@ func TestRemoveErrorOnPreroutingChainCommandError(t *testing.T) {
 }
 
 func TestRemoveErrorOnOutputChainCommandError(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -408,6 +452,7 @@ func TestRemoveErrorOnOutputChainCommandError(t *testing.T) {
 }
 
 func TestRemoveErrorOnInputChainCommandsErrors(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
 	ctrl := gomock.NewController(t)
 	defer ctrl.Finish()
 
@@ -473,10 +518,12 @@ func TestGetLocalhostTrafficFilterInputChainArgs(t *testing.T) {
 }
 
 func TestGetBlockIntrospectionOffhostAccessInputChainArgs(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
+	defaultOffhostIntrospectionInterface, _ = getOffhostIntrospectionInterface()
 	assert.Equal(t, []string{
 		"INPUT",
 		"-p", "tcp",
-		"-i", "eth0",
+		"-i", "ens5",
 		"--dport", "51678",
 		"-j", "DROP",
 	}, getBlockIntrospectionOffhostAccessInputChainArgs())
@@ -504,3 +551,49 @@ func TestGetActionName(t *testing.T) {
 func expectedArgs(table, action, chain string, args []string) []string {
 	return append([]string{"-t", table, action, chain}, args...)
 }
+
+func TestScanIPv4RoutesHappyCase(t *testing.T) {
+	iface, err := scanIPv4RoutesForDefaultInterface(strings.NewReader(testIPV4RouteInput))
+	assert.NoError(t, err)
+	assert.Equal(t, offhostIntrospectionInterface, iface)
+}
+
+func TestScanIPv4RoutesNoDefaultRoute(t *testing.T) {
+	iface, err := scanIPv4RoutesForDefaultInterface(strings.NewReader(""))
+	assert.Error(t, err)
+	assert.Equal(t, "", iface)
+}
+
+func TestScanIPv4RoutesNoDefaultRouteExceptLoopback(t *testing.T) {
+	var testInput = `Iface	Destination	Gateway 	Flags	RefCnt	Use	Metric	Mask		MTU	Window	IRTT                                                       
+lo	00000000	01201FAC	0003	0	0	0	00000000	0	0	0
+`
+	iface, err := scanIPv4RoutesForDefaultInterface(strings.NewReader(testInput))
+	assert.Error(t, err)
+	assert.Equal(t, "", iface)
+}
+
+func TestGetOffhostIntrospectionInterfaceWithEnvOverride(t *testing.T) {
+	os.Setenv(offhostIntrospectonAccessInterfaceEnv, "test_iface")
+	defer os.Unsetenv(offhostIntrospectonAccessInterfaceEnv)
+
+	iface, err := getOffhostIntrospectionInterface()
+	assert.NoError(t, err)
+	assert.Equal(t, "test_iface", iface)
+}
+
+func TestGetOffhostIntrospectionInterfaceUseDefaultV4(t *testing.T) {
+	defer overrideIPRouteInput(testIPV4RouteInput)()
+
+	iface, err := getOffhostIntrospectionInterface()
+	assert.NoError(t, err)
+	assert.Equal(t, offhostIntrospectionInterface, iface)
+}
+
+func TestGetOffhostIntrospectionInterfaceFailure(t *testing.T) {
+	defer overrideIPRouteInput("")()
+
+	iface, err := getOffhostIntrospectionInterface()
+	assert.Error(t, err)
+	assert.Equal(t, "", iface)
+}