Add option --bridge-accept-fwmark

Packets with the given firewall mark are accepted by the bridge
driver's filter-FORWARD rules.

The value can either be an integer mark, or it can include a
mask in the format "<mark>/<mask>".

Signed-off-by: Rob Murray <rob.murray@docker.com>

Author: robmry

daemon/command/config_unix.go

@@ -39,6 +39,7 @@ func installConfigFlags(conf *config.Config, flags *pflag.FlagSet) {
 	flags.BoolVar(&conf.BridgeConfig.EnableUserlandProxy, "userland-proxy", true, "Use userland proxy for loopback traffic")
 	flags.StringVar(&conf.BridgeConfig.UserlandProxyPath, "userland-proxy-path", conf.BridgeConfig.UserlandProxyPath, "Path to the userland proxy binary")
 	flags.BoolVar(&conf.BridgeConfig.AllowDirectRouting, "allow-direct-routing", false, "Allow remote access to published ports on container IP addresses")
+	flags.StringVar(&conf.BridgeConfig.BridgeAcceptFwMark, "bridge-accept-fwmark", "", "In bridge networks, accept packets with this firewall mark/mask")
 	flags.StringVar(&conf.CgroupParent, "cgroup-parent", "", "Set parent cgroup for all containers")
 	flags.StringVar(&conf.RemappedRoot, "userns-remap", "", "User/Group setting for user namespaces")
 	flags.BoolVar(&conf.LiveRestoreEnabled, "live-restore", false, "Enable live restore of docker when containers are still running")

daemon/config/config_linux.go

@@ -6,6 +6,7 @@ import (
 	"net"
 	"os/exec"
 	"path/filepath"
+	"strconv"
 	"strings"
 
 	"github.com/containerd/cgroups/v3"
@@ -49,6 +50,7 @@ type BridgeConfig struct {
 	EnableUserlandProxy      bool   `json:"userland-proxy,omitempty"`
 	UserlandProxyPath        string `json:"userland-proxy-path,omitempty"`
 	AllowDirectRouting       bool   `json:"allow-direct-routing,omitempty"`
+	BridgeAcceptFwMark       string `json:"bridge-accept-fwmark,omitempty"`
 }
 
 // DefaultBridgeConfig stores all the parameters for the default bridge network.
@@ -243,15 +245,15 @@ func validatePlatformConfig(conf *Config) error {
 	if err := verifyDefaultIpcMode(conf.IpcMode); err != nil {
 		return err
 	}
-
 	if err := bridge.ValidateFixedCIDRV6(conf.FixedCIDRv6); err != nil {
 		return errors.Wrap(err, "invalid fixed-cidr-v6")
 	}
-
 	if err := validateFirewallBackend(conf.FirewallBackend); err != nil {
 		return errors.Wrap(err, "invalid firewall-backend")
 	}
-
+	if err := validateFwMarkMask(conf.BridgeAcceptFwMark); err != nil {
+		return errors.Wrap(err, "invalid bridge-accept-fwmark")
+	}
 	return verifyDefaultCgroupNsMode(conf.CgroupNamespaceMode)
 }
 
@@ -311,6 +313,22 @@ func validateFirewallBackend(val string) error {
 	return errors.New(`allowed values are "iptables" and "nftables"`)
 }
 
+func validateFwMarkMask(val string) error {
+	if val == "" {
+		return nil
+	}
+	mark, mask, haveMask := strings.Cut(val, "/")
+	if _, err := strconv.ParseUint(mark, 0, 32); err != nil {
+		return fmt.Errorf("invalid firewall mark %q: %w", val, err)
+	}
+	if haveMask {
+		if _, err := strconv.ParseUint(mask, 0, 32); err != nil {
+			return fmt.Errorf("invalid firewall mask %q: %w", val, err)
+		}
+	}
+	return nil
+}
+
 func verifyDefaultCgroupNsMode(mode string) error {
 	cm := container.CgroupnsMode(mode)
 	if !cm.Valid() {

daemon/config/config_linux_test.go

@@ -396,3 +396,71 @@ func TestDaemonLegacyOptions(t *testing.T) {
 		})
 	}
 }
+
+func TestValidateAcceptFwMarkMark(t *testing.T) {
+	tests := []struct {
+		name   string
+		val    string
+		expErr string
+	}{
+		{
+			name: "empty",
+			val:  "",
+		},
+		{
+			name: "dec/no-mask",
+			val:  "1",
+		},
+		{
+			name: "hex/no-mask",
+			val:  "0x1",
+		},
+		{
+			name: "dec/mask",
+			val:  "1/2",
+		},
+		{
+			name: "hex/mask",
+			val:  "0x1/0x2",
+		},
+		{
+			name: "octal/mask",
+			val:  "010/0xff",
+		},
+		{
+			name:   "bad/mark",
+			val:    "hello/0x2",
+			expErr: `invalid firewall mark "hello/0x2": strconv.ParseUint: parsing "hello": invalid syntax`,
+		},
+		{
+			name:   "bad/mark",
+			val:    "1/hello",
+			expErr: `invalid firewall mask "1/hello": strconv.ParseUint: parsing "hello": invalid syntax`,
+		},
+		{
+			name:   "bad/sep",
+			val:    "1+hello",
+			expErr: `invalid firewall mark "1+hello": strconv.ParseUint: parsing "1+hello": invalid syntax`,
+		},
+		{
+			name:   "bad/no-mask",
+			val:    "1/",
+			expErr: `invalid firewall mask "1/": strconv.ParseUint: parsing "": invalid syntax`,
+		},
+		{
+			name:   "bad/negative",
+			val:    "-1",
+			expErr: `invalid firewall mark "-1": strconv.ParseUint: parsing "-1": invalid syntax`,
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			err := validateFwMarkMask(tc.val)
+			if tc.expErr == "" {
+				assert.NilError(t, err)
+			} else {
+				assert.Check(t, is.ErrorContains(err, tc.expErr))
+			}
+		})
+	}
+}

daemon/daemon_unix.go

@@ -938,6 +938,7 @@ func networkPlatformOptions(conf *config.Config) []nwconfig.Option {
 				"EnableIP6Tables":          conf.BridgeConfig.EnableIP6Tables,
 				"Hairpin":                  !conf.EnableUserlandProxy || conf.UserlandProxyPath == "",
 				"AllowDirectRouting":       conf.BridgeConfig.AllowDirectRouting,
+				"AcceptFwMark":             conf.BridgeConfig.BridgeAcceptFwMark,
 			},
 		}),
 	}

daemon/libnetwork/drivers/bridge/bridge_linux.go

@@ -77,6 +77,7 @@ type configuration struct {
 	// hairpinned.
 	Hairpin            bool
 	AllowDirectRouting bool
+	AcceptFwMark       string
 }
 
 // networkConfiguration for network specific configuration
@@ -429,6 +430,7 @@ func (n *bridgeNetwork) newFirewallerNetwork(ctx context.Context) (_ firewaller.
 		ICC:                   n.config.EnableICC,
 		Masquerade:            n.config.EnableIPMasquerade,
 		TrustedHostInterfaces: n.config.TrustedHostInterfaces,
+		AcceptFwMark:          n.driver.config.AcceptFwMark,
 		Config4:               config4,
 		Config6:               config6,
 	})

daemon/libnetwork/drivers/bridge/internal/firewaller/firewaller.go

@@ -48,6 +48,10 @@ type NetworkConfig struct {
 	// bridge itself). In particular, these are not external interfaces for the purpose of
 	// blocking direct-routing to a container's IP address.
 	TrustedHostInterfaces []string
+	// AcceptFwMark is a firewall mark/mask. Packets with this mark will not be dropped by
+	// per-port blocking rules. So, packets with this mark have access to unpublished
+	// container ports.
+	AcceptFwMark string
 	// Config4 contains IPv4-specific configuration for the network.
 	Config4 NetworkConfigFam
 	// Config6 contains IPv6-specific configuration for the network.

daemon/libnetwork/drivers/bridge/internal/iptabler/network.go

@@ -7,6 +7,8 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
+	"strconv"
+	"strings"
 
 	"github.com/containerd/log"
 	"github.com/docker/docker/daemon/libnetwork/drivers/bridge/internal/firewaller"
@@ -263,6 +265,18 @@ func setDefaultForwardRule(ipVersion iptables.IPVersion, ifName string, unprotec
 }
 
 func (n *network) setupNonInternalNetworkRules(ctx context.Context, ipVer iptables.IPVersion, config firewaller.NetworkConfigFam, enable bool) error {
+	if n.config.AcceptFwMark != "" {
+		fwm, err := iptablesFwMark(n.config.AcceptFwMark)
+		if err != nil {
+			return err
+		}
+		if err := programChainRule(iptables.Rule{IPVer: ipVer, Table: iptables.Filter, Chain: DockerForwardChain, Args: []string{
+			"-m", "mark", "--mark", fwm, "-j", "ACCEPT",
+		}}, "ALLOW FW MARK", enable); err != nil {
+			return err
+		}
+	}
+
 	var natArgs, hpNatArgs []string
 	if config.HostIP.IsValid() {
 		// The user wants IPv4/IPv6 SNAT with the given address.
@@ -459,3 +473,23 @@ func setupInternalNetworkRules(ctx context.Context, bridgeIface string, prefix n
 	// Set Inter Container Communication.
 	return setIcc(ctx, version, bridgeIface, icc, true, insert)
 }
+
+// iptablesFwMark takes a string representing a firewall mark with an optional
+// "/mask" parses the mark and mask, and returns the same "mark/mask" with the
+// numbers converted to decimal, because strings.ParseUint accepts more integer
+// formats than iptables.
+func iptablesFwMark(val string) (string, error) {
+	markStr, maskStr, haveMask := strings.Cut(val, "/")
+	mark, err := strconv.ParseUint(markStr, 0, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid firewall mark %q: %w", val, err)
+	}
+	if haveMask {
+		mask, err := strconv.ParseUint(maskStr, 0, 32)
+		if err != nil {
+			return "", fmt.Errorf("invalid firewall mask %q: %w", val, err)
+		}
+		return fmt.Sprintf("%d/%d", mark, mask), nil
+	}
+	return strconv.FormatUint(mark, 10), nil
+}

daemon/libnetwork/drivers/bridge/internal/nftabler/network.go

@@ -5,6 +5,8 @@ package nftabler
 import (
 	"context"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/containerd/log"
 	"github.com/docker/docker/daemon/libnetwork/drivers/bridge/internal/firewaller"
@@ -157,6 +159,20 @@ func (n *network) configure(ctx context.Context, table nftables.TableRef, conf f
 		}
 		cleanup.Add(cf)
 	} else {
+		// AcceptFwMark
+		if n.config.AcceptFwMark != "" {
+			fwm, err := nftFwMark(n.config.AcceptFwMark)
+			if err != nil {
+				return nil, fmt.Errorf("adding fwmark %q for %q: %w", n.config.AcceptFwMark, n.config.IfName, err)
+			}
+			cf, err = fwdInChain.AppendRuleCf(ctx, fwdInAcceptFwMarkRuleGroup,
+				`meta mark %s counter accept comment "ALLOW FW MARK"`, fwm)
+			if err != nil {
+				return nil, fmt.Errorf("adding ALLOW FW MARK rule for %q: %w", n.config.IfName, err)
+			}
+			cleanup.Add(cf)
+		}
+
 		// Inter-Container Communication
 		cf, err = fwdInChain.AppendRuleCf(ctx, fwdInICCRuleGroup, "iifname == %s counter %s comment ICC",
 			n.config.IfName, iccVerdict)
@@ -270,3 +286,23 @@ func chainNatPostRtOut(ifName string) string {
 func chainNatPostRtIn(ifName string) string {
 	return "nat-postrouting-in__" + ifName
 }
+
+// nftFwMark takes a string representing a firewall mark with an optional
+// "/mask", parses the mark and mask, and returns an nftables expression
+// representing the same mask/mark. Numbers are converted to decimal, because
+// strings.ParseUint accepts more integer formats than nft.
+func nftFwMark(val string) (string, error) {
+	markStr, maskStr, haveMask := strings.Cut(val, "/")
+	mark, err := strconv.ParseUint(markStr, 0, 32)
+	if err != nil {
+		return "", fmt.Errorf("invalid firewall mark %q: %w", val, err)
+	}
+	if haveMask {
+		mask, err := strconv.ParseUint(maskStr, 0, 32)
+		if err != nil {
+			return "", fmt.Errorf("invalid firewall mask %q: %w", val, err)
+		}
+		return fmt.Sprintf("and %d == %d", mask, mark), nil
+	}
+	return strconv.FormatUint(mark, 10), nil
+}

daemon/libnetwork/drivers/bridge/internal/nftabler/nftabler.go

@@ -34,7 +34,8 @@ const (
 )
 
 const (
-	fwdInLegacyLinksRuleGroup = iota + initialRuleGroup + 1
+	fwdInAcceptFwMarkRuleGroup = iota + initialRuleGroup + 1
+	fwdInLegacyLinksRuleGroup
 	fwdInICCRuleGroup
 	fwdInPortsRuleGroup
 	fwdInFinalRuleGroup