Merge pull request moby#50283 from robmry/gatewayness_visible_to_libnet_drivers

Tell libnet's drivers which endpoints have been selected as gateways

Author: akerouanton

libnetwork/driverapi/driverapi.go

@@ -83,7 +83,7 @@ type Driver interface {
 type ExtConner interface {
 	// ProgramExternalConnectivity invokes the driver method which does the necessary
 	// programming to allow the external connectivity dictated by the passed options
-	ProgramExternalConnectivity(ctx context.Context, nid, eid string, options map[string]interface{}) error
+	ProgramExternalConnectivity(ctx context.Context, nid, eid string, options map[string]interface{}, gw4Id, gw6Id string) error
 
 	// RevokeExternalConnectivity asks the driver to remove any external connectivity
 	// programming that was done so far

libnetwork/drivers/bridge/bridge_linux.go

@@ -125,17 +125,18 @@ type connectivityConfiguration struct {
 }
 
 type bridgeEndpoint struct {
-	id              string
-	nid             string
-	srcName         string
-	addr            *net.IPNet
-	addrv6          *net.IPNet
-	macAddress      net.HardwareAddr
-	containerConfig *containerConfiguration
-	extConnConfig   *connectivityConfiguration
-	portMapping     []portBinding // Operational port bindings
-	dbIndex         uint64
-	dbExists        bool
+	id               string
+	nid              string
+	srcName          string
+	addr             *net.IPNet
+	addrv6           *net.IPNet
+	macAddress       net.HardwareAddr
+	containerConfig  *containerConfiguration
+	extConnConfig    *connectivityConfiguration
+	portMapping      []portBinding   // Operational port bindings
+	portBindingState portBindingMode // Not persisted, even on live-restore port mappings are re-created.
+	dbIndex          uint64
+	dbExists         bool
 }
 
 type bridgeNetwork struct {
@@ -1488,10 +1489,17 @@ func (d *driver) Leave(nid, eid string) error {
 	return nil
 }
 
-func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid string, options map[string]interface{}) error {
+type portBindingMode struct {
+	ipv4 bool
+	ipv6 bool
+}
+
+func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid string, options map[string]interface{}, gw4Id, gw6Id string) error {
 	ctx, span := otel.Tracer("").Start(ctx, spanPrefix+".ProgramExternalConnectivity", trace.WithAttributes(
 		attribute.String("nid", nid),
-		attribute.String("eid", eid)))
+		attribute.String("eid", eid),
+		attribute.String("gw4", gw4Id),
+		attribute.String("gw6", gw6Id)))
 	defer span.End()
 
 	// Make sure the network isn't deleted, or in the middle of a firewalld reload, while
@@ -1518,15 +1526,26 @@ func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid strin
 		return err
 	}
 
+	var pbmReq portBindingMode
+	// Act as the IPv4 gateway if explicitly selected.
+	if gw4Id == eid {
+		pbmReq.ipv4 = true
+	}
+	// Act as the IPv6 gateway if explicitly selected - or if there's no IPv6
+	// gateway, but this endpoint is the IPv4 gateway (in which case, the userland
+	// proxy may proxy between host v6 and container v4 addresses.)
+	if gw6Id == eid || (gw6Id == "" && gw4Id == eid) {
+		pbmReq.ipv6 = true
+	}
+
 	// Program any required port mapping and store them in the endpoint
 	if endpoint.extConnConfig != nil && endpoint.extConnConfig.PortBindings != nil {
 		endpoint.portMapping, err = network.addPortMappings(
 			ctx,
-			endpoint.addr,
-			endpoint.addrv6,
+			endpoint,
 			endpoint.extConnConfig.PortBindings,
 			network.config.DefaultBindingIP,
-			endpoint.extConnConfig.NoProxy6To4,
+			pbmReq,
 		)
 		if err != nil {
 			return err
@@ -1543,6 +1562,9 @@ func (d *driver) ProgramExternalConnectivity(ctx context.Context, nid, eid strin
 		}
 	}()
 
+	// Remember the new port binding state.
+	endpoint.portBindingState = pbmReq
+
 	// Clean the connection tracker state of the host for the specific endpoint. This is needed because some flows may
 	// be bound to the local proxy, or to the host (for UDP packets), and won't be redirected to the new endpoints.
 	clearConntrackEntries(d.nlh, endpoint)
@@ -1836,13 +1858,5 @@ func parseConnectivityOptions(cOptions map[string]interface{}) (*connectivityCon
 		}
 	}
 
-	if opt, ok := cOptions[netlabel.NoProxy6To4]; ok {
-		if noProxy6To4, ok := opt.(bool); ok {
-			cc.NoProxy6To4 = noProxy6To4
-		} else {
-			return nil, types.InvalidParameterErrorf("invalid "+netlabel.NoProxy6To4+" in connectivity configuration: %v", opt)
-		}
-	}
-
 	return cc, nil
 }

libnetwork/drivers/bridge/bridge_linux_test.go

@@ -843,7 +843,7 @@ func testQueryEndpointInfo(t *testing.T, ulPxyEnabled bool) {
 		t.Fatalf("Failed to join the endpoint: %v", err)
 	}
 
-	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep1", sbOptions)
+	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep1", sbOptions, "ep1", "")
 	if err != nil {
 		t.Fatalf("Failed to program external connectivity: %v", err)
 	}
@@ -947,7 +947,7 @@ func TestLinkContainers(t *testing.T) {
 		t.Fatalf("Failed to join the endpoint: %v", err)
 	}
 
-	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep1", sbOptions)
+	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep1", sbOptions, "ep1", "")
 	if err != nil {
 		t.Fatalf("Failed to program external connectivity: %v", err)
 	}
@@ -978,7 +978,7 @@ func TestLinkContainers(t *testing.T) {
 		t.Fatal("Failed to link ep1 and ep2")
 	}
 
-	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep2", sbOptions)
+	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep2", sbOptions, "ep2", "ep2")
 	if err != nil {
 		t.Fatalf("Failed to program external connectivity: %v", err)
 	}
@@ -1017,7 +1017,7 @@ func TestLinkContainers(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep2", sbOptions)
+	err = d.ProgramExternalConnectivity(context.Background(), "net1", "ep2", sbOptions, "ep2", "ep2")
 	assert.Check(t, err != nil, "Expected Join to fail given link conditions are not satisfied")
 	checkLink(false)
 }

libnetwork/drivers/bridge/bridge_store.go

@@ -458,8 +458,22 @@ func (n *bridgeNetwork) restorePortAllocations(ep *bridgeEndpoint) {
 		cfg[i] = b.PortBinding
 	}
 
+	// Calculate a portBindingMode - it need not be accurate but, if there were
+	// IPv4/IPv6 bindings before, ensure they are re-created. (If, for example,
+	// there are no IPv6 bindings, it doesn't matter whether that was because this
+	// endpoint is not an IPv6 gateway and "pbmIPv6" was not set in the port
+	// binding state, or there were just no IPv6 port bindings configured.)
+	var pbm portBindingMode
+	for _, b := range ep.portMapping {
+		if b.HostIP.To4() == nil {
+			pbm.ipv6 = true
+		} else {
+			pbm.ipv4 = true
+		}
+	}
+
 	var err error
-	ep.portMapping, err = n.addPortMappings(context.TODO(), ep.addr, ep.addrv6, cfg, n.config.DefaultBindingIP, ep.extConnConfig.NoProxy6To4)
+	ep.portMapping, err = n.addPortMappings(context.TODO(), ep, cfg, n.config.DefaultBindingIP, pbm)
 	if err != nil {
 		log.G(context.TODO()).Warnf("Failed to reserve existing port mapping for endpoint %.7s:%v", ep.id, err)
 	}

libnetwork/drivers/bridge/port_mapping_linux.go

@@ -80,10 +80,10 @@ var startProxy = portmapper.StartProxy
 // each returned portBinding are set to the selected and reserved port.
 func (n *bridgeNetwork) addPortMappings(
 	ctx context.Context,
-	epAddrV4, epAddrV6 *net.IPNet,
+	ep *bridgeEndpoint,
 	cfg []types.PortBinding,
 	defHostIP net.IP,
-	noProxy6To4 bool,
+	pbmReq portBindingMode,
 ) (_ []portBinding, retErr error) {
 	if len(defHostIP) == 0 {
 		defHostIP = net.IPv4zero
@@ -93,11 +93,11 @@ func (n *bridgeNetwork) addPortMappings(
 	}
 
 	var containerIPv4, containerIPv6 net.IP
-	if epAddrV4 != nil {
-		containerIPv4 = epAddrV4.IP
+	if ep.addr != nil {
+		containerIPv4 = ep.addr.IP
 	}
-	if epAddrV6 != nil {
-		containerIPv6 = epAddrV6.IP
+	if ep.addrv6 != nil {
+		containerIPv6 = ep.addrv6.IP
 	}
 
 	bindings := make([]portBinding, 0, len(cfg)*2)
@@ -117,6 +117,9 @@ func (n *bridgeNetwork) addPortMappings(
 	pdc := n.getPortDriverClient()
 	disableNAT4, disableNAT6 := n.getNATDisabled()
 
+	add4 := !ep.portBindingState.ipv4 && pbmReq.ipv4
+	add6 := !ep.portBindingState.ipv6 && pbmReq.ipv6
+
 	// toBind accumulates port bindings that should be allocated the same host port
 	// (if required by NAT config). If the host address is unspecified, and defHostIP
 	// is 0.0.0.0, one iteration of the loop may generate bindings for v4 and v6. If
@@ -128,15 +131,17 @@ func (n *bridgeNetwork) addPortMappings(
 	// bindings to collect, they're applied and toBind is reset.
 	var toBind []portBindingReq
 	for i, c := range sortedCfg {
-		if bindingIPv4, ok := configurePortBindingIPv4(ctx, pdc, disableNAT4, c, containerIPv4, defHostIP); ok {
-			toBind = append(toBind, bindingIPv4)
+		if add4 {
+			if bindingIPv4, ok := configurePortBindingIPv4(ctx, pdc, disableNAT4, c, containerIPv4, defHostIP); ok {
+				toBind = append(toBind, bindingIPv4)
+			}
 		}
 
 		// If the container has no IPv6 address, allow proxying host IPv6 traffic to it
 		// by setting up the binding with the IPv4 interface if the userland proxy is enabled
 		// This change was added to keep backward compatibility
 		containerIP := containerIPv6
-		if containerIPv6 == nil && !noProxy6To4 {
+		if containerIPv6 == nil && pbmReq.ipv4 && add6 {
 			if proxyPath == "" {
 				// There's no way to map from host-IPv6 to container-IPv4 with the userland proxy
 				// disabled.
@@ -157,8 +162,10 @@ func (n *bridgeNetwork) addPortMappings(
 				containerIP = containerIPv4
 			}
 		}
-		if bindingIPv6, ok := configurePortBindingIPv6(ctx, pdc, disableNAT6, c, containerIP, defHostIP); ok {
-			toBind = append(toBind, bindingIPv6)
+		if add6 {
+			if bindingIPv6, ok := configurePortBindingIPv6(ctx, pdc, disableNAT6, c, containerIP, defHostIP); ok {
+				toBind = append(toBind, bindingIPv6)
+			}
 		}
 
 		if i < len(sortedCfg)-1 && needSamePort(c, sortedCfg[i+1]) {
@@ -754,6 +761,7 @@ func (n *bridgeNetwork) releasePorts(ep *bridgeEndpoint) error {
 	n.Lock()
 	pbs := ep.portMapping
 	ep.portMapping = nil
+	ep.portBindingState = portBindingMode{}
 	n.Unlock()
 
 	return releasePortBindings(pbs, n.firewallerNetwork)

libnetwork/drivers/bridge/port_mapping_linux_test.go

@@ -73,7 +73,7 @@ func TestPortMappingConfig(t *testing.T) {
 		t.Fatalf("Failed to join the endpoint: %v", err)
 	}
 
-	if err = d.ProgramExternalConnectivity(context.Background(), "dummy", "ep1", sbOptions); err != nil {
+	if err = d.ProgramExternalConnectivity(context.Background(), "dummy", "ep1", sbOptions, "ep1", ""); err != nil {
 		t.Fatalf("Failed to program external connectivity: %v", err)
 	}
 
@@ -159,7 +159,7 @@ func TestPortMappingV6Config(t *testing.T) {
 		t.Fatalf("Failed to join the endpoint: %v", err)
 	}
 
-	if err = d.ProgramExternalConnectivity(context.Background(), "dummy", "ep1", sbOptions); err != nil {
+	if err = d.ProgramExternalConnectivity(context.Background(), "dummy", "ep1", sbOptions, "ep1", ""); err != nil {
 		t.Fatalf("Failed to program external connectivity: %v", err)
 	}
 
@@ -876,7 +876,20 @@ func TestAddPortMappings(t *testing.T) {
 				}
 			})
 
-			pbs, err := n.addPortMappings(ctx, tc.epAddrV4, tc.epAddrV6, tc.cfg, tc.defHostIP, tc.noProxy6To4)
+			ep := &bridgeEndpoint{
+				id:     "dummyep",
+				nid:    "dummynetwork",
+				addr:   tc.epAddrV4,
+				addrv6: tc.epAddrV6,
+			}
+			var pbm portBindingMode
+			if ep.addr != nil {
+				pbm.ipv4 = true
+			}
+			if ep.addrv6 != nil || (!tc.noProxy6To4 && ep.addr != nil) {
+				pbm.ipv6 = true
+			}
+			pbs, err := n.addPortMappings(ctx, ep, tc.cfg, tc.defHostIP, pbm)
 			if tc.expErr != "" {
 				assert.ErrorContains(t, err, tc.expErr)
 				return