Merge pull request moby#50106 from corhere/libn/overlay-peerdb-soundness

libnetwork/d/overlay: properly model peer db

Author: robmry

libnetwork/drivers/overlay/encryption.go

@@ -91,8 +91,13 @@ func (s *spi) String() string {
 	return fmt.Sprintf("SPI(FWD: 0x%x, REV: 0x%x)", uint32(s.forward), uint32(s.reverse))
 }
 
+type encrNode struct {
+	spi   []spi
+	count int
+}
+
 type encrMap struct {
-	nodes map[netip.Addr][]*spi
+	nodes map[netip.Addr]encrNode
 	sync.Mutex
 }
 
@@ -105,7 +110,7 @@ func (e *encrMap) String() string {
 		b.WriteString(k.String())
 		b.WriteString(":")
 		b.WriteString("[")
-		for _, s := range v {
+		for _, s := range v.spi {
 			b.WriteString(s.String())
 			b.WriteString(",")
 		}
@@ -126,10 +131,10 @@ func (d *driver) setupEncryption(remoteIP netip.Addr) error {
 	}
 	log.G(context.TODO()).Debugf("Programming encryption between %s and %s", localIP, remoteIP)
 
-	indices := make([]*spi, 0, len(keys))
+	indices := make([]spi, 0, len(keys))
 
 	for i, k := range keys {
-		spis := &spi{buildSPI(advIP.AsSlice(), remoteIP.AsSlice(), k.tag), buildSPI(remoteIP.AsSlice(), advIP.AsSlice(), k.tag)}
+		spis := spi{buildSPI(advIP.AsSlice(), remoteIP.AsSlice(), k.tag), buildSPI(remoteIP.AsSlice(), advIP.AsSlice(), k.tag)}
 		dir := reverse
 		if i == 0 {
 			dir = bidir
@@ -149,7 +154,10 @@ func (d *driver) setupEncryption(remoteIP netip.Addr) error {
 	}
 
 	d.secMap.Lock()
-	d.secMap.nodes[remoteIP] = indices
+	node := d.secMap.nodes[remoteIP]
+	node.spi = indices
+	node.count++
+	d.secMap.nodes[remoteIP] = node
 	d.secMap.Unlock()
 
 	return nil
@@ -158,13 +166,20 @@ func (d *driver) setupEncryption(remoteIP netip.Addr) error {
 func (d *driver) removeEncryption(remoteIP netip.Addr) error {
 	log.G(context.TODO()).Debugf("removeEncryption(%s)", remoteIP)
 
-	d.secMap.Lock()
-	indices, ok := d.secMap.nodes[remoteIP]
-	d.secMap.Unlock()
-	if !ok {
+	spi := func() []spi {
+		d.secMap.Lock()
+		defer d.secMap.Unlock()
+		node := d.secMap.nodes[remoteIP]
+		if node.count == 1 {
+			delete(d.secMap.nodes, remoteIP)
+			return node.spi
+		}
+		node.count--
+		d.secMap.nodes[remoteIP] = node
 		return nil
-	}
-	for i, idxs := range indices {
+	}()
+
+	for i, idxs := range spi {
 		dir := reverse
 		if i == 0 {
 			dir = bidir
@@ -263,7 +278,7 @@ func (d *driver) programInput(vni uint32, add bool) error {
 	return nil
 }
 
-func programSA(localIP, remoteIP net.IP, spi *spi, k *key, dir int, add bool) (fSA *netlink.XfrmState, rSA *netlink.XfrmState, lastErr error) {
+func programSA(localIP, remoteIP net.IP, spi spi, k *key, dir int, add bool) (fSA *netlink.XfrmState, rSA *netlink.XfrmState, lastErr error) {
 	var (
 		action      = "Removing"
 		xfrmProgram = ns.NlHandle().XfrmStateDel
@@ -436,12 +451,12 @@ func buildAeadAlgo(k *key, s int) *netlink.XfrmStateAlgo {
 	}
 }
 
-func (d *driver) secMapWalk(f func(netip.Addr, []*spi) ([]*spi, bool)) error {
+func (d *driver) secMapWalk(f func(netip.Addr, []spi) ([]spi, bool)) error {
 	d.secMap.Lock()
-	for node, indices := range d.secMap.nodes {
-		idxs, stop := f(node, indices)
+	for rIP, node := range d.secMap.nodes {
+		idxs, stop := f(rIP, node.spi)
 		if idxs != nil {
-			d.secMap.nodes[node] = idxs
+			d.secMap.nodes[rIP] = encrNode{idxs, node.count}
 		}
 		if stop {
 			break
@@ -457,7 +472,7 @@ func (d *driver) setKeys(keys []*key) error {
 	// Accept the encryption keys and clear any stale encryption map
 	d.Lock()
 	d.keys = keys
-	d.secMap = &encrMap{nodes: map[netip.Addr][]*spi{}}
+	d.secMap = &encrMap{nodes: map[netip.Addr]encrNode{}}
 	d.Unlock()
 	log.G(context.TODO()).Debugf("Initial encryption keys: %v", keys)
 	return nil
@@ -506,7 +521,7 @@ func (d *driver) updateKeys(newKey, primary, pruneKey *key) error {
 		return types.InvalidParameterErrorf("attempting to both make a key (index %d) primary and delete it", priIdx)
 	}
 
-	d.secMapWalk(func(rIP netip.Addr, spis []*spi) ([]*spi, bool) {
+	d.secMapWalk(func(rIP netip.Addr, spis []spi) ([]spi, bool) {
 		return updateNodeKey(lIP.AsSlice(), aIP.AsSlice(), rIP.AsSlice(), spis, d.keys, newIdx, priIdx, delIdx), false
 	})
 
@@ -534,15 +549,15 @@ func (d *driver) updateKeys(newKey, primary, pruneKey *key) error {
  *********************************************************/
 
 // Spis and keys are sorted in such away the one in position 0 is the primary
-func updateNodeKey(lIP, aIP, rIP net.IP, idxs []*spi, curKeys []*key, newIdx, priIdx, delIdx int) []*spi {
+func updateNodeKey(lIP, aIP, rIP net.IP, idxs []spi, curKeys []*key, newIdx, priIdx, delIdx int) []spi {
 	log.G(context.TODO()).Debugf("Updating keys for node: %s (%d,%d,%d)", rIP, newIdx, priIdx, delIdx)
 
 	spis := idxs
 	log.G(context.TODO()).Debugf("Current: %v", spis)
 
 	// add new
 	if newIdx != -1 {
-		spis = append(spis, &spi{
+		spis = append(spis, spi{
 			forward: buildSPI(aIP, rIP, curKeys[newIdx].tag),
 			reverse: buildSPI(rIP, aIP, curKeys[newIdx].tag),
 		})

libnetwork/drivers/overlay/ov_network.go

@@ -1,4 +1,5 @@
-//go:build linux
+// FIXME(thaJeztah): remove once we are a module; the go:build directive prevents go from downgrading language version to go1.16:
+//go:build go1.23 && linux
 
 package overlay
 
@@ -18,6 +19,7 @@ import (
 	"github.com/docker/docker/internal/nlwrap"
 	"github.com/docker/docker/libnetwork/driverapi"
 	"github.com/docker/docker/libnetwork/drivers/overlay/overlayutils"
+	"github.com/docker/docker/libnetwork/internal/countmap"
 	"github.com/docker/docker/libnetwork/internal/netiputil"
 	"github.com/docker/docker/libnetwork/netlabel"
 	"github.com/docker/docker/libnetwork/ns"
@@ -53,6 +55,8 @@ type network struct {
 	endpoints endpointTable
 	driver    *driver
 	joinCnt   int
+	// Ref count of VXLAN Forwarding Database entries programmed into the kernel
+	fdbCnt    countmap.Map[ipmac]
 	sboxInit  bool
 	initEpoch int
 	initErr   error
@@ -99,6 +103,7 @@ func (d *driver) CreateNetwork(ctx context.Context, id string, option map[string
 		driver:    d,
 		endpoints: endpointTable{},
 		subnets:   []*subnet{},
+		fdbCnt:    countmap.Map[ipmac]{},
 	}
 
 	vnis := make([]uint32, 0, len(ipV4Data))
@@ -586,6 +591,7 @@ func (n *network) initSandbox() error {
 
 	// this is needed to let the peerAdd configure the sandbox
 	n.sbox = sbox
+	n.fdbCnt = countmap.Map[ipmac]{}
 
 	return nil
 }

libnetwork/drivers/overlay/overlay.go

@@ -48,7 +48,7 @@ func Register(r driverapi.Registerer, config map[string]interface{}) error {
 		peerDb: peerNetworkMap{
 			mp: map[string]*peerMap{},
 		},
-		secMap: &encrMap{nodes: map[netip.Addr][]*spi{}},
+		secMap: &encrMap{nodes: map[netip.Addr]encrNode{}},
 		config: config,
 	}
 	return r.RegisterDriver(NetworkType, d, driverapi.Capability{