feat: multiple underlay addresses (#5204)

Co-authored-by: Janoš Guljaš <[email protected]>
Co-authored-by: Ljubisa Gacevic <[email protected]>
Co-authored-by: Calin Martinconi <[email protected]>

Author: 4 people

go.mod

@@ -32,6 +32,7 @@ require (
 	github.com/multiformats/go-multiaddr-dns v0.3.1
 	github.com/multiformats/go-multihash v0.2.3
 	github.com/multiformats/go-multistream v0.5.0
+	github.com/multiformats/go-varint v0.0.7
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/prometheus/client_golang v1.21.1
 	github.com/spf13/afero v1.6.0
@@ -130,7 +131,6 @@ require (
 	github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
 	github.com/multiformats/go-multibase v0.2.0 // indirect
 	github.com/multiformats/go-multicodec v0.9.0 // indirect
-	github.com/multiformats/go-varint v0.0.7 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/onsi/ginkgo/v2 v2.15.0 // indirect
 	github.com/opencontainers/runtime-spec v1.2.0 // indirect

pkg/addressbook/addressbook_test.go

@@ -47,7 +47,7 @@ func run(t *testing.T, f bookFunc) {
 		t.Fatal(err)
 	}
 
-	bzzAddr, err := bzz.NewAddress(crypto.NewDefaultSigner(pk), multiaddr, addr1, 1, trxHash)
+	bzzAddr, err := bzz.NewAddress(crypto.NewDefaultSigner(pk), []ma.Multiaddr{multiaddr}, addr1, 1, trxHash)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/api/peer.go

@@ -31,7 +31,8 @@ func (s *Service) peerConnectHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	bzzAddr, err := s.p2p.Connect(r.Context(), paths.MultiAddress)
+	bzzAddr, err := s.p2p.Connect(r.Context(), []multiaddr.Multiaddr{paths.MultiAddress})
+
 	if err != nil {
 		logger.Debug("p2p connect failed", "addresses", paths.MultiAddress, "error", err)
 		logger.Error(nil, "p2p connect failed", "addresses", paths.MultiAddress)

pkg/api/peer_test.go

@@ -45,15 +45,17 @@ func TestConnect(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	bzzAddress, err := bzz.NewAddress(crypto.NewDefaultSigner(privateKey), underlama, overlay, 0, nil)
+	bzzAddress, err := bzz.NewAddress(crypto.NewDefaultSigner(privateKey), []ma.Multiaddr{underlama}, overlay, 0, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	testServer, _, _, _ := newTestServer(t, testServerOptions{
-		P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addr ma.Multiaddr) (*bzz.Address, error) {
-			if addr.String() == errorUnderlay {
-				return nil, testErr
+		P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) {
+			for _, addr := range addrs {
+				if addr.String() == errorUnderlay {
+					return nil, testErr
+				}
 			}
 			return bzzAddress, nil
 		})),
@@ -81,9 +83,11 @@ func TestConnect(t *testing.T) {
 	t.Run("error - add peer", func(t *testing.T) {
 		t.Parallel()
 		testServer, _, _, _ := newTestServer(t, testServerOptions{
-			P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addr ma.Multiaddr) (*bzz.Address, error) {
-				if addr.String() == errorUnderlay {
-					return nil, testErr
+			P2P: mock.New(mock.WithConnectFunc(func(ctx context.Context, addrs []ma.Multiaddr) (*bzz.Address, error) {
+				for _, addr := range addrs {
+					if addr.String() == errorUnderlay {
+						return nil, testErr
+					}
 				}
 				return bzzAddress, nil
 			})),

pkg/bzz/address.go

@@ -14,12 +14,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"slices"
+	"sort"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethersphere/bee/v2/pkg/crypto"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
 
 	ma "github.com/multiformats/go-multiaddr"
+	manet "github.com/multiformats/go-multiaddr/net"
 )
 
 var ErrInvalidAddress = errors.New("invalid address")
@@ -28,33 +31,31 @@ var ErrInvalidAddress = errors.New("invalid address")
 // It consists of a peers underlay (physical) address, overlay (topology) address and signature.
 // Signature is used to verify the `Overlay/Underlay` pair, as it is based on `underlay|networkID`, signed with the public key of Overlay address
 type Address struct {
-	Underlay        ma.Multiaddr
+	Underlays       []ma.Multiaddr
 	Overlay         swarm.Address
 	Signature       []byte
 	Nonce           []byte
 	EthereumAddress []byte
 }
 
 type addressJSON struct {
-	Overlay   string `json:"overlay"`
-	Underlay  string `json:"underlay"`
-	Signature string `json:"signature"`
-	Nonce     string `json:"transaction"`
+	Overlay   string   `json:"overlay"`
+	Underlay  string   `json:"underlay"` // For backward compatibility
+	Underlays []string `json:"underlays"`
+	Signature string   `json:"signature"`
+	Nonce     string   `json:"transaction"`
 }
 
-func NewAddress(signer crypto.Signer, underlay ma.Multiaddr, overlay swarm.Address, networkID uint64, nonce []byte) (*Address, error) {
-	underlayBinary, err := underlay.MarshalBinary()
-	if err != nil {
-		return nil, err
-	}
+func NewAddress(signer crypto.Signer, underlays []ma.Multiaddr, overlay swarm.Address, networkID uint64, nonce []byte) (*Address, error) {
+	underlaysBinary := SerializeUnderlays(underlays)
 
-	signature, err := signer.Sign(generateSignData(underlayBinary, overlay.Bytes(), networkID))
+	signature, err := signer.Sign(generateSignData(underlaysBinary, overlay.Bytes(), networkID))
 	if err != nil {
 		return nil, err
 	}
 
 	return &Address{
-		Underlay:  underlay,
+		Underlays: underlays,
 		Overlay:   overlay,
 		Signature: signature,
 		Nonce:     nonce,
@@ -77,8 +78,13 @@ func ParseAddress(underlay, overlay, signature, nonce []byte, validateOverlay bo
 		}
 	}
 
-	multiUnderlay, err := ma.NewMultiaddrBytes(underlay)
+	multiUnderlays, err := DeserializeUnderlays(underlay)
 	if err != nil {
+		return nil, fmt.Errorf("deserialize underlays: %w: %w", ErrInvalidAddress, err)
+	}
+
+	if len(multiUnderlays) == 0 {
+		// no underlays sent
 		return nil, ErrInvalidAddress
 	}
 
@@ -88,7 +94,7 @@ func ParseAddress(underlay, overlay, signature, nonce []byte, validateOverlay bo
 	}
 
 	return &Address{
-		Underlay:        multiUnderlay,
+		Underlays:       multiUnderlays,
 		Overlay:         swarm.NewAddress(overlay),
 		Signature:       signature,
 		Nonce:           nonce,
@@ -109,21 +115,49 @@ func (a *Address) Equal(b *Address) bool {
 		return a == b
 	}
 
-	return a.Overlay.Equal(b.Overlay) && multiaddrEqual(a.Underlay, b.Underlay) && bytes.Equal(a.Signature, b.Signature) && bytes.Equal(a.Nonce, b.Nonce)
+	return a.Overlay.Equal(b.Overlay) && AreUnderlaysEqual(a.Underlays, b.Underlays) && bytes.Equal(a.Signature, b.Signature) && bytes.Equal(a.Nonce, b.Nonce)
 }
 
-func multiaddrEqual(a, b ma.Multiaddr) bool {
-	if a == nil || b == nil {
-		return a == b
+func AreUnderlaysEqual(a, b []ma.Multiaddr) bool {
+	if len(a) != len(b) {
+		return false
 	}
 
-	return a.Equal(b)
+	used := make([]bool, len(b))
+	for i := range len(a) {
+		found := false
+		for j := range len(b) {
+			if used[j] {
+				continue
+			}
+			if a[i].Equal(b[j]) {
+				used[j] = true
+				found = true
+				break
+			}
+		}
+		if !found {
+			return false
+		}
+	}
+	return true
 }
 
 func (a *Address) MarshalJSON() ([]byte, error) {
+	if len(a.Underlays) == 0 {
+		return nil, fmt.Errorf("no underlays for %s", a.Overlay)
+	}
+
+	// select the underlay address for backward compatibility
+	var underlay string
+	if v := SelectBestAdvertisedAddress(a.Underlays, nil); v != nil {
+		underlay = v.String()
+	}
+
 	return json.Marshal(&addressJSON{
 		Overlay:   a.Overlay.String(),
-		Underlay:  a.Underlay.String(),
+		Underlay:  underlay,
+		Underlays: a.underlaysAsStrings(),
 		Signature: base64.StdEncoding.EncodeToString(a.Signature),
 		Nonce:     common.Bytes2Hex(a.Nonce),
 	})
@@ -143,23 +177,80 @@ func (a *Address) UnmarshalJSON(b []byte) error {
 
 	a.Overlay = addr
 
-	m, err := ma.NewMultiaddr(v.Underlay)
+	// append the underlay for backward compatibility
+	if !slices.Contains(v.Underlays, v.Underlay) {
+		v.Underlays = append(v.Underlays, v.Underlay)
+	}
+
+	multiaddrs, err := parseMultiaddrs(v.Underlays)
 	if err != nil {
 		return err
 	}
 
-	a.Underlay = m
+	a.Underlays = multiaddrs
 	a.Signature, err = base64.StdEncoding.DecodeString(v.Signature)
 	a.Nonce = common.Hex2Bytes(v.Nonce)
 	return err
 }
 
 func (a *Address) String() string {
-	return fmt.Sprintf("[Underlay: %v, Overlay %v, Signature %x, Transaction %x]", a.Underlay, a.Overlay, a.Signature, a.Nonce)
+	return fmt.Sprintf("[Underlay: %v, Overlay %v, Signature %x, Transaction %x]", a.underlaysAsStrings(), a.Overlay, a.Signature, a.Nonce)
 }
 
 // ShortString returns shortened versions of bzz address in a format: [Overlay, Underlay]
 // It can be used for logging
 func (a *Address) ShortString() string {
-	return fmt.Sprintf("[Overlay: %s, Underlay: %s]", a.Overlay.String(), a.Underlay.String())
+	return fmt.Sprintf("[Overlay: %s, Underlays: %v]", a.Overlay.String(), a.underlaysAsStrings())
+}
+
+func (a *Address) underlaysAsStrings() []string {
+	underlays := make([]string, len(a.Underlays))
+	for i, underlay := range a.Underlays {
+		underlays[i] = underlay.String()
+	}
+	return underlays
+}
+
+func parseMultiaddrs(addrs []string) ([]ma.Multiaddr, error) {
+	multiAddrs := make([]ma.Multiaddr, len(addrs))
+	for i, addr := range addrs {
+		multiAddr, err := ma.NewMultiaddr(addr)
+		if err != nil {
+			return nil, err
+		}
+		multiAddrs[i] = multiAddr
+	}
+	return multiAddrs, nil
+}
+
+func SelectBestAdvertisedAddress(addrs []ma.Multiaddr, fallback ma.Multiaddr) ma.Multiaddr {
+	if len(addrs) == 0 {
+		return fallback
+	}
+
+	hasTCPProtocol := func(addr ma.Multiaddr) bool {
+		_, err := addr.ValueForProtocol(ma.P_TCP)
+		return err == nil
+	}
+
+	// Sort addresses to prioritize TCP over other protocols
+	sort.SliceStable(addrs, func(i, j int) bool {
+		iTCP := hasTCPProtocol(addrs[i])
+		jTCP := hasTCPProtocol(addrs[j])
+		return iTCP && !jTCP
+	})
+
+	for _, addr := range addrs {
+		if manet.IsPublicAddr(addr) {
+			return addr
+		}
+	}
+
+	for _, addr := range addrs {
+		if !manet.IsPrivateAddr(addr) {
+			return addr
+		}
+	}
+
+	return addrs[0]
 }