New gossip_net backend using libp2p (#3988)

This is the revival of the tmp/cmr/net2 branch rebased onto develop. 

Some important user-facing changes:

- No separate discovery/communication/etc ports. One port for all public daemon communications.
- Automatic port forwarding with UPnP. If your local network supports UPnP, there should be no configuration required.
- Local peer discovery. If your local network supports mDNS broadcast, coda daemons will automatically discover each other. This includes several daemons on the same machine- no more building peer lists!
- New libp2p keypairs. These are managed  the same as our key pairs with secret_file. Without configuration, key pairs are ephemeral and will disappear when the daemon restarts. (TODO: should we instead persist  the keypair? does it matter for  non-infrastructure?)

Some important internal changes:

- All daemon-daemon connections are now authenticated and confidential.
- Connections are no longer transient and per-request. Individual requests get multiplexed as their own stream over the one connection between the peers. This is analogous to HTTP/2. Outgoing connections will appear to originate from the libp2p listening port, vs some transient port. 

Outstanding details:

- Trust system needs to get augmented to track Peer.t instead of just an IP. Until then we can't implement ban_notify (#4093, #4096).
- Libp2p has little per-connection structured reporting, some things we currently penalize trust for are not detected (eg opening a libp2p connection without also
opening a coda RPC stream) (#4098).
- New pubsub allows banning senders by peer ID. We currently don't do this but we should ban peerIDs that originated bad info and not just the IP of the whoever relayed it to us (#4096).
- ~~Current pubsub validation flow goes a bit against the libp2p grain, and it's not clear to me that the current behavior will survive [this libp2p PR](libp2p/go-libp2p-kad-dht#388). There's an inline comment near the should_forward_message impl (#4097).~~ done
- Connection limit enforcement (#4095)

Other changes:

- Rips out the last vestiges of old membership, which aren't in use.
- The connection info in envelopes is much more accurate now. We shouldn't start trusting it just yet due to some future vagaries around relaying.
- bump nixpkgs version

Future improvements:

- IPv6. There's a hardcoded IPv4 assumption in the helper around IP filtering. 
- Investigate libp2p autorelay. This should help nodes in restrictive networks achieve better connectivity, but has a host of problems.
- Intelligent request routing. I believe we can use the "provider" feature to, at the very least, only send eg sync/bootstrap requests to nodes who believe themselves to be in sync. There are other options.

Author: emberian

build/DEBIAN/control

@@ -1,11 +1,11 @@
 ((import (builtins.fetchTarball {
-  name = "nixpkgs-unstable-2019-03-18";
-  url = https://github.com/nixos/nixpkgs/archive/0125544e2a0552590c87dca1583768b49ba911c0.tar.gz;
-  sha256 = "04xvlqw3zbq91zkfa506b2k1ajmj7pqh3nvdh9maabw6m5jhm5rl";
+  name = "nixpkgs-stable-2019-12-05";
+  url = https://github.com/nixos/nixpkgs/archive/19.09.tar.gz;
+  sha256 = "0mhqhq21y5vrr1f30qd2bvydv4bbbslvyzclhw0kdxmkgg3z4c92";
   })) {}).buildGoModule rec {
   name = "libp2p_helper-${version}";
   version = "0.1";
   src = ./src;
-  modSha256 = "1spndcx0z50cmpfxfd0971nj9n0v77fghxl36hr1pvs6kv0ra5c3";
+  modSha256 = "0wrqxik9z713w50w49ivy5c2vapk07fdmd0zsvk6kfkchkq1nsdy";
 }
 

default.nix

@@ -24,6 +24,7 @@ import (
 	secio "github.com/libp2p/go-libp2p-secio"
 	p2pconfig "github.com/libp2p/go-libp2p/config"
 	mdns "github.com/libp2p/go-libp2p/p2p/discovery"
+	filters "github.com/libp2p/go-maddr-filter"
 	tcp "github.com/libp2p/go-tcp-transport"
 	ws "github.com/libp2p/go-ws-transport"
 	ma "github.com/multiformats/go-multiaddr"
@@ -38,9 +39,11 @@ type Helper struct {
 	Ctx             context.Context
 	Pubsub          *pubsub.PubSub
 	Logger          logging.EventLogger
+	Filters         *filters.Filters
 	DiscoveredPeers chan peer.AddrInfo
 	Rendezvous      string
 	Discovery       *discovery.RoutingDiscovery
+	Me              peer.ID
 }
 
 type customValidator struct {
@@ -62,6 +65,12 @@ func (cv customValidator) Select(key string, values [][]byte) (int, error) {
 // MakeHelper does all the initialization to run one host
 func MakeHelper(ctx context.Context, listenOn []ma.Multiaddr, externalAddr ma.Multiaddr, statedir string, pk crypto.PrivKey, networkID string) (*Helper, error) {
 	logger := logging.Logger("codanet.Helper")
+
+	me, err := peer.IDFromPrivateKey(pk)
+	if err != nil {
+		return nil, err
+	}
+
 	dso := dsb.DefaultOptions
 
 	ds, err := dsb.NewDatastore(path.Join(statedir, "libp2p-peerstore-v0"), &dso)
@@ -93,6 +102,8 @@ func MakeHelper(ctx context.Context, listenOn []ma.Multiaddr, externalAddr ma.Mu
 	// gross hack to exfiltrate the DHT from the side effect of option evaluation
 	kadch := make(chan *kad.IpfsDHT)
 
+	filters := filters.NewFilters()
+
 	// Make sure this doesn't get too out of sync with the defaults,
 	// NewWithoutDefaults is considered unstable.
 	host, err := p2p.NewWithoutDefaults(ctx,
@@ -108,6 +119,7 @@ func MakeHelper(ctx context.Context, listenOn []ma.Multiaddr, externalAddr ma.Mu
 			as = append(as, externalAddr)
 			return as
 		}),
+		p2p.Filters(filters),
 		p2p.NATPortMap(),
 		p2p.Routing(
 			p2pconfig.RoutingC(func(host host.Host) (routing.PeerRouting, error) {
@@ -138,6 +150,8 @@ func MakeHelper(ctx context.Context, listenOn []ma.Multiaddr, externalAddr ma.Mu
 		Logger:          logger,
 		DiscoveredPeers: nil,
 		Rendezvous:      rendezvousString,
+		Filters:         filters,
 		Discovery:       nil,
+		Me:              me,
 	}, nil
 }

src/codanet.go

@@ -1,27 +1,34 @@
 package main
+
 import (
-        crypto "github.com/libp2p/go-libp2p-crypto"
-        b58 "github.com/mr-tron/base58/base58"
-        "os"
-    )
+	crypto "github.com/libp2p/go-libp2p-crypto"
+	b58 "github.com/mr-tron/base58/base58"
+	"os"
+)
 
 func main() {
-    if len(os.Args) != 2 {
-        println("usage: libp2p-priv-to-pub PRIVKEY_BASE58_STRING");
-    }
-    privk_enc := os.Args[1]
-    privk_raw, err := b58.Decode(privk_enc)
-    if err != nil { panic(err); }
+	if len(os.Args) != 2 {
+		println("usage: libp2p-priv-to-pub PRIVKEY_BASE58_STRING")
+	}
+	privk_enc := os.Args[1]
+	privk_raw, err := b58.Decode(privk_enc)
+	if err != nil {
+		panic(err)
+	}
 
-    priv, err := crypto.UnmarshalPrivateKey(privk_raw)
-    if err != nil { panic(err); }
+	priv, err := crypto.UnmarshalPrivateKey(privk_raw)
+	if err != nil {
+		panic(err)
+	}
 
-    pub := priv.GetPublic()
+	pub := priv.GetPublic()
 
-    pubk_raw, err := crypto.MarshalPublicKey(pub)
-    if err != nil { panic(err); }
+	pubk_raw, err := crypto.MarshalPublicKey(pub)
+	if err != nil {
+		panic(err)
+	}
 
-    pubk_enc := b58.Encode(pubk_raw)
+	pubk_enc := b58.Encode(pubk_raw)
 
-    println(pubk_enc)
+	println(pubk_enc)
 }

src/gen_keys/libp2p_priv_to_pub.go

@@ -99,7 +99,7 @@ func main() {
 		Command:     "generate_methodidx",
 		PackageName: "main",
 		TypesAndValues: map[string][]string{
-			"methodIdx": []string{"configure", "listen", "publish", "subscribe", "unsubscribe", "validationComplete", "generateKeypair", "openStream", "closeStream", "resetStream", "sendStreamMsg", "removeStreamHandler", "addStreamHandler", "listeningAddrs", "addPeer", "beginAdvertising"},
+			"methodIdx": []string{"configure", "listen", "publish", "subscribe", "unsubscribe", "validationComplete", "generateKeypair", "openStream", "closeStream", "resetStream", "sendStreamMsg", "removeStreamHandler", "addStreamHandler", "listeningAddrs", "addPeer", "beginAdvertising", "findPeer", "listPeers", "banIP", "unbanIP"},
 		},
 	}
 

src/generate_methodidx/main.go

@@ -4,72 +4,59 @@ go 1.12
 
 require (
 	cloud.google.com/go v0.43.0 // indirect
-	github.com/Kubuxu/go-os-helper v0.0.1 // indirect
-	github.com/btcsuite/btcd v0.0.0-20190629003639-c26ffa870fd8 // indirect
 	github.com/btcsuite/goleveldb v1.0.0 // indirect
 	github.com/campoy/jsonenums v0.0.0-20180221195324-eec6d38da64e
 	github.com/coreos/bbolt v1.3.3 // indirect
 	github.com/coreos/etcd v3.3.13+incompatible // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/davidlazar/go-crypto v0.0.0-20190522120613-62389b5e4ae0 // indirect
-	github.com/dgraph-io/badger v1.6.0 // indirect
 	github.com/go-errors/errors v1.0.1
 	github.com/go-kit/kit v0.9.0 // indirect
-	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
 	github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.9.5 // indirect
 	github.com/hashicorp/go-multierror v1.0.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.3 // indirect
-	github.com/ipfs/go-cid v0.0.3 // indirect
-	github.com/ipfs/go-ds-badger v0.0.5
-	github.com/ipfs/go-ds-leveldb v0.0.2 // indirect
+	github.com/ipfs/go-ds-badger v0.0.7
 	github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
 	github.com/ipfs/go-log v0.0.1
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
-	github.com/kisielk/errcheck v1.2.0 // indirect
 	github.com/kkdai/bstream v1.0.0 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pty v1.1.8 // indirect
+	github.com/libp2p/go-buffer-pool v0.0.2
 	github.com/libp2p/go-conn-security v0.1.0 // indirect
-	github.com/libp2p/go-eventbus v0.0.3 // indirect
-	github.com/libp2p/go-libp2p v0.2.1
-	github.com/libp2p/go-libp2p-circuit v0.1.1 // indirect
-	github.com/libp2p/go-libp2p-core v0.2.0
+	github.com/libp2p/go-libp2p v0.4.2
+	github.com/libp2p/go-libp2p-core v0.2.4
 	github.com/libp2p/go-libp2p-crypto v0.1.0
-	github.com/libp2p/go-libp2p-discovery v0.1.0
+	github.com/libp2p/go-libp2p-discovery v0.2.0
 	github.com/libp2p/go-libp2p-host v0.1.0
 	github.com/libp2p/go-libp2p-interface-connmgr v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-interface-pnet v0.1.0 // indirect
-	github.com/libp2p/go-libp2p-kad-dht v0.1.1
+	github.com/libp2p/go-libp2p-kad-dht v0.3.0
 	github.com/libp2p/go-libp2p-metrics v0.1.0 // indirect
 	github.com/libp2p/go-libp2p-net v0.1.0
 	github.com/libp2p/go-libp2p-peer v0.2.0
-	github.com/libp2p/go-libp2p-peerstore v0.1.3
+	github.com/libp2p/go-libp2p-peerstore v0.1.4
 	github.com/libp2p/go-libp2p-pnet v0.1.0
 	github.com/libp2p/go-libp2p-protocol v0.1.0
-	github.com/libp2p/go-libp2p-pubsub v0.1.0
+	github.com/libp2p/go-libp2p-pubsub v0.2.3
 	github.com/libp2p/go-libp2p-record v0.1.1
 	github.com/libp2p/go-libp2p-routing v0.1.0
-	github.com/libp2p/go-libp2p-secio v0.1.1
-	github.com/libp2p/go-libp2p-testing v0.1.0 // indirect
+	github.com/libp2p/go-libp2p-secio v0.2.1
 	github.com/libp2p/go-libp2p-transport v0.1.0 // indirect
+	github.com/libp2p/go-maddr-filter v0.0.5
 	github.com/libp2p/go-mplex v0.1.0
 	github.com/libp2p/go-stream-muxer v0.1.0
-	github.com/libp2p/go-tcp-transport v0.1.0
+	github.com/libp2p/go-tcp-transport v0.1.1
 	github.com/libp2p/go-testutil v0.1.0 // indirect
-	github.com/libp2p/go-ws-transport v0.1.0
+	github.com/libp2p/go-ws-transport v0.1.2
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/miekg/dns v1.1.15 // indirect
-	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/mr-tron/base58 v1.1.2
-	github.com/multiformats/go-multiaddr v0.0.4
-	github.com/multiformats/go-multiaddr-dns v0.0.3 // indirect
+	github.com/multiformats/go-multiaddr v0.1.1
 	github.com/multiformats/go-multicodec v0.1.6 // indirect
-	github.com/multiformats/go-multihash v0.0.6 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pelletier/go-toml v1.4.0 // indirect
@@ -89,13 +76,12 @@ require (
 	github.com/whyrusleeping/go-smux-yamux v2.0.9+incompatible // indirect
 	github.com/whyrusleeping/yamux v1.2.0 // indirect
 	go.etcd.io/bbolt v1.3.3 // indirect
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
+	golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550
 	golang.org/x/exp v0.0.0-20190731235908-ec7cb31e5a56 // indirect
 	golang.org/x/image v0.0.0-20190802002840-cff245a6509b // indirect
 	golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 // indirect
 	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
 	golang.org/x/tools v0.0.0-20190802220118-1d1727260058 // indirect
-	golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 // indirect
 	google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64 // indirect
 	google.golang.org/grpc v1.22.1 // indirect
 	gopkg.in/src-d/go-cli.v0 v0.0.0-20190422143124-3a646154da79 // indirect