feat: periodically fill the routing table with KADIds (#215)

In order to make queries in fewer hops it's useful to fill the routing
table with imaginary peers that have IDs that are uniformly distributed
over the sha2-256 value space.  This means we've always got a peer
ID on hand that's kind of xor-close to anything we might be looking up.

We can then issue period DHT queries for any actual existant peers that
are close to our imaginary peers to ensure we've got a good range of
peers we know about, so making queries requires fewer hops to locate
ones that are xor-close to the data we are interested in.

BREAKING CHANGE: .start() is now async and random walk has been removed

Author: achingbrain

.aegir.js

@@ -1,5 +1,33 @@
+'use strict'
+
+/** @type {import('aegir').Options["build"]["config"]} */
+const esbuild = {
+  plugins: [
+    {
+      name: 'node built ins',
+      setup (build) {
+        build.onResolve({ filter: /^stream$/ }, () => {
+          return { path: require.resolve('readable-stream') }
+        })
+        build.onResolve({ filter: /^crypto$/ }, () => {
+          return { path: require.resolve('crypto-browserify') }
+        })
+      }
+    }
+  ]
+}
+
+/** @type {import('aegir').PartialOptions} */
 module.exports = {
+  test: {
+    browser: {
+      config: {
+        buildConfig: esbuild
+      }
+    }
+  },
   build: {
-    bundlesizeMax: '182kB'
+    bundlesizeMax: '499KB',
+    config: esbuild
   }
 }

README.md

@@ -72,17 +72,15 @@ function addDHT(libp2p) {
 Note that you may want to supply your own peer discovery function and datastore
 ### Peer Routing
 
-[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/peer-routing)
+[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/packages/interfaces/src/peer-routing)
 
 ### Content Routing
 
-[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/content-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/content-routing)
+[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/content-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/packages/interfaces/src/content-routing)
 
 ### Peer Discovery
 
-[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-discovery/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/peer-discovery)
-
-`libp2p-kad-dht` provides a discovery service called `Random Walk` (random walks on the DHT to discover more nodes). It is accessible through `dht.randomWalk` and exposes the [Peer Discovery interface](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/peer-discovery).
+[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-discovery/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/packages/interfaces/src/peer-discovery)
 
 ### Implementation Summary
 

package.json

@@ -47,13 +47,13 @@
   "homepage": "https://github.com/libp2p/js-libp2p-kad-dht",
   "types": "dist/src/index.d.ts",
   "dependencies": {
-    "abort-controller": "^3.0.0",
     "debug": "^4.3.1",
     "err-code": "^3.0.0",
     "hashlru": "^2.3.0",
     "heap": "~0.2.6",
     "interface-datastore": "^5.1.1",
     "it-first": "^1.0.4",
+    "it-length": "^1.0.3",
     "it-length-prefixed": "^5.0.2",
     "it-pipe": "^1.1.0",
     "k-bucket": "^5.1.0",
@@ -65,21 +65,20 @@
     "p-map": "^4.0.0",
     "p-queue": "^6.6.2",
     "p-timeout": "^4.1.0",
-    "p-times": "^3.0.0",
     "peer-id": "^0.15.0",
     "protobufjs": "^6.10.2",
     "streaming-iterables": "^6.0.0",
     "uint8arrays": "^3.0.0",
-    "varint": "^6.0.0",
-    "xor-distance": "^2.0.0"
+    "varint": "^6.0.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
-    "@types/node": "^16.6.1",
     "aegir": "^35.0.1",
     "async-iterator-all": "^1.0.0",
+    "crypto-browserify": "^3.12.0",
     "datastore-level": "^6.0.2",
     "delay": "^5.0.0",
+    "execa": "^5.1.1",
     "it-pair": "^1.0.0",
     "libp2p": "^0.32.3",
     "lodash.random": "^3.2.0",
@@ -88,7 +87,8 @@
     "p-each-series": "^2.1.0",
     "p-map-series": "^2.1.0",
     "p-retry": "^4.2.0",
-    "sinon": "^11.1.1"
+    "sinon": "^11.1.1",
+    "which": "^2.0.2"
   },
   "contributors": [
     "Vasco Santos <[email protected]>",
@@ -110,5 +110,8 @@
     "Pedro Teixeira <[email protected]>",
     "Qmstream <[email protected]>",
     "Thomas Eizinger <[email protected]>"
-  ]
+  ],
+  "browser": {
+    "src/routing-table/generated-prefix-list.json": "src/routing-table/generated-prefix-list-browser.json"
+  }
 }

src/constants.js

@@ -33,11 +33,3 @@ exports.K = 20
 
 // Alpha is the concurrency for asynchronous requests
 exports.ALPHA = 3
-
-exports.defaultRandomWalk = {
-  enabled: true,
-  queriesPerPeriod: 1,
-  interval: 5 * minute,
-  timeout: 10 * second,
-  delay: 10 * second
-}

src/index.js

@@ -8,7 +8,7 @@ const { MemoryDatastore } = require('interface-datastore')
 const { equals: uint8ArrayEquals } = require('uint8arrays/equals')
 const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
 
-const RoutingTable = require('./routing')
+const RoutingTable = require('./routing-table')
 const utils = require('./utils')
 const c = require('./constants')
 const Network = require('./network')
@@ -17,7 +17,6 @@ const contentRouting = require('./content-routing')
 const peerRouting = require('./peer-routing')
 const Message = require('./message')
 const Providers = require('./providers')
-const RandomWalk = require('./random-walk')
 const QueryManager = require('./query-manager')
 
 const Record = libp2pRecord.Record
@@ -34,13 +33,6 @@ const Record = libp2pRecord.Record
  * @typedef {object} PeerData
  * @property {PeerId} id
  * @property {Multiaddr[]} multiaddrs
- *
- * @typedef {object} RandomWalkOptions
- * @property {boolean} enabled discovery enabled (default: true)
- * @property {number} queriesPerPeriod how many queries to run per period (default: 1)
- * @property {number} interval how often to run the the random-walk process, in milliseconds (default: 300000)
- * @property {number} timeout how long to wait for the the random-walk query to run, in milliseconds (default: 30000)
- * @property {number} delay how long to wait before starting the first random walk, in milliseconds (default: 10000)
  */
 
 /**
@@ -65,7 +57,6 @@ class KadDHT extends EventEmitter {
    * @param {Datastore} props.datastore - datastore (default MemoryDatastore)
    * @param {object} props.validators - validators object with namespace as keys and function(key, record, callback)
    * @param {object} props.selectors - selectors object with namespace as keys and function(key, records)
-   * @param {RandomWalkOptions} props.randomWalk - randomWalk options
    * @param {function(import('libp2p-record').Record, PeerId): void} [props.onPut] - Called when an entry is added to or changed in the datastore
    * @param {function(import('libp2p-record').Record): void} [props.onRemove] - Called when an entry is removed from the datastore
    */
@@ -83,13 +74,6 @@ class KadDHT extends EventEmitter {
     concurrency = c.ALPHA,
     validators = {},
     selectors = {},
-    randomWalk = {
-      enabled: false,
-      queriesPerPeriod: 1,
-      interval: 300000,
-      timeout: 30000,
-      delay: 10000
-    },
     onPut = () => {},
     onRemove = () => {}
   }) {
@@ -170,7 +154,7 @@ class KadDHT extends EventEmitter {
      *
      * @type {RoutingTable}
      */
-    this.routingTable = new RoutingTable(this.peerId, this.kBucketSize)
+    this.routingTable = new RoutingTable(this, { kBucketSize: this.kBucketSize })
 
     /**
      * Reference to the datastore, uses an in-memory store if none given.
@@ -200,13 +184,6 @@ class KadDHT extends EventEmitter {
 
     this._log = utils.logger(this.peerId)
 
-    /**
-     * Random walk management
-     *
-     * @type {RandomWalk}
-     */
-    this.randomWalk = new RandomWalk(this, randomWalk)
-
     /**
      * Keeps track of running queries
      *
@@ -237,17 +214,14 @@ class KadDHT extends EventEmitter {
    * Start listening to incoming connections.
    */
   start () {
-    if (this._running) {
-      return
-    }
-
     this._running = true
-    this.providers.start()
-    this._queryManager.start()
-    this.network.start()
 
-    // Start random walk, it will not run if it's disabled
-    this.randomWalk.start()
+    return Promise.all([
+      this.providers.start(),
+      this._queryManager.start(),
+      this.network.start(),
+      this.routingTable.start()
+    ])
   }
 
   /**
@@ -256,10 +230,13 @@ class KadDHT extends EventEmitter {
    */
   stop () {
     this._running = false
-    this.randomWalk.stop()
-    this.network.stop()
-    this._queryManager.stop()
-    this.providers.stop()
+
+    return Promise.all([
+      this.providers.stop(),
+      this._queryManager.stop(),
+      this.network.stop(),
+      this.routingTable.stop()
+    ])
   }
 
   /**

src/peer-list/peer-distance-list.js

@@ -1,10 +1,10 @@
 'use strict'
 
-// @ts-ignore
-const distance = require('xor-distance')
 const utils = require('../utils')
 const pMap = require('p-map')
 const { equals: uint8ArrayEquals } = require('uint8arrays/equals')
+const { compare: uint8ArrayCompare } = require('uint8arrays/compare')
+const { xor: uint8ArrayXor } = require('uint8arrays/xor')
 
 /**
  * @typedef {import('peer-id')} PeerId
@@ -56,11 +56,11 @@ class PeerDistanceList {
     const dhtKey = await utils.convertPeerId(peerId)
     const el = {
       peerId,
-      distance: distance(this.originDhtKey, dhtKey)
+      distance: uint8ArrayXor(this.originDhtKey, dhtKey)
     }
 
     this.peerDistances.push(el)
-    this.peerDistances.sort((a, b) => distance.compare(a.distance, b.distance))
+    this.peerDistances.sort((a, b) => uint8ArrayCompare(a.distance, b.distance))
     this.peerDistances = this.peerDistances.slice(0, this.capacity)
   }
 
@@ -83,9 +83,9 @@ class PeerDistanceList {
     const furthestDistance = this.peerDistances[this.peerDistances.length - 1].distance
 
     for (const dhtKey of dhtKeys) {
-      const keyDistance = distance(this.originDhtKey, dhtKey)
+      const keyDistance = uint8ArrayXor(this.originDhtKey, dhtKey)
 
-      if (distance.compare(keyDistance, furthestDistance) < 0) {
+      if (uint8ArrayCompare(keyDistance, furthestDistance) < 0) {
         return true
       }
     }

src/peer-list/peer-queue.js

@@ -2,8 +2,7 @@
 
 // @ts-ignore
 const Heap = require('heap')
-// @ts-ignore
-const distance = require('xor-distance')
+const { xor: uint8ArrayXor } = require('uint8arrays/xor')
 const debug = require('debug')
 
 const utils = require('../utils')
@@ -65,7 +64,7 @@ class PeerQueue {
 
     const el = {
       id: id,
-      distance: distance(this.from, key)
+      distance: uint8ArrayXor(this.from, key)
     }
 
     this.heap.push(el)

src/providers.js

@@ -68,6 +68,12 @@ class Providers {
    * Start the provider cleanup service
    */
   start () {
+    if (this._started) {
+      return
+    }
+
+    this._started = true
+
     this._cleaner = setInterval(
       () => this._cleanup(),
       this.cleanupInterval
@@ -78,6 +84,8 @@ class Providers {
    * Release any resources.
    */
   stop () {
+    this._started = false
+
     if (this._cleaner) {
       clearInterval(this._cleaner)
       this._cleaner = null