Revert "feat: periodically fill the routing table with KADIds (#215)"

This reverts commit 10f0cc8.

Author: vasco-santos

.aegir.js

@@ -1,33 +1,5 @@
-'use strict'
-
-/** @type {import('aegir').Options["build"]["config"]} */
-const esbuild = {
-  plugins: [
-    {
-      name: 'node built ins',
-      setup (build) {
-        build.onResolve({ filter: /^stream$/ }, () => {
-          return { path: require.resolve('readable-stream') }
-        })
-        build.onResolve({ filter: /^crypto$/ }, () => {
-          return { path: require.resolve('crypto-browserify') }
-        })
-      }
-    }
-  ]
-}
-
-/** @type {import('aegir').PartialOptions} */
 module.exports = {
-  test: {
-    browser: {
-      config: {
-        buildConfig: esbuild
-      }
-    }
-  },
   build: {
-    bundlesizeMax: '499KB',
-    config: esbuild
+    bundlesizeMax: '182kB'
   }
 }

README.md

@@ -72,15 +72,17 @@ function addDHT(libp2p) {
 Note that you may want to supply your own peer discovery function and datastore
 ### Peer Routing
 
-[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/packages/interfaces/src/peer-routing)
+[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/peer-routing)
 
 ### Content Routing
 
-[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/content-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/packages/interfaces/src/content-routing)
+[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/content-routing/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/content-routing)
 
 ### Peer Discovery
 
-[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-discovery/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/packages/interfaces/src/peer-discovery)
+[![](https://raw.githubusercontent.com/libp2p/js-libp2p-interfaces/master/src/peer-discovery/img/badge.png)](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/peer-discovery)
+
+`libp2p-kad-dht` provides a discovery service called `Random Walk` (random walks on the DHT to discover more nodes). It is accessible through `dht.randomWalk` and exposes the [Peer Discovery interface](https://github.com/libp2p/js-libp2p-interfaces/tree/master/src/peer-discovery).
 
 ### Implementation Summary
 

package.json

@@ -47,13 +47,13 @@
   "homepage": "https://github.com/libp2p/js-libp2p-kad-dht",
   "types": "dist/src/index.d.ts",
   "dependencies": {
+    "abort-controller": "^3.0.0",
     "debug": "^4.3.1",
     "err-code": "^3.0.0",
     "hashlru": "^2.3.0",
     "heap": "~0.2.6",
     "interface-datastore": "^5.1.1",
     "it-first": "^1.0.4",
-    "it-length": "^1.0.3",
     "it-length-prefixed": "^5.0.2",
     "it-pipe": "^1.1.0",
     "k-bucket": "^5.1.0",
@@ -65,20 +65,21 @@
     "p-map": "^4.0.0",
     "p-queue": "^6.6.2",
     "p-timeout": "^4.1.0",
+    "p-times": "^3.0.0",
     "peer-id": "^0.15.0",
     "protobufjs": "^6.10.2",
     "streaming-iterables": "^6.0.0",
     "uint8arrays": "^3.0.0",
-    "varint": "^6.0.0"
+    "varint": "^6.0.0",
+    "xor-distance": "^2.0.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
+    "@types/node": "^16.6.1",
     "aegir": "^35.0.1",
     "async-iterator-all": "^1.0.0",
-    "crypto-browserify": "^3.12.0",
     "datastore-level": "^6.0.2",
     "delay": "^5.0.0",
-    "execa": "^5.1.1",
     "it-pair": "^1.0.0",
     "libp2p": "^0.32.3",
     "lodash.random": "^3.2.0",
@@ -87,8 +88,7 @@
     "p-each-series": "^2.1.0",
     "p-map-series": "^2.1.0",
     "p-retry": "^4.2.0",
-    "sinon": "^11.1.1",
-    "which": "^2.0.2"
+    "sinon": "^11.1.1"
   },
   "contributors": [
     "Vasco Santos <[email protected]>",
@@ -110,8 +110,5 @@
     "Pedro Teixeira <[email protected]>",
     "Qmstream <[email protected]>",
     "Thomas Eizinger <[email protected]>"
-  ],
-  "browser": {
-    "src/routing-table/generated-prefix-list.json": "src/routing-table/generated-prefix-list-browser.json"
-  }
+  ]
 }

src/constants.js

@@ -33,3 +33,11 @@ exports.K = 20
 
 // Alpha is the concurrency for asynchronous requests
 exports.ALPHA = 3
+
+exports.defaultRandomWalk = {
+  enabled: true,
+  queriesPerPeriod: 1,
+  interval: 5 * minute,
+  timeout: 10 * second,
+  delay: 10 * second
+}

src/index.js

@@ -8,7 +8,7 @@ const { MemoryDatastore } = require('interface-datastore')
 const { equals: uint8ArrayEquals } = require('uint8arrays/equals')
 const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
 
-const RoutingTable = require('./routing-table')
+const RoutingTable = require('./routing')
 const utils = require('./utils')
 const c = require('./constants')
 const Network = require('./network')
@@ -17,6 +17,7 @@ const contentRouting = require('./content-routing')
 const peerRouting = require('./peer-routing')
 const Message = require('./message')
 const Providers = require('./providers')
+const RandomWalk = require('./random-walk')
 const QueryManager = require('./query-manager')
 
 const Record = libp2pRecord.Record
@@ -33,6 +34,13 @@ const Record = libp2pRecord.Record
  * @typedef {object} PeerData
  * @property {PeerId} id
  * @property {Multiaddr[]} multiaddrs
+ *
+ * @typedef {object} RandomWalkOptions
+ * @property {boolean} enabled discovery enabled (default: true)
+ * @property {number} queriesPerPeriod how many queries to run per period (default: 1)
+ * @property {number} interval how often to run the the random-walk process, in milliseconds (default: 300000)
+ * @property {number} timeout how long to wait for the the random-walk query to run, in milliseconds (default: 30000)
+ * @property {number} delay how long to wait before starting the first random walk, in milliseconds (default: 10000)
  */
 
 /**
@@ -57,6 +65,7 @@ class KadDHT extends EventEmitter {
    * @param {Datastore} props.datastore - datastore (default MemoryDatastore)
    * @param {object} props.validators - validators object with namespace as keys and function(key, record, callback)
    * @param {object} props.selectors - selectors object with namespace as keys and function(key, records)
+   * @param {RandomWalkOptions} props.randomWalk - randomWalk options
    * @param {function(import('libp2p-record').Record, PeerId): void} [props.onPut] - Called when an entry is added to or changed in the datastore
    * @param {function(import('libp2p-record').Record): void} [props.onRemove] - Called when an entry is removed from the datastore
    */
@@ -74,6 +83,13 @@ class KadDHT extends EventEmitter {
     concurrency = c.ALPHA,
     validators = {},
     selectors = {},
+    randomWalk = {
+      enabled: false,
+      queriesPerPeriod: 1,
+      interval: 300000,
+      timeout: 30000,
+      delay: 10000
+    },
     onPut = () => {},
     onRemove = () => {}
   }) {
@@ -154,7 +170,7 @@ class KadDHT extends EventEmitter {
      *
      * @type {RoutingTable}
      */
-    this.routingTable = new RoutingTable(this, { kBucketSize: this.kBucketSize })
+    this.routingTable = new RoutingTable(this.peerId, this.kBucketSize)
 
     /**
      * Reference to the datastore, uses an in-memory store if none given.
@@ -184,6 +200,13 @@ class KadDHT extends EventEmitter {
 
     this._log = utils.logger(this.peerId)
 
+    /**
+     * Random walk management
+     *
+     * @type {RandomWalk}
+     */
+    this.randomWalk = new RandomWalk(this, randomWalk)
+
     /**
      * Keeps track of running queries
      *
@@ -214,14 +237,17 @@ class KadDHT extends EventEmitter {
    * Start listening to incoming connections.
    */
   start () {
+    if (this._running) {
+      return
+    }
+
     this._running = true
+    this.providers.start()
+    this._queryManager.start()
+    this.network.start()
 
-    return Promise.all([
-      this.providers.start(),
-      this._queryManager.start(),
-      this.network.start(),
-      this.routingTable.start()
-    ])
+    // Start random walk, it will not run if it's disabled
+    this.randomWalk.start()
   }
 
   /**
@@ -230,13 +256,10 @@ class KadDHT extends EventEmitter {
    */
   stop () {
     this._running = false
-
-    return Promise.all([
-      this.providers.stop(),
-      this._queryManager.stop(),
-      this.network.stop(),
-      this.routingTable.stop()
-    ])
+    this.randomWalk.stop()
+    this.network.stop()
+    this._queryManager.stop()
+    this.providers.stop()
   }
 
   /**

src/peer-list/peer-distance-list.js

@@ -1,10 +1,10 @@
 'use strict'
 
+// @ts-ignore
+const distance = require('xor-distance')
 const utils = require('../utils')
 const pMap = require('p-map')
 const { equals: uint8ArrayEquals } = require('uint8arrays/equals')
-const { compare: uint8ArrayCompare } = require('uint8arrays/compare')
-const { xor: uint8ArrayXor } = require('uint8arrays/xor')
 
 /**
  * @typedef {import('peer-id')} PeerId
@@ -56,11 +56,11 @@ class PeerDistanceList {
     const dhtKey = await utils.convertPeerId(peerId)
     const el = {
       peerId,
-      distance: uint8ArrayXor(this.originDhtKey, dhtKey)
+      distance: distance(this.originDhtKey, dhtKey)
     }
 
     this.peerDistances.push(el)
-    this.peerDistances.sort((a, b) => uint8ArrayCompare(a.distance, b.distance))
+    this.peerDistances.sort((a, b) => distance.compare(a.distance, b.distance))
     this.peerDistances = this.peerDistances.slice(0, this.capacity)
   }
 
@@ -83,9 +83,9 @@ class PeerDistanceList {
     const furthestDistance = this.peerDistances[this.peerDistances.length - 1].distance
 
     for (const dhtKey of dhtKeys) {
-      const keyDistance = uint8ArrayXor(this.originDhtKey, dhtKey)
+      const keyDistance = distance(this.originDhtKey, dhtKey)
 
-      if (uint8ArrayCompare(keyDistance, furthestDistance) < 0) {
+      if (distance.compare(keyDistance, furthestDistance) < 0) {
         return true
       }
     }

src/peer-list/peer-queue.js

@@ -2,7 +2,8 @@
 
 // @ts-ignore
 const Heap = require('heap')
-const { xor: uint8ArrayXor } = require('uint8arrays/xor')
+// @ts-ignore
+const distance = require('xor-distance')
 const debug = require('debug')
 
 const utils = require('../utils')
@@ -64,7 +65,7 @@ class PeerQueue {
 
     const el = {
       id: id,
-      distance: uint8ArrayXor(this.from, key)
+      distance: distance(this.from, key)
     }
 
     this.heap.push(el)

src/providers.js

@@ -68,12 +68,6 @@ class Providers {
    * Start the provider cleanup service
    */
   start () {
-    if (this._started) {
-      return
-    }
-
-    this._started = true
-
     this._cleaner = setInterval(
       () => this._cleanup(),
       this.cleanupInterval
@@ -84,8 +78,6 @@ class Providers {
    * Release any resources.
    */
   stop () {
-    this._started = false
-
     if (this._cleaner) {
       clearInterval(this._cleaner)
       this._cleaner = null