fix: refactor query logic (#237)

1. Queries are plain functions now
1. Queries are run by the QueryManager
1. Query functions yield events - dial, send request, receive response, value, error, etc
1. Timeouts have been replaced with AbortSignals
1. Halting in-progress queries is done by aborting an AbortSignal from the top level or breaking out of the `for await..of` loop
1. Internal components do not depend on the top-level DHT component any more
1. Some methods have been moved to other components to break circular dependencies
1. The top level KadDHT class now has a ts interface and a factory function
1. There are two DHTs, lan (for private addresses) and wan (for public addresses)
1. Client mode is now on by default for the wan DHT, off for the lan DHT
1. DHT queries for CIDs search for the multihash rather than the full CID

Author: achingbrain

.github/workflows/test.yml

@@ -0,0 +1,39 @@
+name: ci
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - run: npm install
+    - run: npx aegir lint
+    - uses: gozala/[email protected]
+    - run: npx aegir build
+    - run: npx aegir dep-check
+    - uses: ipfs/aegir/actions/bundle-size@master
+      name: size
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+  test-node:
+    needs: check
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, ubuntu-latest, macos-latest]
+        node: [14, 16]
+      fail-fast: true
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node }}
+      - run: npm install
+      - run: npx nyc --reporter=lcov aegir test -t node -- --bail
+      - uses: codecov/codecov-action@v1

.gitignore

@@ -41,3 +41,4 @@ test/test-data/go-ipfs-repo/LOG.old
 # while testing npm5
 package-lock.json
 yarn.lock
+.nyc_output

.travis.yml

@@ -1,10 +1,10 @@
-# js-libp2p-kad-dht
+# js-libp2p-kad-dht <!-- omit in toc -->
 
 [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
 [![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
 [![](https://img.shields.io/badge/freenode-%23ipfs-blue.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23ipfs)
 [![Discourse posts](https://img.shields.io/discourse/https/discuss.libp2p.io/posts.svg)](https://discuss.libp2p.io)
-[![Build Status](https://travis-ci.org/libp2p/js-libp2p-kad-dht.svg?style=flat-square)](https://travis-ci.org/libp2p/js-libp2p-kad-dht)
+[![Build status](https://github.com/libp2p/js-libp2p-kad-dht/actions/workflows/test.yml/badge.svg?branch=master)](https://github.com/libp2p/js-libp2p-kad-dht/actions/workflows/test.yml)
 [![Coverage Status](https://coveralls.io/repos/github/libp2p/js-libp2p-kad-dht/badge.svg?branch=master)](https://coveralls.io/github/libp2p/js-libp2p-kad-dht?branch=master)
 [![Dependency Status](https://david-dm.org/libp2p/js-libp2p-kad-dht.svg?style=flat-square)](https://david-dm.org/libp2p/js-libp2p-kad-dht)
 [![Bundle Size](https://flat.badgen.net/bundlephobia/minzip/libp2p-kad-dht)](https://bundlephobia.com/result?p=libp2p-kad-dht)
@@ -15,19 +15,23 @@
 
 > JavaScript implementation of the Kademlia DHT for libp2p, based on [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht).
 
-## Lead Maintainer
+## Lead Maintainer  <!-- omit in toc -->
 
 [Vasco Santos](https://github.com/vasco-santos).
 
-## Table of Contents
+## Table of Contents  <!-- omit in toc -->
 
 - [Install](#install)
   - [npm](#npm)
   - [Use in Node.js](#use-in-nodejs)
 - [API](#api)
+  - [Custom secondary DHT in libp2p](#custom-secondary-dht-in-libp2p)
+  - [Peer Routing](#peer-routing)
+  - [Content Routing](#content-routing)
+  - [Peer Discovery](#peer-discovery)
+  - [Implementation Summary](#implementation-summary)
 - [Contribute](#contribute)
 - [License](#license)
-
 ## Install
 
 ### npm
@@ -39,7 +43,7 @@
 ### Use in Node.js
 
 ```js
-const KadDHT = require('libp2p-kad-dht')
+import { create } from 'libp2p-kad-dht'
 ```
 
 ## API
@@ -51,20 +55,18 @@ The libp2p-kad-dht module offers 3 APIs: Peer Routing, Content Routing and Peer
 ### Custom secondary DHT in libp2p
 
 ```js
+import { create } from 'libp2p-kad-dht'
+
 /**
  * @param {Libp2p} libp2p
  */
 function addDHT(libp2p) {
-    const customDHT = new KadDHT({
+    const customDHT = create({
         libp2p,
-        dialer: libp2p.dialer,
-        peerId: libp2p.peerId,
-        peerStore: libp2p.peerStore,
-        registrar: libp2p.registrar,
         protocolPrefix: '/custom'
     })
     customDHT.start()
-    customDHT.on('peer', libp2p._onDiscoveryPeer)
+
     return customDHT
 }
 ```

README.md

@@ -1,67 +1,26 @@
-# js-libp2p-kad-dht
+# js-libp2p-kad-dht <!-- omit in toc -->
 
-js-libp2p-kad-dht is a JavaScript implementation of the [Kademlia DHT](http://www.scs.stanford.edu/%7Edm/home/papers/kpos.pdf) with some features of S/Kademlia. A "provider" node uses the DHT to advertise that it has a particular piece of content, and "querying" nodes will search the DHT for peers that have a particular piece of content. Content is modeled as a value that is identified by a key, where the key and value are Buffers.
+js-libp2p-kad-dht is a JavaScript implementation of the [Kademlia DHT][] with some features of [S/Kademlia][]. A "provider" node uses the DHT to advertise that it has a particular piece of content, and "querying" nodes will search the DHT for peers that have a particular piece of content. Content is modelled as a value that is identified by a key, where the key and value are Uint8Arrays.
 
-#### DHT Identifiers
+## Table of contents <!-- omit in toc -->
 
-The DHT uses a sha2-256 hash for identifiers:
-- For peers the DHT identifier is the hash of the [PeerId][PeerId]
-- For content the DHT identifier is the hash of the key (eg a Block CID)
+- [Spec](#spec)
+- [Extensions](#extensions)
+  - [Disjoint paths](#disjoint-paths)
 
-#### FIND_NODE
+## Spec
 
-`findPeer (PeerId):` [PeerInfo][PeerInfo]
+js-libp2p-kad-dht follows the [libp2p/kad-dht spec](https://github.com/libp2p/specs/tree/master/kad-dht) and implements the algorithms described in the [IPFS DHT documentation](https://docs.ipfs.io/concepts/dht/).
 
-The address space is so large (256 bits) that there are big gaps between DHT ids, and nodes frequently join and leave the DHT.
+## Extensions
 
-To find a particular node
-- the `querying node` converts the [PeerId][PeerId] to a DHT id
-- the `querying node` sends a request to the nearest peers to that DHT id that it knows about
-- those peers respond with the nearest peers to the DHT id that they know about
-- the `querying node` sorts the responses and recursively queries the closest peers to the DHT id, continuing until it finds the node or it has queried all the closest peers.
+js-libp2p-kad-dht implements some additional functionality not described in the libp2p KAD-DHT spec.
 
-#### PUT
+### Disjoint paths
 
-`put (Key, Value)`
-
-To store a value in the DHT, the `provider node`
-- converts the key to a DHT id
-- follows the "closest peers" algorithm as above to find the nearest peers to the DHT id
-- sends the value to those nearest peers
-
-Note that DHT nodes will only store values that are accepted by its "validators", configurable functions that validate the key/value to ensure the node can control what kind of content it stores (eg IPNS records).
-
-#### GET
-
-`get (Key): [Value]`
-
-To retrieve a value from the DHT
-- the `querying node` converts the key to a DHT id
-- the `querying node` follows the "closest peers" algorithm to find the nearest peers to the DHT id
-- at each iteration of the algorithm, if the peer has the value it responds with the value itself in addition to closer peers.
-
-Note that the value for a particular key is stored by many nodes, and these nodes receive `PUT` requests asynchronously, so it's possible that nodes may have distinct values for the same key. For example if node A `PUT`s the value `hello` to key `greeting` and node B concurrently `PUT`s the value `bonjour` to key `greeting`, some nodes close to the key `greeting` may receive `hello` first and others may receive `bonjour` first.
-
-Therefore a `GET` request to the DHT may collect distinct values (eg `hello` and `bonjour`) for a particular key from the nodes close to the key. The DHT has "selectors", configurable functions that choose the "best" value (for example IPNS records include a sequence number, and the "best" value is the record with the highest sequence number).
-
-#### PROVIDE
-
-`provide (Key)`
-
-To advertise that it has the content for a particular key
-- the `provider node` converts the key to a DHT id
-- the `provider node` follows the "closest peers" algorithm to find the nearest peers to the DHT id
-- the `provider node` sends a "provide" message to each of the nearest peers
-- each of the nearest peers saves the association between the "provider" peer and the key
-
-#### FIND_PROVIDERS
-
-`findProviders (Key):` [[PeerInfo][PeerInfo]]
-
-To find providers for a particular key
-- the `querying node` converts the key to a DHT id
-- the `querying node` follows the "closest peers" algorithm to find the nearest peers to the DHT id
-- at each iteration of the algorithm, if the peer knows which nodes are providing the value it responds with the provider nodes in addition to closer peers.
+js-libp2p-kad-dht uses disjoint paths when performing DHT queries. These are described in the [S/Kademlia][] paper.
 
+[Kademlia DHT]: (http://www.scs.stanford.edu/%7Edm/home/papers/kpos.pdf)
+[S/Kademlia]: (https://git.gnunet.org/bibliography.git/plain/docs/SKademlia2007.pdf)
 [PeerId]: https://github.com/libp2p/js-peer-id
 [PeerInfo]: https://github.com/libp2p/js-peer-info

docs/IMPL_SUMMARY.MD

@@ -5,8 +5,8 @@
   "leadMaintainer": "Vasco Santos <[email protected]>",
   "main": "src/index.js",
   "scripts": {
-    "lint": "aegir ts -p check && aegir lint",
     "prepare": "npm run build",
+    "lint": "aegir ts -p check && aegir lint",
     "build": "npm run build:proto && npm run build:proto-types && aegir build",
     "build:proto": "pbjs -t static-module -w commonjs -r libp2p-dht-message --force-number --no-verify --no-delimited --no-create --no-beautify --no-defaults --lint eslint-disable -o src/message/dht.js ./src/message/dht.proto",
     "build:proto-types": "pbts -o src/message/dht.d.ts src/message/dht.js",
@@ -18,7 +18,8 @@
     "release-major": "aegir release --type major --docs -t node",
     "coverage": "aegir coverage",
     "coverage-publish": "aegir-coverage publish",
-    "sim": "node test/simulation/index.js"
+    "sim": "node test/simulation/index.js",
+    "dep-check": "aegir dep-check"
   },
   "files": [
     "src",
@@ -47,47 +48,53 @@
   "homepage": "https://github.com/libp2p/js-libp2p-kad-dht",
   "types": "dist/src/index.d.ts",
   "dependencies": {
+    "any-signal": "^2.1.2",
     "datastore-core": "^6.0.7",
     "debug": "^4.3.1",
     "err-code": "^3.0.0",
     "hashlru": "^2.3.0",
-    "heap": "~0.2.6",
     "interface-datastore": "^6.0.2",
+    "it-all": "^1.0.5",
+    "it-drain": "^1.0.4",
     "it-first": "^1.0.4",
     "it-length": "^1.0.3",
     "it-length-prefixed": "^5.0.2",
+    "it-map": "^1.0.5",
+    "it-merge": "^1.0.3",
+    "it-parallel": "^2.0.1",
     "it-pipe": "^1.1.0",
+    "it-take": "^1.0.2",
     "k-bucket": "^5.1.0",
     "libp2p-crypto": "^0.19.5",
     "libp2p-interfaces": "^1.0.0",
     "libp2p-record": "^0.10.4",
     "multiaddr": "^10.0.0",
     "multiformats": "^9.4.5",
+    "native-abort-controller": "^1.0.4",
+    "p-defer": "^3.0.0",
     "p-map": "^4.0.0",
     "p-queue": "^6.6.2",
-    "p-timeout": "^4.1.0",
     "peer-id": "^0.15.0",
+    "private-ip": "^2.3.3",
     "protobufjs": "^6.10.2",
     "streaming-iterables": "^6.0.0",
-    "timeout-abort-controller": "^1.1.1",
+    "timeout-abort-controller": "^2.0.0",
     "uint8arrays": "^3.0.0",
     "varint": "^6.0.0"
   },
   "devDependencies": {
     "@types/debug": "^4.1.7",
     "aegir": "^35.0.1",
     "async-iterator-all": "^1.0.0",
-    "crypto-browserify": "^3.12.0",
     "datastore-level": "^7.0.1",
     "delay": "^5.0.0",
     "execa": "^5.1.1",
+    "it-filter": "^1.0.3",
+    "it-last": "^1.0.6",
     "it-pair": "^1.0.0",
-    "libp2p": "^0.32.3",
+    "libp2p": "^0.33.0",
     "lodash.random": "^3.2.0",
     "lodash.range": "^3.2.0",
-    "p-defer": "^3.0.0",
-    "p-each-series": "^2.1.0",
-    "p-map-series": "^2.1.0",
     "p-retry": "^4.2.0",
     "sinon": "^11.1.1",
     "which": "^2.0.2"

package.json

@@ -33,3 +33,6 @@ exports.K = 20
 
 // Alpha is the concurrency for asynchronous requests
 exports.ALPHA = 3
+
+// How often we look for our closest DHT neighbours
+exports.QUERY_SELF_INTERVAL = Number(5 * minute)