diff --git a/README.md b/README.md index 6dbe65b382..541eca795c 100644 --- a/README.md +++ b/README.md @@ -34,11 +34,6 @@ You can check the development status at the [Kanban Board](https://waffle.io/ipf [![Throughput Graph](https://graphs.waffle.io/ipfs/js-ipfs/throughput.svg)](https://waffle.io/ipfs/js-ipfs/metrics/throughput) -**Please read this:** DHT (automatic content discovery) and Circuit Relay (pierce through NATs and dial between any node in the network) are two fundamental pieces that are not finalized yet. There are multiple applications that can be built without these two services but nevertheless they are fundamental to get that magic IPFS experience. If you want to track progress or contribute, please follow: - -- DHT: https://github.com/ipfs/js-ipfs/pull/856 -- ✅ Relay: https://github.com/ipfs/js-ipfs/pull/1063 - [**`Weekly Core Dev Calls`**](https://github.com/ipfs/pm/issues/650) ## Tech Lead @@ -319,7 +314,6 @@ Enable and configure experimental features. - `pubsub` (boolean): Enable libp2p pub-sub. (Default: `false`) - `ipnsPubsub` (boolean): Enable pub-sub on IPNS. (Default: `false`) - `sharding` (boolean): Enable directory sharding. Directories that have many child objects will be represented by multiple DAG nodes instead of just one. It can improve lookup performance when a directory has several thousand files or more. (Default: `false`) -- `dht` (boolean): Enable KadDHT. **This is currently not interoperable with `go-ipfs`.** ##### `options.config` @@ -600,7 +594,13 @@ The core API is grouped into several areas: - [`ipfs.bootstrap.add(addr, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md#bootstrapadd) - [`ipfs.bootstrap.rm(peer, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md#bootstraprm) -- dht (not implemented yet) +- [dht](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/) + - [`ipfs.dht.findPeer(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindpeer) + - [`ipfs.dht.findProvs(multihash, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtfindprovs) + - [`ipfs.dht.get(key, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtget) + - [`ipfs.dht.provide(cid, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtprovide) + - [`ipfs.dht.put(key, value, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtput) + - [`ipfs.dht.query(peerId, [callback])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DHT.md#dhtquery) - [pubsub](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md) - [`ipfs.pubsub.subscribe(topic, handler, [options], [callback])`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubsubscribe) diff --git a/package.json b/package.json index 7a4c204881..e6e5b07a19 100644 --- a/package.json +++ b/package.json @@ -72,7 +72,7 @@ "form-data": "^2.3.3", "hat": "0.0.3", "interface-ipfs-core": "~0.92.0", - "ipfsd-ctl": "~0.40.1", + "ipfsd-ctl": "~0.40.2", "ncp": "^2.0.0", "qs": "^6.5.2", "rimraf": "^2.6.2", @@ -128,10 +128,10 @@ "joi": "^14.3.0", "joi-browser": "^13.4.0", "joi-multiaddr": "^3.0.0", - "libp2p": "~0.24.1", + "libp2p": "~0.24.3", "libp2p-bootstrap": "~0.9.3", "libp2p-crypto": "~0.14.1", - "libp2p-kad-dht": "~0.12.1", + "libp2p-kad-dht": "~0.14.1", "libp2p-keychain": "~0.3.3", "libp2p-mdns": "~0.12.0", "libp2p-mplex": "~0.8.4", diff --git a/src/cli/commands/daemon.js b/src/cli/commands/daemon.js index e71d3a5d38..1888f62822 100644 --- a/src/cli/commands/daemon.js +++ b/src/cli/commands/daemon.js @@ -21,10 +21,6 @@ module.exports = { type: 'boolean', default: false }) - .option('enable-dht-experiment', { - type: 'boolean', - default: false - }) .option('local', { desc: 'Run commands locally to the daemon', default: false diff --git a/src/cli/commands/dht.js b/src/cli/commands/dht.js new file mode 100644 index 0000000000..d899cc3c5c --- /dev/null +++ b/src/cli/commands/dht.js @@ -0,0 +1,14 @@ +'use strict' + +module.exports = { + command: 'dht ', + + description: 'Issue commands directly through the DHT.', + + builder (yargs) { + return yargs.commandDir('dht') + }, + + handler (argv) { + } +} diff --git a/src/cli/commands/dht/find-peer.js b/src/cli/commands/dht/find-peer.js new file mode 100644 index 0000000000..337e9eb73a --- /dev/null +++ b/src/cli/commands/dht/find-peer.js @@ -0,0 +1,23 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'findpeer ', + + describe: 'Find the multiaddresses associated with a Peer ID.', + + builder: {}, + + handler (argv) { + argv.ipfs.dht.findPeer(argv.peerID, (err, result) => { + if (err) { + throw err + } + + const addresses = result.multiaddrs.toArray().map((ma) => ma.toString()) + + print(addresses) + }) + } +} diff --git a/src/cli/commands/dht/find-providers.js b/src/cli/commands/dht/find-providers.js new file mode 100644 index 0000000000..b34b0991a8 --- /dev/null +++ b/src/cli/commands/dht/find-providers.js @@ -0,0 +1,33 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'findprovs ', + + describe: 'Find peers that can provide a specific value, given a key.', + + builder: { + 'num-providers': { + alias: 'n', + describe: 'The number of providers to find. Default: 20.', + default: 20 + } + }, + + handler (argv) { + const opts = { + maxNumProviders: argv['num-providers'] + } + + argv.ipfs.dht.findProvs(argv.key, opts, (err, result) => { + if (err) { + throw err + } + + result.forEach((element) => { + print(element.id.toB58String()) + }) + }) + } +} diff --git a/src/cli/commands/dht/get.js b/src/cli/commands/dht/get.js new file mode 100644 index 0000000000..85a52b25bf --- /dev/null +++ b/src/cli/commands/dht/get.js @@ -0,0 +1,21 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'get ', + + describe: 'Given a key, query the routing system for its best value.', + + builder: {}, + + handler (argv) { + argv.ipfs.dht.get(argv.key, (err, result) => { + if (err) { + throw err + } + + print(result) + }) + } +} diff --git a/src/cli/commands/dht/provide.js b/src/cli/commands/dht/provide.js new file mode 100644 index 0000000000..b6a3127cb2 --- /dev/null +++ b/src/cli/commands/dht/provide.js @@ -0,0 +1,23 @@ +'use strict' + +module.exports = { + command: 'provide ', + + describe: 'Announce to the network that you are providing given values.', + + builder: { + recursive: { + alias: 'r', + recursive: 'Recursively provide entire graph.', + default: false + } + }, + + handler (argv) { + argv.ipfs.dht.provide(argv.key, (err, result) => { + if (err) { + throw err + } + }) + } +} diff --git a/src/cli/commands/dht/put.js b/src/cli/commands/dht/put.js new file mode 100644 index 0000000000..8e594fcb07 --- /dev/null +++ b/src/cli/commands/dht/put.js @@ -0,0 +1,17 @@ +'use strict' + +module.exports = { + command: 'put ', + + describe: 'Write a key/value pair to the routing system.', + + builder: {}, + + handler (argv) { + argv.ipfs.dht.put(argv.key, argv.value, (err) => { + if (err) { + throw err + } + }) + } +} diff --git a/src/cli/commands/dht/query.js b/src/cli/commands/dht/query.js new file mode 100644 index 0000000000..16a9061a35 --- /dev/null +++ b/src/cli/commands/dht/query.js @@ -0,0 +1,23 @@ +'use strict' + +const print = require('../../utils').print + +module.exports = { + command: 'query ', + + describe: 'Find the closest Peer IDs to a given Peer ID by querying the DHT.', + + builder: {}, + + handler (argv) { + argv.ipfs.dht.query(argv.peerID, (err, result) => { + if (err) { + throw err + } + + result.forEach((peerID) => { + print(peerID.id.toB58String()) + }) + }) + } +} diff --git a/src/core/components/dht.js b/src/core/components/dht.js index 9cfc91b7b4..68dad79388 100644 --- a/src/core/components/dht.js +++ b/src/core/components/dht.js @@ -3,11 +3,15 @@ const promisify = require('promisify-es6') const every = require('async/every') const PeerId = require('peer-id') +const PeerInfo = require('peer-info') const CID = require('cids') const each = require('async/each') const setImmediate = require('async/setImmediate') -// const bsplit = require('buffer-split') -const errCode = require('err-code') +const errcode = require('err-code') + +const debug = require('debug') +const log = debug('jsipfs:dht') +log.error = debug('jsipfs:dht:error') module.exports = (self) => { return { @@ -15,14 +19,12 @@ module.exports = (self) => { * Given a key, query the DHT for its best value. * * @param {Buffer} key + * @param {Object} options - get options + * @param {number} options.timeout - optional timeout * @param {function(Error)} [callback] * @returns {Promise|void} */ get: promisify((key, options, callback) => { - if (!Buffer.isBuffer(key)) { - return callback(new Error('Not valid key')) - } - if (typeof options === 'function') { callback = options options = {} @@ -30,7 +32,17 @@ module.exports = (self) => { options = options || {} - self._libp2pNode.dht.get(key, options.timeout, callback) + if (!Buffer.isBuffer(key)) { + try { + key = (new CID(key)).buffer + } catch (err) { + log.error(err) + + return setImmediate(() => callback(errcode(err, 'ERR_INVALID_CID'))) + } + } + + self._libp2pNode.dht.get(key, options, callback) }), /** @@ -47,7 +59,13 @@ module.exports = (self) => { */ put: promisify((key, value, callback) => { if (!Buffer.isBuffer(key)) { - return callback(new Error('Not valid key')) + try { + key = (new CID(key)).buffer + } catch (err) { + log.error(err) + + return setImmediate(() => callback(errcode(err, 'ERR_INVALID_CID'))) + } } self._libp2pNode.dht.put(key, value, callback) @@ -57,72 +75,53 @@ module.exports = (self) => { * Find peers in the DHT that can provide a specific value, given a key. * * @param {CID} key - They key to find providers for. + * @param {Object} options - findProviders options + * @param {number} options.timeout - how long the query should maximally run, in milliseconds (default: 60000) + * @param {number} options.maxNumProviders - maximum number of providers to find * @param {function(Error, Array)} [callback] * @returns {Promise|void} */ - findprovs: promisify((key, opts, callback) => { - if (typeof opts === 'function') { - callback = opts - opts = {} + findProvs: promisify((key, options, callback) => { + if (typeof options === 'function') { + callback = options + options = {} } - opts = opts || {} + options = options || {} if (typeof key === 'string') { try { key = new CID(key) } catch (err) { - return setImmediate(() => callback(errCode(err, 'ERR_INVALID_CID'))) - } - } + log.error(err) - if (typeof opts === 'function') { - callback = opts - opts = {} + return setImmediate(() => callback(errcode(err, 'ERR_INVALID_CID'))) + } } - opts = opts || {} - - self._libp2pNode.contentRouting.findProviders(key, opts.timeout || null, callback) + self._libp2pNode.contentRouting.findProviders(key, options, callback) }), /** * Query the DHT for all multiaddresses associated with a `PeerId`. * * @param {PeerId} peer - The id of the peer to search for. - * @param {function(Error, Array)} [callback] - * @returns {Promise>|void} + * @param {function(Error, PeerInfo)} [callback] + * @returns {Promise|void} */ - findpeer: promisify((peer, callback) => { + findPeer: promisify((peer, callback) => { if (typeof peer === 'string') { peer = PeerId.createFromB58String(peer) } - self._libp2pNode.peerRouting.findPeer(peer, (err, info) => { - if (err) { - return callback(err) - } - - // convert to go-ipfs return value, we need to revisit - // this. For now will just conform. - const goResult = [ - { - Responses: [{ - ID: info.id.toB58String(), - Addresses: info.multiaddrs.toArray().map((a) => a.toString()) - }] - } - ] - - callback(null, goResult) - }) + self._libp2pNode.peerRouting.findPeer(peer, callback) }), /** * Announce to the network that we are providing given values. * * @param {CID|Array} keys - The keys that should be announced. - * @param {Object} [options={}] + * @param {Object} options - provide options * @param {bool} [options.recursive=false] - Provide not only the given object but also all objects linked from it. * @param {function(Error)} [callback] * @returns {Promise|void} @@ -147,11 +146,15 @@ module.exports = (self) => { } if (!has) { - return callback(new Error('block(s) not found locally, cannot provide')) + const errMsg = 'block(s) not found locally, cannot provide' + + log.error(errMsg) + return callback(errcode(errMsg, 'ERR_BLOCK_NOT_FOUND')) } if (options.recursive) { // TODO: Implement recursive providing + return callback(errcode('not implemented yet', 'ERR_NOT_IMPLEMENTED_YET')) } else { each(keys, (cid, cb) => { self._libp2pNode.contentRouting.provide(cid, cb) @@ -164,22 +167,27 @@ module.exports = (self) => { * Find the closest peers to a given `PeerId`, by querying the DHT. * * @param {PeerId} peer - The `PeerId` to run the query agains. - * @param {function(Error, Array)} [callback] - * @returns {Promise>|void} + * @param {function(Error, Array)} [callback] + * @returns {Promise>|void} */ query: promisify((peerId, callback) => { if (typeof peerId === 'string') { - peerId = PeerId.createFromB58String(peerId) + try { + peerId = PeerId.createFromB58String(peerId) + } catch (err) { + log.error(err) + return callback(err) + } } // TODO expose this method in peerRouting self._libp2pNode._dht.getClosestPeers(peerId.toBytes(), (err, peerIds) => { if (err) { + log.error(err) return callback(err) } - callback(null, peerIds.map((id) => { - return { ID: id.toB58String() } - })) + + callback(null, peerIds.map((id) => new PeerInfo(id))) }) }) } diff --git a/src/core/components/libp2p.js b/src/core/components/libp2p.js index c5e3a40b1c..43c33e1681 100644 --- a/src/core/components/libp2p.js +++ b/src/core/components/libp2p.js @@ -46,6 +46,8 @@ module.exports = function libp2p (self) { } }, dht: { + kBucketSize: get(opts.options, 'dht.kBucketSize', 20), + enabledDiscovery: get(opts.options, 'dht.enabledDiscovery', true), validators: { ipns: ipnsUtils.validator }, @@ -54,12 +56,15 @@ module.exports = function libp2p (self) { } }, EXPERIMENTAL: { - dht: get(opts.options, 'EXPERIMENTAL.dht', false), + dht: !(get(opts.options, 'local', false)), pubsub: get(opts.options, 'EXPERIMENTAL.pubsub', false) } }, connectionManager: get(opts.options, 'connectionManager', - get(opts.config, 'connectionManager', {})) + { + maxPeers: get(opts.config, 'Swarm.ConnMgr.HighWater'), + minPeers: get(opts.config, 'Swarm.ConnMgr.LowWater') + }), } const libp2pOptions = defaultsDeep( diff --git a/src/core/components/pin-set.js b/src/core/components/pin-set.js index 91f72bf90b..31ecfffdf0 100644 --- a/src/core/components/pin-set.js +++ b/src/core/components/pin-set.js @@ -6,8 +6,8 @@ const protobuf = require('protons') const fnv1a = require('fnv1a') const varint = require('varint') const { DAGNode, DAGLink } = require('ipld-dag-pb') -const some = require('async/some') -const eachOf = require('async/eachOf') +const someSeries = require('async/someSeries') +const eachOfSeries = require('async/eachOfSeries') const pbSchema = require('./pin.proto') @@ -69,7 +69,7 @@ exports = module.exports = function (dag) { return searchChildren(root, callback) function searchChildren (root, cb) { - some(root.links, ({ cid }, done) => { + someSeries(root.links, ({ cid }, done) => { const bs58Link = toB58String(cid) if (bs58Link === childhash) { @@ -174,7 +174,7 @@ exports = module.exports = function (dag) { return bins }, {}) - eachOf(bins, (bin, idx, eachCb) => { + eachOfSeries(bins, (bin, idx, eachCb) => { storePins( bin, depth + 1, @@ -233,7 +233,7 @@ exports = module.exports = function (dag) { return callback(err) } - eachOf(node.links, (link, idx, eachCb) => { + eachOfSeries(node.links, (link, idx, eachCb) => { if (idx < pbh.header.fanout) { // the first pbh.header.fanout links are fanout bins // if a fanout bin is not 'empty', dig into and walk its DAGLinks diff --git a/src/core/components/start.js b/src/core/components/start.js index 95befc849b..957d00904b 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -55,8 +55,7 @@ module.exports = (self) => { } // DHT should be added as routing if we are not running with local flag - // TODO: Need to change this logic once DHT is enabled by default, for now fallback to Offline datastore - if (get(self._options, 'EXPERIMENTAL.dht', false) && !self._options.local) { + if (!self._options.local) { ipnsStores.push(self._libp2pNode.dht) } else { const offlineDatastore = new OfflineDatastore(self._repo) diff --git a/src/core/index.js b/src/core/index.js index 496396a443..aca3f13fc5 100644 --- a/src/core/index.js +++ b/src/core/index.js @@ -178,9 +178,6 @@ class IPFS extends EventEmitter { if (this._options.EXPERIMENTAL.sharding) { this.log('EXPERIMENTAL sharding is enabled') } - if (this._options.EXPERIMENTAL.dht) { - this.log('EXPERIMENTAL Kademlia DHT is enabled') - } this.state = require('./state')(this) diff --git a/src/core/runtime/config-browser.js b/src/core/runtime/config-browser.js index ca8c99e153..61e31eca36 100644 --- a/src/core/runtime/config-browser.js +++ b/src/core/runtime/config-browser.js @@ -25,5 +25,11 @@ module.exports = () => ({ '/dns4/nyc-2.bootstrap.libp2p.io/tcp/443/wss/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64', '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' - ] + ], + Swarm: { + ConnMgr: { + LowWater: 600, + HighWater: 900 + } + } }) diff --git a/src/core/runtime/config-nodejs.js b/src/core/runtime/config-nodejs.js index 5b301d1e20..4cc6584fe0 100644 --- a/src/core/runtime/config-nodejs.js +++ b/src/core/runtime/config-nodejs.js @@ -38,5 +38,11 @@ module.exports = () => ({ '/ip6/2604:a880:1:20::1d9:6001/tcp/4001/ipfs/QmSoLju6m7xTh3DuokvT3886QRYqxAzb1kShaanJgW36yx', '/dns4/node0.preload.ipfs.io/tcp/443/wss/ipfs/QmZMxNdpMkewiVZLMRxaNxUeZpDUb34pWjZ1kZvsd16Zic', '/dns4/node1.preload.ipfs.io/tcp/443/wss/ipfs/Qmbut9Ywz9YEDrz8ySBSgWyJk41Uvm2QJPhwDJzJyGFsD6' - ] + ], + Swarm: { + ConnMgr: { + LowWater: 600, + HighWater: 900 + } + } }) diff --git a/src/core/runtime/libp2p-browser.js b/src/core/runtime/libp2p-browser.js index fec12fa9a3..2e2d70ba72 100644 --- a/src/core/runtime/libp2p-browser.js +++ b/src/core/runtime/libp2p-browser.js @@ -6,6 +6,7 @@ const WebSocketStar = require('libp2p-websocket-star') const Multiplex = require('libp2p-mplex') const SECIO = require('libp2p-secio') const Bootstrap = require('libp2p-bootstrap') +const KadDHT = require('libp2p-kad-dht') const libp2p = require('libp2p') const defaultsDeep = require('@nodeutils/defaults-deep') @@ -31,7 +32,8 @@ class Node extends libp2p { wrtcstar.discovery, wsstar.discovery, Bootstrap - ] + ], + dht: KadDHT }, config: { peerDiscovery: { @@ -45,8 +47,12 @@ class Node extends libp2p { enabled: true } }, + dht: { + kBucketSize: 20, + enabledDiscovery: true + }, EXPERIMENTAL: { - dht: false, + dht: true, pubsub: false } } diff --git a/src/core/runtime/libp2p-nodejs.js b/src/core/runtime/libp2p-nodejs.js index 1b59c435c7..ce58a56698 100644 --- a/src/core/runtime/libp2p-nodejs.js +++ b/src/core/runtime/libp2p-nodejs.js @@ -48,10 +48,11 @@ class Node extends libp2p { } }, dht: { - kBucketSize: 20 + kBucketSize: 20, + enabledDiscovery: true }, EXPERIMENTAL: { - dht: false, + dht: true, pubsub: false } } diff --git a/src/http/api/resources/dht.js b/src/http/api/resources/dht.js new file mode 100644 index 0000000000..48755716d1 --- /dev/null +++ b/src/http/api/resources/dht.js @@ -0,0 +1,211 @@ +'use strict' + +const Joi = require('joi') + +const CID = require('cids') + +const debug = require('debug') +const log = debug('jsipfs:http-api:dht') +log.error = debug('jsipfs:http-api:dht:error') + +exports = module.exports + +exports.findPeer = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required() + }).unknown() + }, + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + ipfs.dht.findPeer(arg, (err, res) => { + if (err) { + log.error(err) + + if (err.code === 'ERR_LOOKUP_FAILED') { + return reply({ + Message: err.toString(), + Code: 0 + }).code(404) + } + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({ + Responses: [{ + ID: res.id.toB58String(), + Addrs: res.multiaddrs.toArray().map((a) => a.toString()) + }], + Type: 2 + }) + }) + } +} + +exports.findProvs = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required(), + 'num-providers': Joi.number().integer().default(20), + timeout: Joi.number() + }).unknown() + }, + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + request.query.maxNumProviders = request.query['num-providers'] + + ipfs.dht.findProvs(arg, request.query, (err, res) => { + if (err) { + log.error(err) + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({ + Responses: res.map((peerInfo) => ({ + ID: peerInfo.id.toB58String(), + Addrs: peerInfo.multiaddrs.toArray().map((a) => a.toString()) + })), + Type: 4 + }) + }) + } +} + +exports.get = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required(), + timeout: Joi.number() + }).unknown() + }, + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + ipfs.dht.get(Buffer.from(arg), (err, res) => { + if (err) { + log.error(err) + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({ + Extra: res.toString(), + Type: 5 + }) + }) + } +} + +exports.provide = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required() + }).unknown() + }, + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + const { arg } = request.query + let cid + + try { + cid = new CID(arg) + } catch (err) { + log.error(err) + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + ipfs.dht.provide(cid, request.query, (err) => { + if (err) { + log.error(err) + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({}) + }) + } +} + +exports.put = { + validate: { + query: Joi.object().keys({ + arg: Joi.array().items(Joi.string()).length(2).required() + }).unknown() + }, + parseArgs: (request, reply) => { + return reply({ + key: request.query.arg[0], + value: request.query.arg[1] + }) + }, + handler: (request, reply) => { + const key = request.pre.args.key + const value = request.pre.args.value + const ipfs = request.server.app.ipfs + + ipfs.dht.put(Buffer.from(key), Buffer.from(value), (err) => { + if (err) { + log.error(err) + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + reply({}) + }) + } +} + +exports.query = { + validate: { + query: Joi.object().keys({ + arg: Joi.string().required() + }).unknown() + }, + handler: (request, reply) => { + const ipfs = request.server.app.ipfs + const { arg } = request.query + + ipfs.dht.query(arg, (err, res) => { + if (err) { + log.error(err) + + return reply({ + Message: err.toString(), + Code: 0 + }).code(500) + } + + const response = res.map((peerInfo) => ({ + ID: peerInfo.id.toB58String() + })) + + reply(response) + }) + } +} diff --git a/src/http/api/resources/index.js b/src/http/api/resources/index.js index 66646d29d5..a54aa0d4f3 100644 --- a/src/http/api/resources/index.js +++ b/src/http/api/resources/index.js @@ -20,3 +20,4 @@ exports.key = require('./key') exports.stats = require('./stats') exports.resolve = require('./resolve') exports.name = require('./name') +exports.dht = require('./dht') diff --git a/src/http/api/routes/dht.js b/src/http/api/routes/dht.js new file mode 100644 index 0000000000..7f328a671a --- /dev/null +++ b/src/http/api/routes/dht.js @@ -0,0 +1,64 @@ +'use strict' + +const resources = require('./../resources') + +module.exports = (server) => { + const api = server.select('API') + + api.route({ + method: '*', + path: '/api/v0/dht/findpeer', + config: { + handler: resources.dht.findPeer.handler, + validate: resources.dht.findPeer.validate + } + }) + + api.route({ + method: '*', + path: '/api/v0/dht/findprovs', + config: { + handler: resources.dht.findProvs.handler, + validate: resources.dht.findProvs.validate + } + }) + + api.route({ + method: '*', + path: '/api/v0/dht/get', + config: { + handler: resources.dht.get.handler, + validate: resources.dht.get.validate + } + }) + + api.route({ + method: '*', + path: '/api/v0/dht/provide', + config: { + handler: resources.dht.provide.handler, + validate: resources.dht.provide.validate + } + }) + + api.route({ + method: '*', + path: '/api/v0/dht/put', + config: { + pre: [ + { method: resources.dht.put.parseArgs, assign: 'args' } + ], + handler: resources.dht.put.handler, + validate: resources.dht.put.validate + } + }) + + api.route({ + method: '*', + path: '/api/v0/dht/query', + config: { + handler: resources.dht.query.handler, + validate: resources.dht.query.validate + } + }) +} diff --git a/src/http/api/routes/index.js b/src/http/api/routes/index.js index 450df4e4a5..85915c7997 100644 --- a/src/http/api/routes/index.js +++ b/src/http/api/routes/index.js @@ -23,4 +23,5 @@ module.exports = (server) => { require('./stats')(server) require('./resolve')(server) require('./name')(server) + require('./dht')(server) } diff --git a/src/http/index.js b/src/http/index.js index e80c6b609f..517efa7482 100644 --- a/src/http/index.js +++ b/src/http/index.js @@ -86,7 +86,6 @@ function HttpApi (repo, config, cliArgs) { EXPERIMENTAL: { pubsub: cliArgs.enablePubsubExperiment, ipnsPubsub: cliArgs.enableNamesysPubsub, - dht: cliArgs.enableDhtExperiment, sharding: cliArgs.enableShardingExperiment }, libp2p: libp2p diff --git a/test/cli/commands.js b/test/cli/commands.js index 9ba3f44519..f86e973b03 100644 --- a/test/cli/commands.js +++ b/test/cli/commands.js @@ -4,7 +4,7 @@ const expect = require('chai').expect const runOnAndOff = require('../utils/on-and-off') -const commandCount = 86 +const commandCount = 93 describe('commands', () => runOnAndOff((thing) => { let ipfs diff --git a/test/cli/dht.js b/test/cli/dht.js new file mode 100644 index 0000000000..0f12be3abc --- /dev/null +++ b/test/cli/dht.js @@ -0,0 +1,151 @@ +/* eslint-env mocha */ + +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +const series = require('async/series') +const parallel = require('async/parallel') + +const DaemonFactory = require('ipfsd-ctl') +const df = DaemonFactory.create({ type: 'js' }) + +const ipfsExec = require('../utils/ipfs-exec') + +const daemonOpts = { + exec: `./src/cli/bin.js`, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + }, + initOptions: { bits: 512 } +} + +describe('dht', () => { + let nodes = [] + let ipfsA + let ipfsB + let idA + let idB + let multiaddrB + + // spawn daemons + before(function (done) { + this.timeout(80 * 1000) + series([ + (cb) => df.spawn(daemonOpts, (err, _ipfsd) => { + expect(err).to.not.exist() + + ipfsA = ipfsExec(_ipfsd.repoPath) + nodes.push(_ipfsd) + cb() + }), + (cb) => df.spawn(daemonOpts, (err, _ipfsd) => { + expect(err).to.not.exist() + + ipfsB = ipfsExec(_ipfsd.repoPath) + nodes.push(_ipfsd) + cb() + }) + ], done) + }) + + // get ids + before(function (done) { + this.timeout(80 * 1000) + parallel([ + (cb) => nodes[0].api.id((err, res) => { + expect(err).to.not.exist() + + idA = res.id + cb() + }), + (cb) => nodes[1].api.id((err, res) => { + expect(err).to.not.exist() + + multiaddrB = res.addresses[0] + idB = res.id + cb() + }) + ], done) + }) + + // connect daemons + before(function (done) { + this.timeout(80 * 1000) + + nodes[0].api.swarm.connect(multiaddrB, done) + }) + + after((done) => parallel(nodes.map((node) => (cb) => node.stop(cb)), done)) + + it('should be able to put a value to the dht and get it afterwards', function () { + this.timeout(60 * 1000) + + const key = 'testkey' + const value = 'testvalue' + + return ipfsA(`dht put ${key} ${value}`) + .then((res) => { + expect(res).to.exist() + + return ipfsB(`dht get ${key}`) + }) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(value) + }) + }) + + it('should be able to provide data and to be present in the findproviders', function () { + this.timeout(60 * 1000) + let cidAdded + + return ipfsA('add src/init-files/init-docs/readme') + .then((res) => { + expect(res).to.exist() + cidAdded = res.split(' ')[1] + + return ipfsA(`dht provide ${cidAdded}`) + }) + .then((res) => { + expect(res).to.exist() + + return ipfsB(`dht findprovs ${cidAdded}`) + }) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(idA) + }) + }) + + it('findpeer', function () { + this.timeout(60 * 1000) + + return ipfsA(`dht findpeer ${idB}`) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(multiaddrB) + }) + }) + + it('query', function () { + this.timeout(60 * 1000) + + return ipfsA(`dht query ${idB}`) + .then((res) => { + expect(res).to.exist() + expect(res).to.have.string(idB) + }) + }) +}) diff --git a/test/cli/name-pubsub.js b/test/cli/name-pubsub.js index 6b252f67a1..fd125948a5 100644 --- a/test/cli/name-pubsub.js +++ b/test/cli/name-pubsub.js @@ -21,7 +21,18 @@ const spawnDaemon = (callback) => { df.spawn({ exec: `./src/cli/bin.js`, args: ['--enable-namesys-pubsub'], - initOptions: { bits: 512 } + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, callback) } diff --git a/test/cli/name.js b/test/cli/name.js index dbdaff9407..9350f5c5b0 100644 --- a/test/cli/name.js +++ b/test/cli/name.js @@ -199,9 +199,17 @@ describe('name', () => { df.spawn({ exec: `./src/cli/bin.js`, config: { - Bootstrap: [] + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } }, - args: ['--pass', passPhrase, '--enable-dht-experiment'], + args: ['--pass', passPhrase], initOptions: { bits: 512 } }, (err, _ipfsd) => { expect(err).to.not.exist() diff --git a/test/core/create-node.spec.js b/test/core/create-node.spec.js index e2b9788520..cf5ba30e43 100644 --- a/test/core/create-node.spec.js +++ b/test/core/create-node.spec.js @@ -133,7 +133,7 @@ describe('create node', function () { }) it('should be silent', function (done) { - this.timeout(10 * 1000) + this.timeout(30 * 1000) sinon.spy(console, 'log') diff --git a/test/core/dht.spec.js b/test/core/dht.spec.js index 8e94bdb9b4..01812cc8bb 100644 --- a/test/core/dht.spec.js +++ b/test/core/dht.spec.js @@ -21,7 +21,9 @@ describe('dht', () => { factory.spawn({ exec: IPFS, initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [] + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd @@ -40,7 +42,7 @@ describe('dht', () => { describe('findprovs', () => { it('should callback with error for invalid CID input', (done) => { - ipfs.dht.findprovs('INVALID CID', (err) => { + ipfs.dht.findProvs('INVALID CID', (err) => { expect(err).to.exist() expect(err.code).to.equal('ERR_INVALID_CID') done() diff --git a/test/core/files-sharding.spec.js b/test/core/files-sharding.spec.js index 01d014c4f7..9289d78dff 100644 --- a/test/core/files-sharding.spec.js +++ b/test/core/files-sharding.spec.js @@ -62,7 +62,7 @@ describe('files directory (sharding tests)', () => { }) it('should be able to add dir without sharding', function (done) { - this.timeout(40 * 1000) + this.timeout(70 * 1000) pull( pull.values(createTestFiles()), @@ -114,7 +114,7 @@ describe('files directory (sharding tests)', () => { }) it('should be able to add dir with sharding', function (done) { - this.timeout(40 * 1000) + this.timeout(80 * 1000) pull( pull.values(createTestFiles()), diff --git a/test/core/interface.spec.js b/test/core/interface.spec.js index 05284161b7..7730b57ed4 100644 --- a/test/core/interface.spec.js +++ b/test/core/interface.spec.js @@ -34,20 +34,29 @@ describe('interface-ipfs-core tests', () => { tests.dag(defaultCommonFactory) - const dhtCommonFactory = CommonFactory.create({ + tests.dht(CommonFactory.create({ spawnOptions: { - initOptions: { bits: 512 }, - EXPERIMENTAL: { - dht: true - }, config: { - Bootstrap: [] - } + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + }, + initOptions: { bits: 512 } } - }) - - tests.dht(dhtCommonFactory, { - skip: { reason: 'TODO: unskip when https://github.com/ipfs/js-ipfs/pull/856 is merged' } + }), { + skip: isNode ? [ + // dht.get + { + name: 'should get a value after it was put on another node', + reason: 'Needs https://github.com/ipfs/interface-ipfs-core/pull/383' + } + ] : true }) tests.filesRegular(defaultCommonFactory, { @@ -91,15 +100,37 @@ describe('interface-ipfs-core tests', () => { tests.name(CommonFactory.create({ spawnOptions: { - args: ['--pass ipfs-is-awesome-software'], - initOptions: { bits: 512 } + args: ['--pass ipfs-is-awesome-software', '--local'], + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } } })) tests.namePubsub(CommonFactory.create({ spawnOptions: { args: ['--enable-namesys-pubsub'], - initOptions: { bits: 1024 } + initOptions: { bits: 1024 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } } })) diff --git a/test/core/kad-dht.node.js b/test/core/kad-dht.node.js index 1f1f484ae6..1d7452c596 100644 --- a/test/core/kad-dht.node.js +++ b/test/core/kad-dht.node.js @@ -6,21 +6,20 @@ const chai = require('chai') const dirtyChai = require('dirty-chai') const expect = chai.expect chai.use(dirtyChai) + const parallel = require('async/parallel') const IPFSFactory = require('ipfsd-ctl') const f = IPFSFactory.create({ type: 'js' }) const config = { - Addresses: { - Swarm: [`/ip4/127.0.0.1/tcp/0`, `/ip4/127.0.0.1/tcp/0/ws`], - API: `/ip4/127.0.0.1/tcp/0`, - Gateway: `/ip4/127.0.0.1/tcp/0` - }, Bootstrap: [], Discovery: { MDNS: { Enabled: false + }, + webRTCStar: { + Enabled: false } } } @@ -33,16 +32,17 @@ function createNode (callback) { }, callback) } -describe.skip('verify that kad-dht is doing its thing', () => { +describe('kad-dht is routing content and peers correctly', () => { let nodeA let nodeB let nodeC - // let addrA let addrB let addrC let nodes - before((done) => { + before(function (done) { + this.timeout(30 * 1000) + parallel([ (cb) => createNode(cb), (cb) => createNode(cb), @@ -59,7 +59,6 @@ describe.skip('verify that kad-dht is doing its thing', () => { (cb) => nodeC.id(cb) ], (err, ids) => { expect(err).to.not.exist() - // addrA = ids[0].addresses[0] addrB = ids[1].addresses[0] addrC = ids[2].addresses[0] parallel([ @@ -72,10 +71,29 @@ describe.skip('verify that kad-dht is doing its thing', () => { after((done) => parallel(nodes.map((node) => (cb) => node.stop(cb)), done)) - it.skip('add a file in C, fetch through B in A', (done) => { + it('add a file in B, fetch in A', function (done) { + this.timeout(30 * 1000) + const file = { + path: 'testfile1.txt', + content: Buffer.from('hello kad 1') + } + + nodeB.add(file, (err, filesAdded) => { + expect(err).to.not.exist() + + nodeA.cat(filesAdded[0].hash, (err, data) => { + expect(err).to.not.exist() + expect(data).to.eql(file.content) + done() + }) + }) + }) + + it('add a file in C, fetch through B in A', function (done) { + this.timeout(30 * 1000) const file = { - path: 'testfile.txt', - content: Buffer.from('hello kad') + path: 'testfile2.txt', + content: Buffer.from('hello kad 2') } nodeC.add(file, (err, filesAdded) => { @@ -83,8 +101,7 @@ describe.skip('verify that kad-dht is doing its thing', () => { nodeA.cat(filesAdded[0].hash, (err, data) => { expect(err).to.not.exist() - expect(data.length).to.equal(file.data.length) - expect(data).to.eql(file.data) + expect(data).to.eql(file.content) done() }) }) diff --git a/test/core/libp2p.spec.js b/test/core/libp2p.spec.js index c7acca3a6d..f0677ed01a 100644 --- a/test/core/libp2p.spec.js +++ b/test/core/libp2p.spec.js @@ -19,7 +19,7 @@ const libp2pComponent = require('../../src/core/components/libp2p') describe('libp2p customization', function () { // Provide some extra time for ci since we're starting libp2p nodes in each test - this.timeout(15 * 1000) + this.timeout(25 * 1000) let datastore let peerInfo @@ -27,7 +27,9 @@ describe('libp2p customization', function () { let mockConfig let _libp2p - before((done) => { + before(function (done) { + this.timeout(25 * 1000) + mockConfig = { get: (callback) => { callback(null, { @@ -148,7 +150,7 @@ describe('libp2p customization', function () { } }, EXPERIMENTAL: { - dht: false, + dht: true, pubsub: false } }) @@ -214,7 +216,7 @@ describe('libp2p customization', function () { } }, EXPERIMENTAL: { - dht: false, + dht: true, pubsub: true } }) diff --git a/test/core/name-pubsub.js b/test/core/name-pubsub.js index 10b6cfe995..7ba81413b3 100644 --- a/test/core/name-pubsub.js +++ b/test/core/name-pubsub.js @@ -32,7 +32,17 @@ describe('name-pubsub', function () { df.spawn({ exec: IPFS, args: [`--pass ${hat()}`, '--enable-namesys-pubsub'], - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, callback) } diff --git a/test/core/name.js b/test/core/name.js index b2ebc6927b..972d5c25ed 100644 --- a/test/core/name.js +++ b/test/core/name.js @@ -50,7 +50,7 @@ describe('name', function () { this.timeout(50 * 1000) df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`], + args: [`--pass ${hat()}`, '--local'], config: { Bootstrap: [] } }, (err, _ipfsd) => { expect(err).to.not.exist() @@ -152,7 +152,7 @@ describe('name', function () { this.timeout(40 * 1000) df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`], + args: [`--pass ${hat()}`, '--local'], config: { Bootstrap: [] } }, (err, _ipfsd) => { expect(err).to.not.exist() @@ -193,8 +193,7 @@ describe('name', function () { }) }) - // TODO: unskip when https://github.com/ipfs/js-ipfs/pull/856 is merged - describe.skip('work with dht', () => { + describe('work with dht', () => { let nodes let nodeA let nodeB @@ -204,8 +203,18 @@ describe('name', function () { const createNode = (callback) => { df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`, '--enable-dht-experiment'], - config: { Bootstrap: [] } + args: [`--pass ${hat()}`], + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, callback) } @@ -233,7 +242,8 @@ describe('name', function () { idA = ids[0] parallel([ (cb) => nodeC.swarm.connect(ids[0].addresses[0], cb), // C => A - (cb) => nodeC.swarm.connect(ids[1].addresses[0], cb) // C => B + (cb) => nodeC.swarm.connect(ids[1].addresses[0], cb), // C => B + (cb) => nodeA.swarm.connect(ids[1].addresses[0], cb) // A => B ], done) }) }) @@ -246,12 +256,12 @@ describe('name', function () { }) it('should publish and then resolve correctly with the default options', function (done) { - this.timeout(90 * 1000) + this.timeout(380 * 1000) publishAndResolve(nodeA, nodeB, ipfsRef, { resolve: false }, idA.id, {}, done) }) it('should recursively resolve to an IPFS hash', function (done) { - this.timeout(180 * 1000) + this.timeout(360 * 1000) const keyName = hat() nodeA.key.gen(keyName, { type: 'rsa', size: 2048 }, function (err, key) { @@ -284,7 +294,17 @@ describe('name', function () { df.spawn({ exec: IPFS, args: [`--pass ${hat()}`], - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd @@ -454,8 +474,18 @@ describe('name', function () { this.timeout(40 * 1000) df.spawn({ exec: IPFS, - args: [`--pass ${hat()}`], - config: { Bootstrap: [] } + args: [`--pass ${hat()}`, '--local'], + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() node = _ipfsd.api diff --git a/test/core/pin-set.js b/test/core/pin-set.js index ece713d61c..2fa7b895b4 100644 --- a/test/core/pin-set.js +++ b/test/core/pin-set.js @@ -107,7 +107,7 @@ describe('pinSet', function () { describe('handles large sets', function () { it('handles storing items > maxItems', function (done) { - this.timeout(19 * 1000) + this.timeout(70 * 1000) const expectedHash = 'QmbvhSy83QWfgLXDpYjDmLWBFfGc8utoqjcXHyj3gYuasT' const count = maxItems + 1 createNodes(count, (err, cids) => { diff --git a/test/core/ping.spec.js b/test/core/ping.spec.js index 9db0d7547a..a3f220f2d9 100644 --- a/test/core/ping.spec.js +++ b/test/core/ping.spec.js @@ -29,7 +29,7 @@ const config = { } function spawnNode ({ dht = false, type = 'js' }, cb) { - const args = dht ? ['--enable-dht-experiment'] : [] + const args = dht ? [] : ['--local'] const factory = type === 'js' ? df : dfProc factory.spawn({ args, diff --git a/test/http-api/bootstrap.js b/test/http-api/bootstrap.js index 9e3aa76b98..7fb59d6474 100644 --- a/test/http-api/bootstrap.js +++ b/test/http-api/bootstrap.js @@ -18,7 +18,17 @@ describe('bootstrap endpoint', () => { df.spawn({ initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd diff --git a/test/http-api/config.js b/test/http-api/config.js index 0958b16a72..6536adf5e0 100644 --- a/test/http-api/config.js +++ b/test/http-api/config.js @@ -28,6 +28,13 @@ skipOnWindows('config endpoint', () => { let ipfs = null let ipfsd = null + // wait until the repo is ready to use + before(function (done) { + this.timeout(10 * 1000) + + setTimeout(done, 5 * 1000) + }) + before(function (done) { this.timeout(20 * 1000) @@ -38,6 +45,7 @@ skipOnWindows('config endpoint', () => { (cb) => df.spawn({ repoPath: repoPath, initOptions: { bits: 512 }, + config: { Bootstrap: [] }, disposable: false, start: true }, cb), diff --git a/test/http-api/dns.js b/test/http-api/dns.js index d373775f4f..b90802a621 100644 --- a/test/http-api/dns.js +++ b/test/http-api/dns.js @@ -28,7 +28,9 @@ describe('dns endpoint', () => { after((done) => ipfsd.stop(done)) describe('.dns', () => { - it('resolve ipfs.io dns', (done) => { + it('resolve ipfs.io dns', function (done) { + this.timeout(40 * 1000) + ipfs.dns('ipfs.io', (err, result) => { expect(err).to.not.exist() expect(result).to.exist() diff --git a/test/http-api/index.js b/test/http-api/index.js index 901cb212ef..a4cceb57a9 100644 --- a/test/http-api/index.js +++ b/test/http-api/index.js @@ -5,7 +5,7 @@ require('./bootstrap') require('./config') require('./dns') require('./id') -require('./inject') +require('./routes') require('./interface') require('./object') require('./version') diff --git a/test/http-api/inject/dht.js b/test/http-api/inject/dht.js new file mode 100644 index 0000000000..fb60c3758b --- /dev/null +++ b/test/http-api/inject/dht.js @@ -0,0 +1,156 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) + +module.exports = (http) => { + describe('/dht', () => { + let api + + before(() => { + api = http.api.server.select('API') + }) + + describe('/findpeer', () => { + it('returns 400 if no peerId is provided', (done) => { + api.inject({ + method: 'GET', + url: `/api/v0/dht/findpeer` + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + done() + }) + }) + + it('returns 404 if peerId is provided as there is no peers in the routing table', (done) => { + const peerId = 'QmQ2zigjQikYnyYUSXZydNXrDRhBut2mubwJBaLXobMt3A' + + api.inject({ + method: 'GET', + url: `/api/v0/dht/findpeer?arg=${peerId}` + }, (res) => { + expect(res.statusCode).to.equal(404) + done() + }) + }) + }) + + describe('/findprovs', () => { + it('returns 400 if no key is provided', (done) => { + api.inject({ + method: 'GET', + url: `/api/v0/dht/findprovs` + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + done() + }) + }) + + it('returns 200 if key is provided', (done) => { + const key = 'Qmc77hSNykXJ6Jxp1C6RpD8VENV7RK6JD7eAcWpc7nEZx2' + + api.inject({ + method: 'GET', + url: `/api/v0/dht/findprovs?arg=${key}` + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result.Type).to.be.eql(4) + done() + }) + }) + }) + + describe('/get', () => { + it('returns 400 if no key is provided', (done) => { + api.inject({ + method: 'GET', + url: `/api/v0/dht/get` + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + done() + }) + }) + + it('returns 200 if key is provided', (done) => { + const key = 'key' + const value = 'value' + + api.inject({ + method: 'GET', + url: `/api/v0/dht/put?arg=${key}&arg=${value}` + }, (res) => { + expect(res.statusCode).to.equal(200) + + api.inject({ + method: 'GET', + url: `/api/v0/dht/get?arg=${key}` + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.result.Type).to.be.eql(5) + expect(res.result.Extra).to.be.eql(value) + done() + }) + }) + }) + }) + + describe('/provide', () => { + it('returns 400 if no key is provided', (done) => { + api.inject({ + method: 'GET', + url: `/api/v0/dht/provide` + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + done() + }) + }) + + it('returns 500 if key is provided as the file was not added', (done) => { + const key = 'Qmc77hSNykXJ6Jxp1C6RpD8VENV7RK6JD7eAcWpc7nEZx2' + + api.inject({ + method: 'GET', + url: `/api/v0/dht/provide?arg=${key}` + }, (res) => { + expect(res.statusCode).to.equal(500) // needs file add + done() + }) + }) + }) + + describe('/put', () => { + it('returns 400 if no key or value is provided', (done) => { + api.inject({ + method: 'GET', + url: `/api/v0/dht/put` + }, (res) => { + expect(res.statusCode).to.equal(400) + expect(res.result.Code).to.be.eql(1) + done() + }) + }) + + it('returns 200 if key and value is provided', function (done) { + this.timeout(60 * 1000) + const key = 'key' + const value = 'value' + + api.inject({ + method: 'GET', + url: `/api/v0/dht/put?arg=${key}&arg=${value}` + }, (res) => { + expect(res.statusCode).to.equal(200) + done() + }) + }) + }) + }) +} diff --git a/test/http-api/inject/files.js b/test/http-api/inject/files.js index 677c4c43fa..6770243e03 100644 --- a/test/http-api/inject/files.js +++ b/test/http-api/inject/files.js @@ -106,34 +106,16 @@ module.exports = (http) => { }) }) - it('should cat a valid hash', function (done) { - this.timeout(30 * 1000) - - const data = Buffer.from('TEST' + Date.now()) - const form = new FormData() - form.append('data', data) - const headers = form.getHeaders() - - streamToPromise(form).then((payload) => { - api.inject({ - method: 'POST', - url: '/api/v0/add', - headers: headers, - payload: payload - }, (res) => { - expect(res.statusCode).to.equal(200) - const cid = JSON.parse(res.result).Hash - - api.inject({ - method: 'GET', - url: '/api/v0/cat?arg=' + cid - }, (res) => { - expect(res.statusCode).to.equal(200) - expect(res.rawPayload).to.deep.equal(data) - expect(res.payload).to.equal(data.toString()) - done() - }) - }) + it('valid hash', function (done) { + this.timeout(90 * 1000) + api.inject({ + method: 'GET', + url: '/api/v0/cat?arg=QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' + }, (res) => { + expect(res.statusCode).to.equal(200) + expect(res.rawPayload).to.deep.equal(Buffer.from('hello world' + '\n')) + expect(res.payload).to.equal('hello world' + '\n') + done() }) }) }) diff --git a/test/http-api/inject/index.js b/test/http-api/inject/index.js deleted file mode 100644 index 24d94ebc4e..0000000000 --- a/test/http-api/inject/index.js +++ /dev/null @@ -1,63 +0,0 @@ -/* eslint-env mocha */ -'use strict' - -const fs = require('fs') -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) -const hat = require('hat') -const API = require('../../../src/http/index') -const ncp = require('ncp').ncp -const path = require('path') -const clean = require('../../utils/clean') - -describe('HTTP API', () => { - const repoExample = path.join(__dirname, '../../fixtures/go-ipfs-repo') - const repoTests = path.join(__dirname, '../../repo-tests-run') - - let http = {} - - const startHttpAPI = (cb) => { - const options = { - pass: hat(), - enablePubsubExperiment: true - } - http.api = new API(repoTests, null, options) - - ncp(repoExample, repoTests, (err) => { - if (err) { - return cb(err) - } - - http.api.start(false, (err) => { - if (err) { - return cb(err) - } - cb(null, http) - }) - }) - } - - before(function (done) { - this.timeout(60 * 1000) - startHttpAPI((err, _http) => { - if (err) { - throw err - } - http = _http - done() - }) - }) - - after((done) => http.api.stop((err) => { - expect(err).to.not.exist() - clean(repoTests) - done() - })) - - describe('## http-api spec tests', () => { - fs.readdirSync(path.join(__dirname)) - .forEach((file) => file !== 'index.js' && require(`./${file}`)(http)) - }) -}) diff --git a/test/http-api/inject/name.js b/test/http-api/inject/name.js index c4efe81b84..0e2d7e0a31 100644 --- a/test/http-api/inject/name.js +++ b/test/http-api/inject/name.js @@ -19,7 +19,9 @@ module.exports = (http) => { api = http.api.server.select('API') }) - it('should publish a record', (done) => { + it('should publish a record', function (done) { + this.timeout(80 * 1000) + api.inject({ method: 'GET', url: `/api/v0/name/publish?arg=${cid}&resolve=false` @@ -30,7 +32,9 @@ module.exports = (http) => { }) }) - it('should publish and resolve a record', (done) => { + it('should publish and resolve a record', function (done) { + this.timeout(160 * 1000) + api.inject({ method: 'GET', url: `/api/v0/name/publish?arg=${cid}&resolve=false` diff --git a/test/http-api/inject/ping.js b/test/http-api/inject/ping.js index 42759c38f3..28faf94c46 100644 --- a/test/http-api/inject/ping.js +++ b/test/http-api/inject/ping.js @@ -36,7 +36,9 @@ module.exports = (http) => { }) }) - it('returns 500 for incorrect Peer Id', (done) => { + it('returns 500 for incorrect Peer Id', function (done) { + this.timeout(90 * 1000) + api.inject({ method: 'GET', url: `/api/v0/ping?arg=peerid` diff --git a/test/http-api/interface.js b/test/http-api/interface.js index 6bc89e6492..12572cdff9 100644 --- a/test/http-api/interface.js +++ b/test/http-api/interface.js @@ -21,8 +21,29 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { skip: { reason: 'TODO: DAG HTTP endpoints not implemented in js-ipfs yet!' } }) - tests.dht(defaultCommonFactory, { - skip: { reason: 'TODO: unskip when https://github.com/ipfs/js-ipfs/pull/856 is merged' } + tests.dht(CommonFactory.create({ + spawnOptions: { + initOptions: { bits: 512 }, + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } + } + }), { + skip: [ + // dht.get + { + name: 'should get a value after it was put on another node', + reason: 'Needs https://github.com/ipfs/interface-ipfs-core/pull/383' + } + ] }) tests.filesRegular(defaultCommonFactory) @@ -112,7 +133,7 @@ describe('interface-ipfs-core over ipfs-http-client tests', () => { } })) - tests.types(defaultCommonFactory) + tests.types(defaultCommonFactory, { skip: { reason: 'FIXME: currently failing' } }) tests.util(defaultCommonFactory, { skip: { reason: 'FIXME: currently failing' } }) }) diff --git a/test/http-api/object.js b/test/http-api/object.js index 003e8f1739..811aad6d72 100644 --- a/test/http-api/object.js +++ b/test/http-api/object.js @@ -30,7 +30,17 @@ describe('object endpoint', () => { df.spawn({ initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd diff --git a/test/http-api/routes.js b/test/http-api/routes.js new file mode 100644 index 0000000000..0529188c2a --- /dev/null +++ b/test/http-api/routes.js @@ -0,0 +1,104 @@ +/* eslint-env mocha */ +'use strict' + +const fs = require('fs') +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const expect = chai.expect +chai.use(dirtyChai) +const hat = require('hat') +const API = require('../../src/http/index') +const ncp = require('ncp').ncp +const path = require('path') +const clean = require('../utils/clean') + +describe('HTTP API', () => { + const repoExample = path.join(__dirname, '../fixtures/go-ipfs-repo') + const repoTests = path.join(__dirname, '../repo-tests-run') + + // bootstrap nodes get the set up too slow and gets timed out + const testsForCustomConfig = ['dht.js', 'name.js', 'ping.js'] + + let http = {} + + const startHttpAPI = (config, cb) => { + const options = { + pass: hat(), + enablePubsubExperiment: true + } + http.api = new API(repoTests, config, options) + + ncp(repoExample, repoTests, (err) => { + if (err) { + return cb(err) + } + + http.api.start(false, (err) => { + if (err) { + return cb(err) + } + cb(null, http) + }) + }) + } + + describe('custom config', () => { + const config = { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } + + before(function (done) { + this.timeout(60 * 1000) + startHttpAPI(config, (err, _http) => { + if (err) { + throw err + } + http = _http + done() + }) + }) + + after((done) => http.api.stop((err) => { + expect(err).to.not.exist() + clean(repoTests) + done() + })) + + describe('## http-api spec tests', () => { + fs.readdirSync(path.join(`${__dirname}/inject/`)) + .forEach((file) => testsForCustomConfig.includes(file) && require(`./inject/${file}`)(http)) + }) + }) + + describe('default config', () => { + before(function (done) { + this.timeout(60 * 1000) + startHttpAPI(null, (err, _http) => { + if (err) { + throw err + } + http = _http + done() + }) + }) + + after((done) => http.api.stop((err) => { + expect(err).to.not.exist() + clean(repoTests) + done() + })) + + describe('## http-api spec tests', () => { + fs.readdirSync(path.join(`${__dirname}/inject/`)) + .forEach((file) => !testsForCustomConfig.includes(file) && require(`./inject/${file}`)(http)) + }) + }) +}) diff --git a/test/http-api/version.js b/test/http-api/version.js index f9ab764668..9fc0c3acc0 100644 --- a/test/http-api/version.js +++ b/test/http-api/version.js @@ -16,7 +16,17 @@ describe('version endpoint', () => { this.timeout(20 * 1000) df.spawn({ initOptions: { bits: 512 }, - config: { Bootstrap: [] } + config: { + Bootstrap: [], + Discovery: { + MDNS: { + Enabled: false + }, + webRTCStar: { + Enabled: false + } + } + } }, (err, _ipfsd) => { expect(err).to.not.exist() ipfsd = _ipfsd