Merge branch 'master' into feat/ping

Author: alanshaw

README.md

@@ -43,7 +43,7 @@ You can check the development status at the [Waffle Board](https://waffle.io/ipf
 **Please read this:** DHT (automatic content discovery) and Circuit Relay (pierce through NATs and dial between any node in the network) are two fundamental pieces that are not finalized yet. There are multiple applications that can be built without these two services but nevertheless they are fundamental to get that magic IPFS experience. If you want to track progress or contribute, please follow:
 
 - DHT: https://github.com/ipfs/js-ipfs/pull/856
-- Relay: https://github.com/ipfs/js-ipfs/pull/1063
+- ✅ Relay: https://github.com/ipfs/js-ipfs/pull/1063
 
 [**`Weekly Dev Calls`**](https://github.com/ipfs/js-ipfs/issues/1179)
 
@@ -413,8 +413,8 @@ The core API is grouped into several areas:
 - [dht (not implemented, yet!)](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/)
 
 - [pubsub](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md)
-  - [`ipfs.pubsub.subscribe(topic, options, handler, callback)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubsubscribe)
-  - [`ipfs.pubsub.unsubscribe(topic, handler)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubunsubscribe)
+  - [`ipfs.pubsub.subscribe(topic, handler, options, callback)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubsubscribe)
+  - [`ipfs.pubsub.unsubscribe(topic, handler, callback)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubunsubscribe)
   - [`ipfs.pubsub.publish(topic, data, callback)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubpublish)
   - [`ipfs.pubsub.ls(topic, callback)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubls)
   - [`ipfs.pubsub.peers(topic, callback)`](https://github.com/ipfs/interface-ipfs-core/tree/master/SPEC/PUBSUB.md#pubsubpeers)

package.json

@@ -73,8 +73,8 @@
     "expose-loader": "^0.7.5",
     "form-data": "^2.3.2",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.64.3",
-    "ipfsd-ctl": "~0.33.2",
+    "interface-ipfs-core": "~0.65.5",
+    "ipfsd-ctl": "~0.34.0",
     "lodash": "^4.17.10",
     "mocha": "^5.1.1",
     "ncp": "^2.0.0",
@@ -105,30 +105,30 @@
     "hapi-set-header": "^1.0.2",
     "hoek": "^5.0.3",
     "human-to-milliseconds": "^1.0.0",
-    "ipfs-api": "^20.2.1",
+    "ipfs-api": "^21.0.0",
     "ipfs-bitswap": "~0.20.0",
     "ipfs-block": "~0.7.1",
     "ipfs-block-service": "~0.14.0",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "0.20.0",
     "ipfs-unixfs": "~0.1.14",
     "ipfs-unixfs-engine": "~0.29.0",
-    "ipld": "^0.17.0",
-    "ipld-dag-cbor": "^0.12.0",
+    "ipld": "~0.17.0",
+    "ipld-dag-cbor": "~0.12.0",
     "ipld-dag-pb": "~0.14.4",
-    "is-ipfs": "^0.3.2",
+    "is-ipfs": "~0.3.2",
     "is-pull-stream": "0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^13.2.0",
     "joi-browser": "^13.0.1",
     "joi-multiaddr": "^2.0.0",
-    "libp2p": "^0.20.4",
-    "libp2p-circuit": "^0.2.0",
+    "libp2p": "~0.20.4",
+    "libp2p-circuit": "~0.2.0",
     "libp2p-floodsub": "~0.15.0",
     "libp2p-kad-dht": "~0.10.0",
     "libp2p-keychain": "~0.3.1",
     "libp2p-mdns": "~0.11.0",
-    "libp2p-mplex": "^0.7.0",
+    "libp2p-mplex": "~0.7.0",
     "libp2p-railing": "~0.8.1",
     "libp2p-secio": "~0.10.0",
     "libp2p-tcp": "~0.12.0",
@@ -144,7 +144,7 @@
     "mime-types": "^2.1.18",
     "mkdirp": "~0.5.1",
     "multiaddr": "^5.0.0",
-    "multibase": "^0.4.0",
+    "multibase": "~0.4.0",
     "multihashes": "~0.4.13",
     "once": "^1.4.0",
     "path-exists": "^3.0.0",

src/cli/bin.js

@@ -5,6 +5,8 @@
 const yargs = require('yargs')
 const updateNotifier = require('update-notifier')
 const readPkgUp = require('read-pkg-up')
+const fs = require('fs')
+const path = require('path')
 const utils = require('./utils')
 const print = utils.print
 
@@ -16,6 +18,10 @@ updateNotifier({
 
 const args = process.argv.slice(2)
 
+// Determine if the first argument is a sub-system command
+const commandNames = fs.readdirSync(path.join(__dirname, 'commands'))
+const isCommand = commandNames.includes(`${args[0]}.js`)
+
 const cli = yargs
   .option('silent', {
     desc: 'Write no output',
@@ -28,7 +34,14 @@ const cli = yargs
     type: 'string',
     default: ''
   })
-  .commandDir('commands')
+  .commandDir('commands', {
+    // Only include the commands for the sub-system we're using, or include all
+    // if no sub-system command has been passed.
+    include (path, filename) {
+      if (!isCommand) return true
+      return `${args[0]}.js` === filename
+    }
+  })
   .epilog(utils.ipfsPathHelp)
   .demandCommand(1)
   .fail((msg, err, yargs) => {
@@ -43,16 +56,19 @@ const cli = yargs
     yargs.showHelp()
   })
 
-// NOTE: This creates an alias of
-// `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
-// This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
-const addCmd = require('./commands/files/add')
-const catCmd = require('./commands/files/cat')
-const getCmd = require('./commands/files/get')
-const aliases = [addCmd, catCmd, getCmd]
-aliases.forEach((alias) => {
-  cli.command(alias.command, alias.describe, alias.builder, alias.handler)
-})
+// If not a sub-system command then load the top level aliases
+if (!isCommand) {
+  // NOTE: This creates an alias of
+  // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
+  // This will stay until https://github.com/ipfs/specs/issues/98 is resolved.
+  const addCmd = require('./commands/files/add')
+  const catCmd = require('./commands/files/cat')
+  const getCmd = require('./commands/files/get')
+  const aliases = [addCmd, catCmd, getCmd]
+  aliases.forEach((alias) => {
+    cli.command(alias.command, alias.describe, alias.builder, alias.handler)
+  })
+}
 
 // Need to skip to avoid locking as these commands
 // don't require a daemon

src/cli/utils.js

@@ -2,9 +2,7 @@
 
 const fs = require('fs')
 const os = require('os')
-const APIctl = require('ipfs-api')
 const multiaddr = require('multiaddr')
-const IPFS = require('../core')
 const path = require('path')
 const debug = require('debug')
 const log = debug('cli')
@@ -35,6 +33,8 @@ function getAPICtl (apiAddr) {
     const apiPath = path.join(exports.getRepoPath(), 'api')
     apiAddr = multiaddr(fs.readFileSync(apiPath).toString()).toString()
   }
+  // Required inline to reduce startup time
+  const APIctl = require('ipfs-api')
   return APIctl(apiAddr)
 }
 
@@ -43,6 +43,8 @@ exports.getIPFS = (argv, callback) => {
     return callback(null, getAPICtl(argv.api), (cb) => cb())
   }
 
+  // Required inline to reduce startup time
+  const IPFS = require('../core')
   const node = new IPFS({
     repo: exports.getRepoPath(),
     init: false,

src/core/components/pubsub.js

@@ -1,13 +1,13 @@
 'use strict'
 
 const promisify = require('promisify-es6')
+const setImmediate = require('async/setImmediate')
 
 module.exports = function pubsub (self) {
   return {
-    subscribe: (topic, options, handler, callback) => {
+    subscribe: (topic, handler, options, callback) => {
       if (typeof options === 'function') {
-        callback = handler
-        handler = options
+        callback = options
         options = {}
       }
 
@@ -20,13 +20,19 @@ module.exports = function pubsub (self) {
             resolve()
           })
         })
-      } else {
-        self._libp2pNode.pubsub.subscribe(topic, options, handler, callback)
       }
+
+      self._libp2pNode.pubsub.subscribe(topic, options, handler, callback)
     },
 
-    unsubscribe: (topic, handler) => {
+    unsubscribe: (topic, handler, callback) => {
       self._libp2pNode.pubsub.unsubscribe(topic, handler)
+
+      if (!callback) {
+        return Promise.resolve()
+      }
+
+      setImmediate(() => callback())
     },
 
     publish: promisify((topic, data, callback) => {

src/http/api/resources/pubsub.js

@@ -33,16 +33,13 @@ exports.subscribe = {
     res.write('{}\n')
 
     const unsubscribe = () => {
-      ipfs.pubsub.unsubscribe(topic, handler)
-      res.end()
+      ipfs.pubsub.unsubscribe(topic, handler, () => res.end())
     }
 
     request.once('disconnect', unsubscribe)
     request.once('finish', unsubscribe)
 
-    ipfs.pubsub.subscribe(topic, {
-      discover: discover
-    }, handler, (err) => {
+    ipfs.pubsub.subscribe(topic, handler, { discover: discover }, (err) => {
       if (err) {
         return reply(err)
       }

test/cli/daemon.js

@@ -5,8 +5,6 @@ const expect = require('chai').expect
 const clean = require('../utils/clean')
 const ipfsCmd = require('../utils/ipfs-exec')
 const isWindows = require('../utils/platforms').isWindows
-const pull = require('pull-stream')
-const toPull = require('stream-to-pull-stream')
 const os = require('os')
 const path = require('path')
 const hat = require('hat')
@@ -37,31 +35,20 @@ function testSignal (ipfs, sig) {
   }).then(() => {
     const proc = ipfs('daemon')
     return new Promise((resolve, reject) => {
-      pull(
-        toPull(proc.stdout),
-        pull.collect((err, res) => {
-          expect(err).to.not.exist()
-          const data = res.toString()
-          if (data.includes(`Daemon is ready`)) {
-            if (proc.kill(sig)) {
-              resolve()
-            } else {
-              reject(new Error(`Unable to ${sig} process`))
-            }
+      proc.stdout.on('data', (data) => {
+        if (data.toString().includes(`Daemon is ready`)) {
+          if (proc.kill(sig)) {
+            resolve()
+          } else {
+            reject(new Error(`Unable to ${sig} process`))
           }
-        })
-      )
-
-      pull(
-        toPull(proc.stderr),
-        pull.collect((err, res) => {
-          expect(err).to.not.exist()
-          const data = res.toString()
-          if (data.length > 0) {
-            reject(new Error(data))
-          }
-        })
-      )
+        }
+      })
+      proc.stderr.on('data', (data) => {
+        if (data.toString().length > 0) {
+          reject(new Error(data))
+        }
+      })
     })
   })
 }
@@ -79,6 +66,8 @@ describe('daemon', () => {
 
   skipOnWindows('do not crash if Addresses.Swarm is empty', function (done) {
     this.timeout(100 * 1000)
+    // These tests are flaky, but retrying 3 times seems to make it work 99% of the time
+    this.retries(3)
 
     ipfs('init').then(() => {
       return ipfs('config', 'Addresses', JSON.stringify({
@@ -87,10 +76,18 @@ describe('daemon', () => {
         Gateway: '/ip4/127.0.0.1/tcp/0'
       }), '--json')
     }).then(() => {
-      return ipfs('daemon')
-    }).then((res) => {
-      expect(res).to.have.string('Daemon is ready')
-      done()
+      const res = ipfs('daemon')
+      const timeout = setTimeout(() => {
+        done(new Error('Daemon did not get ready in time'))
+      }, 1000 * 120)
+      res.stdout.on('data', (data) => {
+        const line = data.toString()
+        if (line.includes('Daemon is ready')) {
+          clearTimeout(timeout)
+          res.kill()
+          done()
+        }
+      })
     }).catch(err => done(err))
   })