Merge pull request #797 from ipfs/feat/datastore

feat: Awesome datastore

Author: daviddias

.gitignore

@@ -33,3 +33,6 @@ node_modules
 
 lib
 dist
+test/test-data/go-ipfs-repo/LOCK
+test/test-data/go-ipfs-repo/LOG
+test/test-data/go-ipfs-repo/LOG.old

examples/dag/eth.js

@@ -30,7 +30,7 @@ createNode((err, ipfs) => {
       const cid = new CID(1, 'eth-block', multihash)
       // console.log(cid.toBaseEncodedString())
 
-      ipfs.block.put(new Block(data), cid, cb)
+      ipfs.block.put(new Block(data, cid), cb)
     })
   }, (err) => {
     if (err) {

gulpfile.js

@@ -4,7 +4,6 @@ const gulp = require('gulp')
 const parallel = require('async/parallel')
 const series = require('async/series')
 const createTempRepo = require('./test/utils/create-repo-node.js')
-const IPFS = require('./src/core')
 const HTTPAPI = require('./src/http-api')
 const leftPad = require('left-pad')
 
@@ -16,38 +15,26 @@ let nodes = []
 function spawnDaemon (num, callback) {
   num = leftPad(num, 3, 0)
 
-  const node = new IPFS({
-    repo: createTempRepo(),
-    init: {
-      bits: 1024
+  const config = {
+    Addresses: {
+      Swarm: [
+        `/ip4/127.0.0.1/tcp/10${num}`,
+        `/ip4/127.0.0.1/tcp/20${num}/ws`
+      ],
+      API: `/ip4/127.0.0.1/tcp/31${num}`,
+      Gateway: `/ip4/127.0.0.1/tcp/32${num}`
     },
-    start: false,
-    EXPERIMENTAL: {
-      pubsub: true
-    },
-    config: {
-      Addresses: {
-        Swarm: [
-          `/ip4/127.0.0.1/tcp/10${num}`,
-          `/ip4/127.0.0.1/tcp/20${num}/ws`
-        ],
-        API: `/ip4/127.0.0.1/tcp/31${num}`,
-        Gateway: `/ip4/127.0.0.1/tcp/32${num}`
-      },
-      Bootstrap: [],
-      Discovery: {
-        MDNS: {
-          Enabled: false
-        }
+    Bootstrap: [],
+    Discovery: {
+      MDNS: {
+        Enabled: false
       }
     }
-  })
+  }
 
-  node.on('init', () => {
-    const daemon = new HTTPAPI(node.repo.path())
-    nodes.push(daemon)
-    setTimeout(() => daemon.start(callback), 400)
-  })
+  const daemon = new HTTPAPI(createTempRepo(), config)
+  nodes.push(daemon)
+  daemon.start(true, callback)
 }
 
 gulp.task('libnode:start', (done) => {

package.json

@@ -63,19 +63,19 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs#readme",
   "devDependencies": {
-    "aegir": "^11.0.0",
+    "aegir": "^11.0.1",
     "buffer-loader": "0.0.1",
     "chai": "^3.5.0",
-    "delay": "^1.3.1",
+    "delay": "^2.0.0",
     "detect-node": "^2.0.3",
     "dir-compare": "^1.3.0",
     "dirty-chai": "^1.2.2",
-    "eslint-plugin-react": "^6.10.0",
-    "execa": "^0.6.1",
+    "eslint-plugin-react": "^6.10.3",
+    "execa": "^0.6.3",
     "expose-loader": "^0.7.3",
     "form-data": "^2.1.2",
     "gulp": "^3.9.1",
-    "interface-ipfs-core": "~0.25.0",
+    "interface-ipfs-core": "~0.26.0",
     "ipfsd-ctl": "~0.20.0",
     "left-pad": "^1.1.3",
     "lodash": "^4.17.4",
@@ -93,36 +93,35 @@
   "dependencies": {
     "async": "^2.1.5",
     "bl": "^1.2.0",
-    "boom": "^4.2.0",
+    "boom": "^4.3.0",
     "cids": "^0.4.2",
     "debug": "^2.6.3",
-    "fs-pull-blob-store": "~0.4.1",
+    "fsm-event": "^2.1.0",
     "glob": "^7.1.1",
     "hapi": "^16.1.0",
     "hapi-set-header": "^1.0.2",
     "hoek": "^4.1.0",
-    "idb-pull-blob-store": "~0.5.1",
-    "ipfs-api": "^12.1.7",
-    "ipfs-bitswap": "~0.9.5",
-    "ipfs-block": "~0.5.5",
-    "ipfs-block-service": "~0.8.3",
+    "ipfs-api": "^13.0.0",
+    "ipfs-bitswap": "~0.10.0",
+    "ipfs-block": "~0.6.0",
+    "ipfs-block-service": "~0.9.0",
     "ipfs-multipart": "~0.1.0",
-    "ipfs-repo": "~0.11.3",
+    "ipfs-repo": "~0.13.0",
     "ipfs-unixfs": "~0.1.11",
-    "ipfs-unixfs-engine": "~0.17.0",
-    "ipld-resolver": "~0.10.1",
+    "ipfs-unixfs-engine": "~0.18.0",
+    "ipld-resolver": "~0.11.0",
     "isstream": "^0.1.2",
-    "joi": "^10.2.2",
-    "libp2p-floodsub": "~0.7.4",
-    "libp2p-ipfs-browser": "~0.20.0",
-    "libp2p-ipfs-nodejs": "~0.20.0",
+    "joi": "^10.3.0",
+    "libp2p-floodsub": "~0.7.5",
+    "libp2p-ipfs-browser": "~0.20.1",
+    "libp2p-ipfs-nodejs": "~0.20.1",
     "lodash.flatmap": "^4.5.0",
     "lodash.get": "^4.4.2",
     "lodash.has": "^4.5.2",
     "lodash.set": "^4.3.2",
     "lodash.sortby": "^4.7.0",
     "lodash.values": "^4.3.0",
-    "mafmt": "^2.1.6",
+    "mafmt": "^2.1.7",
     "mkdirp": "^0.5.1",
     "multiaddr": "^2.2.2",
     "multihashes": "~0.4.4",
@@ -141,12 +140,13 @@
     "pull-zip": "^2.0.1",
     "read-pkg-up": "^2.0.0",
     "readable-stream": "1.1.14",
+    "safe-buffer": "^5.0.1",
     "stream-to-pull-stream": "^1.7.2",
     "tar-stream": "^1.5.2",
     "temp": "^0.8.3",
     "through2": "^2.0.3",
     "update-notifier": "^2.1.0",
-    "yargs": "^6.6.0"
+    "yargs": "7.0.2"
   },
   "contributors": [
     "Andrew de Andrade <[email protected]>",

src/cli/bin.js

@@ -5,6 +5,7 @@
 const yargs = require('yargs')
 const updateNotifier = require('update-notifier')
 const readPkgUp = require('read-pkg-up')
+const utils = require('./utils')
 
 const pkg = readPkgUp.sync({cwd: __dirname}).pkg
 updateNotifier({
@@ -14,7 +15,7 @@ updateNotifier({
 
 const cli = yargs
   .commandDir('commands')
-  .demand(1)
+  .demandCommand(1)
 
 // NOTE: This creates an alias of
 // `jsipfs files {add, get, cat}` to `jsipfs {add, get, cat}`.
@@ -27,9 +28,35 @@ aliases.forEach((alias) => {
   cli.command(alias.command, alias.describe, alias.builder, alias.handler)
 })
 
-// finalize cli setup
-cli // eslint-disable-line
-  .help()
-  .strict()
-  .completion()
-  .argv
+const args = process.argv.slice(2)
+
+// Need to skip to avoid locking as these commands
+// don't require a daemon
+if (args[0] === 'daemon' || args[0] === 'init') {
+  return cli
+    .help()
+    .strict(false)
+    .completion()
+    .parse(args)
+}
+
+utils.getIPFS((err, ipfs, cleanup) => {
+  if (err) {
+    throw err
+  }
+
+  // finalize cli setup
+  cli // eslint-disable-line
+    .help()
+    .strict(false)
+    .completion()
+    .parse(args, {
+      ipfs: ipfs
+    }, (err, argv, output) => {
+      cleanup(() => {
+        if (err) {
+          throw err
+        }
+      })
+    })
+})

src/cli/commands/bitswap/stat.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const utils = require('../../utils')
 const CID = require('cids')
 
 module.exports = {
@@ -10,34 +9,28 @@ module.exports = {
 
   builder: {},
 
-  handler () {
-    utils.getIPFS((err, ipfs) => {
+  handler (argv) {
+    argv.ipfs.bitswap.stat((err, stats) => {
       if (err) {
         throw err
       }
 
-      ipfs.bitswap.stat((err, stats) => {
-        if (err) {
-          throw err
-        }
-
-        stats.Wantlist = stats.Wantlist || []
-        stats.Wantlist = stats.Wantlist.map((entry) => {
-          const buf = new Buffer(entry.cid.hash.data)
-          const cid = new CID(entry.cid.version, entry.cid.codec, buf)
-          return cid.toBaseEncodedString()
-        })
-        stats.Peers = stats.Peers || []
+      stats.Wantlist = stats.Wantlist || []
+      stats.Wantlist = stats.Wantlist.map((entry) => {
+        const buf = new Buffer(entry.cid.hash.data)
+        const cid = new CID(entry.cid.version, entry.cid.codec, buf)
+        return cid.toBaseEncodedString()
+      })
+      stats.Peers = stats.Peers || []
 
-        console.log(`bitswap status
+      console.log(`bitswap status
   blocks received: ${stats.BlocksReceived}
   dup blocks received: ${stats.DupBlksReceived}
   dup data received: ${stats.DupDataReceived}B
   wantlist [${stats.Wantlist.length} keys]
     ${stats.Wantlist.join('\n    ')}
   partners [${stats.Peers.length}]
     ${stats.Peers.join('\n    ')}`)
-      })
     })
   }
 }

src/cli/commands/bitswap/unwant.js

@@ -1,19 +1,11 @@
 'use strict'
 
-const utils = require('../../utils')
-
 module.exports = {
   command: 'unwant <key>',
 
   describe: 'Remove a given block from your wantlist.',
 
   handler (argv) {
-    utils.getIPFS((err, ipfs) => {
-      if (err) {
-        throw err
-      }
-
-      throw new Error('Not implemented yet')
-    })
+    throw new Error('Not implemented yet')
   }
 }

src/cli/commands/bitswap/wantlist.js

@@ -1,7 +1,5 @@
 'use strict'
 
-const utils = require('../../utils')
-
 module.exports = {
   command: 'wantlist',
 
@@ -17,18 +15,12 @@ module.exports = {
 
   handler (argv) {
     // TODO: handle argv.peer
-    utils.getIPFS((err, ipfs) => {
+    argv.ipfs.bitswap.wantlist((err, res) => {
       if (err) {
         throw err
       }
-
-      ipfs.bitswap.wantlist((err, res) => {
-        if (err) {
-          throw err
-        }
-        res.Keys.forEach((cidStr) => {
-          console.log(cidStr)
-        })
+      res.Keys.forEach((cidStr) => {
+        console.log(cidStr)
       })
     })
   }

src/cli/commands/block/get.js

@@ -1,10 +1,6 @@
 'use strict'
 
-const utils = require('../../utils')
 const CID = require('cids')
-const debug = require('debug')
-const log = debug('cli:block')
-log.error = debug('cli:block:error')
 
 module.exports = {
   command: 'get <key>',
@@ -14,21 +10,15 @@ module.exports = {
   builder: {},
 
   handler (argv) {
-    utils.getIPFS((err, ipfs) => {
+    const cid = new CID(argv.key)
+
+    argv.ipfs.block.get(cid, (err, block) => {
       if (err) {
         throw err
       }
 
-      const cid = new CID(argv.key)
-
-      ipfs.block.get(cid, (err, block) => {
-        if (err) {
-          throw err
-        }
-
-        process.stdout.write(block.data)
-        process.stdout.write('\n')
-      })
+      process.stdout.write(block.data)
+      process.stdout.write('\n')
     })
   }
 }