refactor: update ipld formats, async/await mfs/unixfs & base32 cids (#2068)

This is part of the Awesome Endeavour: Async Iterators: #1670

Depends on

* [x] ipld/js-ipld-dag-pb#137
* [x] ipfs-inactive/interface-js-ipfs-core#473
* [x] ipfs-inactive/js-ipfs-http-client#1010
* [x] ipfs/js-ipfs-http-response#25

resolves #1995

BREAKING CHANGE: The default string encoding for version 1 CIDs has changed to `base32`.

IPLD formats have been updated to the latest versions. IPLD nodes returned by `ipfs.dag` and `ipfs.object` commands have significant breaking changes. If you are using these commands in your application you are likely to encounter the following changes to `dag-pb` nodes (the default node type that IPFS creates):

* `DAGNode` properties have been renamed as follows:
    * `data` => `Data`
    * `links` => `Links`
    * `size` => `size` (Note: no change)
* `DAGLink` properties have been renamed as follows:
    * `cid` => `Hash`
    * `name` => `Name`
    * `size` => `Tsize`

See CHANGELOGs for each IPLD format for it's respective changes, you can read more about the [`dag-pb` changes in the CHANGELOG](https://github.com/ipld/js-ipld-dag-pb/blob/master)

License: MIT
Signed-off-by: Alan Shaw <[email protected]>

Author: achingbrain

examples/traverse-ipld-graphs/get-path-accross-formats.js

@@ -17,19 +17,20 @@ createNode((err, ipfs) => {
   series([
     (cb) => {
       const someData = Buffer.from('capoeira')
+      let node
 
-      dagPB.DAGNode.create(someData, (err, node) => {
+      try {
+        node = dagPB.DAGNode.create(someData)
+      } catch (err) {
+        return cb(err)
+      }
+
+      ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => {
         if (err) {
           cb(err)
         }
-
-        ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => {
-          if (err) {
-            cb(err)
-          }
-          cidPBNode = cid
-          cb()
-        })
+        cidPBNode = cid
+        cb()
       })
     },
     (cb) => {

examples/traverse-ipld-graphs/tree.js

@@ -17,19 +17,20 @@ createNode((err, ipfs) => {
   series([
     (cb) => {
       const someData = Buffer.from('capoeira')
+      let node
 
-      dagPB.DAGNode.create(someData, (err, node) => {
+      try {
+        dagPB.DAGNode.create(someData)
+      } catch (err) {
+        return cb(err)
+      }
+
+      ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => {
         if (err) {
           cb(err)
         }
-
-        ipfs.dag.put(node, { format: 'dag-pb', hashAlg: 'sha2-256' }, (err, cid) => {
-          if (err) {
-            cb(err)
-          }
-          cidPBNode = cid
-          cb()
-        })
+        cidPBNode = cid
+        cb()
       })
     },
     (cb) => {

package.json

@@ -72,12 +72,12 @@
     "execa": "^1.0.0",
     "form-data": "^2.3.3",
     "hat": "0.0.3",
+    "interface-ipfs-core": "~0.103.0",
     "ipfsd-ctl": "~0.42.0",
     "libp2p-websocket-star": "~0.10.2",
     "ncp": "^2.0.0",
     "qs": "^6.5.2",
     "rimraf": "^2.6.2",
-    "interface-ipfs-core": "~0.102.0",
     "sinon": "^7.3.1",
     "stream-to-promise": "^2.2.0"
   },
@@ -86,6 +86,9 @@
     "@hapi/hapi": "^18.3.1",
     "@hapi/joi": "^15.0.1",
     "async": "^2.6.1",
+    "async-iterator-all": "0.0.2",
+    "async-iterator-to-pull-stream": "^1.1.0",
+    "async-iterator-to-stream": "^1.1.0",
     "base32.js": "~0.1.0",
     "bignumber.js": "^8.0.2",
     "binary-querystring": "~0.1.2",
@@ -94,8 +97,9 @@
     "bs58": "^4.0.1",
     "buffer-peek-stream": "^1.0.1",
     "byteman": "^1.3.5",
-    "cid-tool": "~0.2.0",
-    "cids": "~0.5.8",
+    "callbackify": "^1.1.0",
+    "cid-tool": "~0.3.0",
+    "cids": "~0.7.1",
     "class-is": "^1.1.0",
     "datastore-core": "~0.6.0",
     "datastore-pubsub": "~0.1.1",
@@ -109,39 +113,39 @@
     "glob": "^7.1.3",
     "hapi-pino": "^6.0.0",
     "human-to-milliseconds": "^1.0.0",
-    "interface-datastore": "~0.7.0",
-    "ipfs-bitswap": "~0.23.0",
-    "ipfs-block": "~0.8.0",
+    "interface-datastore": "~0.6.0",
+    "ipfs-bitswap": "~0.24.0",
+    "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.15.1",
-    "ipfs-http-client": "^31.1.0",
-    "ipfs-http-response": "~0.2.1",
-    "ipfs-mfs": "~0.10.2",
+    "ipfs-http-client": "^32.0.0",
+    "ipfs-http-response": "~0.3.0",
+    "ipfs-mfs": "~0.11.2",
     "ipfs-multipart": "~0.1.0",
-    "ipfs-repo": "~0.26.5",
+    "ipfs-repo": "~0.26.6",
     "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-exporter": "~0.36.1",
-    "ipfs-unixfs-importer": "~0.38.5",
-    "ipld": "~0.21.1",
-    "ipld-bitcoin": "~0.1.8",
-    "ipld-dag-cbor": "~0.13.1",
-    "ipld-dag-pb": "~0.15.3",
-    "ipld-ethereum": "^2.0.1",
-    "ipld-git": "~0.3.0",
-    "ipld-raw": "^2.0.1",
-    "ipld-zcash": "~0.1.6",
+    "ipfs-unixfs-exporter": "~0.37.4",
+    "ipfs-unixfs-importer": "~0.39.6",
+    "ipfs-utils": "~0.0.3",
+    "ipld": "~0.24.1",
+    "ipld-bitcoin": "~0.3.0",
+    "ipld-dag-cbor": "~0.15.0",
+    "ipld-dag-pb": "~0.17.3",
+    "ipld-ethereum": "^4.0.0",
+    "ipld-git": "~0.5.0",
+    "ipld-raw": "^4.0.0",
+    "ipld-zcash": "~0.3.0",
     "ipns": "~0.5.0",
-    "is-ipfs": "~0.6.0",
+    "is-ipfs": "~0.6.1",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^2.0.0",
     "iso-url": "~0.4.6",
-    "ipfs-utils": "~0.0.3",
     "just-flatten-it": "^2.1.0",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
     "libp2p": "~0.25.3",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.16.0",
-    "libp2p-kad-dht": "~0.14.12",
+    "libp2p-kad-dht": "~0.15.0",
     "libp2p-keychain": "~0.4.1",
     "libp2p-mdns": "~0.12.0",
     "libp2p-record": "~0.6.1",
@@ -158,7 +162,7 @@
     "multiaddr": "^6.0.5",
     "multiaddr-to-uri": "^4.0.1",
     "multibase": "~0.6.0",
-    "multicodec": "~0.5.0",
+    "multicodec": "~0.5.1",
     "multihashes": "~0.4.14",
     "multihashing-async": "~0.6.0",
     "node-fetch": "^2.3.0",
@@ -177,6 +181,7 @@
     "pull-pushable": "^2.2.0",
     "pull-sort": "^1.0.1",
     "pull-stream": "^3.6.9",
+    "pull-stream-to-async-iterator": "^1.0.1",
     "pull-stream-to-stream": "^1.3.4",
     "pull-traverse": "^1.0.3",
     "readable-stream": "^3.1.1",

src/cli/bin.js

@@ -2,6 +2,18 @@
 
 'use strict'
 
+process.on('uncaughtException', (err) => {
+  console.info(err)
+
+  throw err
+})
+
+process.on('unhandledRejection', (err) => {
+  console.info(err)
+
+  throw err
+})
+
 const YargsPromise = require('yargs-promise')
 const updateNotifier = require('update-notifier')
 const utils = require('./utils')
@@ -35,7 +47,7 @@ async function main (args) {
       }
     })
     .catch(({ error, argv }) => {
-      getIpfs = argv.getIpfs
+      getIpfs = argv && argv.getIpfs
       debug(error)
       // the argument can have a different shape depending on where the error came from
       if (error.message || (error.error && error.error.message)) {

src/cli/commands/dag/get.js

@@ -32,7 +32,7 @@ module.exports = {
       try {
         result = await ipfs.dag.get(cid, path, options)
       } catch (err) {
-        return print(`dag get failed: ${err.message}`)
+        return print(`dag get failed: ${err}`)
       }
 
       if (options.localResolve) {

src/cli/commands/object/get.js

@@ -25,21 +25,21 @@ module.exports = {
     resolve((async () => {
       const ipfs = await getIpfs()
       const node = await ipfs.object.get(key, { enc: 'base58' })
-      let data = node.data
+      let data = node.Data || ''
 
       if (Buffer.isBuffer(data)) {
-        data = node.data.toString(dataEncoding || undefined)
+        data = node.Data.toString(dataEncoding || undefined)
       }
 
       const answer = {
         Data: data,
         Hash: cidToString(key, { base: cidBase, upgrade: false }),
-        Size: node.size,
-        Links: node.links.map((l) => {
+        Size: node.Size,
+        Links: node.Links.map((l) => {
           return {
-            Name: l.name,
-            Size: l.size,
-            Hash: cidToString(l.cid, { base: cidBase, upgrade: false })
+            Name: l.Name,
+            Size: l.Tsize,
+            Hash: cidToString(l.Hash, { base: cidBase, upgrade: false })
           }
         })
       }

src/cli/commands/object/links.js

@@ -23,8 +23,8 @@ module.exports = {
       const links = await ipfs.object.links(key, { enc: 'base58' })
 
       links.forEach((link) => {
-        const cidStr = cidToString(link.cid, { base: cidBase, upgrade: false })
-        print(`${cidStr} ${link.size} ${link.name}`)
+        const cidStr = cidToString(link.Hash, { base: cidBase, upgrade: false })
+        print(`${cidStr} ${link.Tsize} ${link.Name}`)
       })
     })())
   }

src/cli/commands/object/patch/add-link.js

@@ -3,7 +3,6 @@
 const dagPB = require('ipld-dag-pb')
 const DAGLink = dagPB.DAGLink
 const multibase = require('multibase')
-const promisify = require('promisify-es6')
 const { print } = require('../../../utils')
 const { cidToString } = require('../../../../utils/cid')
 
@@ -17,14 +16,21 @@ module.exports = {
       describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
       type: 'string',
       choices: multibase.names
+    },
+    'cid-version': {
+      describe: 'The CID version of the DAGNode to link to',
+      type: 'number',
+      default: 0
     }
   },
 
-  handler ({ getIpfs, root, name, ref, cidBase, resolve }) {
+  handler ({ getIpfs, root, name, ref, cidBase, cidVersion, resolve }) {
     resolve((async () => {
       const ipfs = await getIpfs()
       const nodeA = await ipfs.object.get(ref, { enc: 'base58' })
-      const result = await promisify(dagPB.util.cid)(nodeA)
+      const result = await dagPB.util.cid(dagPB.util.serialize(nodeA), {
+        cidVersion
+      })
       const link = new DAGLink(name, nodeA.size, result)
       const cid = await ipfs.object.patch.addLink(root, link, { enc: 'base58' })
       print(cidToString(cid, { base: cidBase, upgrade: false }))

src/core/components/bitswap.js

@@ -9,7 +9,7 @@ const PeerId = require('peer-id')
 const errCode = require('err-code')
 
 function formatWantlist (list, cidBase) {
-  return Array.from(list).map((e) => ({ '/': e[1].cid.toBaseEncodedString() }))
+  return Array.from(list).map((e) => ({ '/': e[1].cid.toBaseEncodedString(cidBase) }))
 }
 
 module.exports = function bitswap (self) {