feat: cid base option (#1552)

Implements an option for the CLI, HTTP API and core (where appropriate) that will allow the user to pick the multibase encoding for any CIDs that are returned as strings.

**NOTE** Using the CID base option in `bitswap`, `dag` and `object` APIs **WILL NOT** auto upgrade your CID to v1 if it is a v0 CID and **WILL NOT** apply the encoding you specified. This is because these APIs return IPLD objects with links and changing the version of the links would result in a different hash for the node if you were to re-add it. Also, the CID you used to retrieve the node wouldn't actually refer to the node you got back any longer. [Read this](ipfs/kubo#5349 (comment)) for further context.

This PR adds a `--cid-base` option to the following **CLI commands**:

* `jsipfs bitswap stat`
* `jsipfs bitswap unwant`
* `jsipfs bitswap wantlist`
* `jsipfs block put`
* `jsipfs block stat`
* `jsipfs add`
* `jsipfs ls`
* `jsipfs object get`
* `jsipfs object links`
* `jsipfs object new`
* `jsipfs object patch add-link`
* `jsipfs object patch append-data`
* `jsipfs object patch rm-link`
* `jsipfs object patch set-data`
* `jsipfs object put`
* `jsipfs object stat`
* `jsipfs pin add`
* `jsipfs pin ls`
* `jsipfs pin rm`
* `jsipfs resolve`

Note: these two MFS commands _already_ implement the `--cid-base` option:

* `jsipfs files ls`
* `jsipfs files stat`

It also adds `?cid-base=` query option to the following **HTTP endpoints**:

* `/api/v0/bitswap/wantlist`
* `/api/v0/bitswap/stat`
* `/api/v0/bitswap/unwant`
* `/api/v0/block/put`
* `/api/v0/block/stat`
* `/api/v0/add`
* `/api/v0/ls`
* `/api/v0/object/new`
* `/api/v0/object/get`
* `/api/v0/object/put`
* `/api/v0/object/stat`
* `/api/v0/object/links`
* `/api/v0/object/patch/append-data`
* `/api/v0/object/patch/set-data`
* `/api/v0/object/patch/add-link`
* `/api/v0/object/patch/rm-link`
* `/api/v0/pin/ls`
* `/api/v0/pin/add`
* `/api/v0/pin/rm`
* `/api/v0/resolve`

It adds a `cidBase` option to the following **core functions**:

* `resolve`

License: MIT
Signed-off-by: Alan Shaw <[email protected]>

Author: Alan Shaw

package.json

@@ -108,7 +108,7 @@
     "ipfs-bitswap": "~0.21.0",
     "ipfs-block": "~0.8.0",
     "ipfs-block-service": "~0.15.1",
-    "ipfs-http-client": "^28.0.1",
+    "ipfs-http-client": "^28.1.0",
     "ipfs-http-response": "~0.2.1",
     "ipfs-mfs": "~0.8.0",
     "ipfs-multipart": "~0.1.0",
@@ -122,7 +122,7 @@
     "ipld-git": "~0.2.2",
     "ipld-zcash": "~0.1.6",
     "ipns": "~0.4.3",
-    "is-ipfs": "~0.4.7",
+    "is-ipfs": "~0.4.8",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",
     "joi": "^14.3.0",

src/cli/commands/add.js

@@ -11,9 +11,9 @@ const getFolderSize = require('get-folder-size')
 const byteman = require('byteman')
 const waterfall = require('async/waterfall')
 const mh = require('multihashes')
-const utils = require('../utils')
-const print = require('../utils').print
-const createProgressBar = require('../utils').createProgressBar
+const multibase = require('multibase')
+const { print, isDaemonOn, createProgressBar } = require('../utils')
+const { cidToString } = require('../../utils/cid')
 
 function checkPath (inPath, recursive) {
   // This function is to check for the following possible inputs
@@ -90,7 +90,7 @@ function addPipeline (index, addStream, list, argv) {
       sortBy(added, 'path')
         .reverse()
         .map((file) => {
-          const log = [ 'added', file.hash ]
+          const log = [ 'added', cidToString(file.hash, { base: argv.cidBase }) ]
 
           if (!quiet && file.path.length > 0) log.push(file.path)
 
@@ -156,10 +156,15 @@ module.exports = {
       describe: 'CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. (experimental)',
       default: 0
     },
+    'cid-base': {
+      describe: 'Number base to display CIDs in.',
+      type: 'string',
+      choices: multibase.names
+    },
     hash: {
       type: 'string',
       choices: Object.keys(mh.names),
-      describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)'
+      describe: 'Hash function to use. Will set CID version to 1 if used. (experimental)'
     },
     quiet: {
       alias: 'q',
@@ -202,7 +207,7 @@ module.exports = {
       chunker: argv.chunker
     }
 
-    if (options.enableShardingExperiment && utils.isDaemonOn()) {
+    if (options.enableShardingExperiment && isDaemonOn()) {
       throw new Error('Error: Enabling the sharding experiment should be done on the daemon')
     }
     const ipfs = argv.ipfs

src/cli/commands/bitswap/stat.js

@@ -1,22 +1,29 @@
 'use strict'
 
-const print = require('../../utils').print
+const multibase = require('multibase')
+const { print } = require('../../utils')
+const { cidToString } = require('../../../utils/cid')
 
 module.exports = {
   command: 'stat',
 
   describe: 'Show some diagnostic information on the bitswap agent.',
 
-  builder: {},
+  builder: {
+    'cid-base': {
+      describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
+      type: 'string',
+      choices: multibase.names
+    }
+  },
 
-  handler (argv) {
-    argv.ipfs.bitswap.stat((err, stats) => {
+  handler ({ ipfs, cidBase }) {
+    ipfs.bitswap.stat((err, stats) => {
       if (err) {
         throw err
       }
 
-      stats.wantlist = stats.wantlist || []
-      stats.wantlist = stats.wantlist.map(entry => entry['/'])
+      stats.wantlist = stats.wantlist.map(k => cidToString(k['/'], { base: cidBase, upgrade: false }))
       stats.peers = stats.peers || []
 
       print(`bitswap status

src/cli/commands/bitswap/unwant.js

@@ -1,6 +1,8 @@
 'use strict'
 
-const print = require('../../utils').print
+const multibase = require('multibase')
+const { print } = require('../../utils')
+const { cidToString } = require('../../../utils/cid')
 
 module.exports = {
   command: 'unwant <key>',
@@ -12,14 +14,19 @@ module.exports = {
       alias: 'k',
       describe: 'Key to remove from your wantlist',
       type: 'string'
+    },
+    'cid-base': {
+      describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
+      type: 'string',
+      choices: multibase.names
     }
   },
-  handler (argv) {
-    argv.ipfs.bitswap.unwant(argv.key, (err) => {
+  handler ({ ipfs, key, cidBase }) {
+    ipfs.bitswap.unwant(key, (err) => {
       if (err) {
         throw err
       }
-      print(`Key ${argv.key} removed from wantlist`)
+      print(`Key ${cidToString(key, { base: cidBase, upgrade: false })} removed from wantlist`)
     })
   }
 }

src/cli/commands/bitswap/wantlist.js

@@ -1,6 +1,8 @@
 'use strict'
 
-const print = require('../../utils').print
+const multibase = require('multibase')
+const { print } = require('../../utils')
+const { cidToString } = require('../../../utils/cid')
 
 module.exports = {
   command: 'wantlist [peer]',
@@ -12,17 +14,20 @@ module.exports = {
       alias: 'p',
       describe: 'Specify which peer to show wantlist for.',
       type: 'string'
+    },
+    'cid-base': {
+      describe: 'Number base to display CIDs in. Note: specifying a CID base for v0 CIDs will have no effect.',
+      type: 'string',
+      choices: multibase.names
     }
   },
 
-  handler (argv) {
-    argv.ipfs.bitswap.wantlist(argv.peer, (err, res) => {
+  handler ({ ipfs, peer, cidBase }) {
+    ipfs.bitswap.wantlist(peer, (err, list) => {
       if (err) {
         throw err
       }
-      res.Keys.forEach((cid) => {
-        print(cid['/'])
-      })
+      list.Keys.forEach(k => print(cidToString(k['/'], { base: cidBase, upgrade: false })))
     })
   }
 }

src/cli/commands/block/get.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const CID = require('cids')
 const print = require('../../utils').print
 
 module.exports = {
@@ -10,10 +9,8 @@ module.exports = {
 
   builder: {},
 
-  handler (argv) {
-    const cid = new CID(argv.key)
-
-    argv.ipfs.block.get(cid, (err, block) => {
+  handler ({ ipfs, key }) {
+    ipfs.block.get(key, (err, block) => {
       if (err) {
         throw err
       }

src/cli/commands/block/put.js

@@ -1,33 +1,19 @@
 'use strict'
 
-const CID = require('cids')
-const multihashing = require('multihashing-async')
 const bl = require('bl')
 const fs = require('fs')
-const Block = require('ipfs-block')
-const waterfall = require('async/waterfall')
-const print = require('../../utils').print
+const multibase = require('multibase')
+const { print } = require('../../utils')
+const { cidToString } = require('../../../utils/cid')
 
 function addBlock (data, opts) {
   const ipfs = opts.ipfs
-  let cid
 
-  waterfall([
-    (cb) => multihashing(data, opts.mhtype || 'sha2-256', cb),
-    (multihash, cb) => {
-      if (opts.format !== 'dag-pb' || opts.version !== 0) {
-        cid = new CID(1, opts.format || 'dag-pb', multihash)
-      } else {
-        cid = new CID(0, 'dag-pb', multihash)
-      }
-
-      ipfs.block.put(new Block(data, cid), cb)
-    }
-  ], (err) => {
+  ipfs.block.put(data, opts, (err, block) => {
     if (err) {
       throw err
     }
-    print(cid.toBaseEncodedString())
+    print(cidToString(block.cid, { base: opts.cidBase }))
   })
 }
 
@@ -52,8 +38,12 @@ module.exports = {
     },
     version: {
       describe: 'cid version',
-      type: 'number',
-      default: 0
+      type: 'number'
+    },
+    'cid-base': {
+      describe: 'Number base to display CIDs in.',
+      type: 'string',
+      choices: multibase.names
     }
   },
 

src/cli/commands/block/rm.js

@@ -1,8 +1,6 @@
 'use strict'
 
-const utils = require('../../utils')
-const mh = require('multihashes')
-const print = utils.print
+const { print, isDaemonOn } = require('../../utils')
 
 module.exports = {
   command: 'rm <key>',
@@ -11,18 +9,18 @@ module.exports = {
 
   builder: {},
 
-  handler (argv) {
-    if (utils.isDaemonOn()) {
+  handler ({ ipfs, key }) {
+    if (isDaemonOn()) {
       // TODO implement this once `js-ipfs-http-client` supports it
       throw new Error('rm block with daemon running is not yet implemented')
     }
 
-    argv.ipfs.block.rm(mh.fromB58String(argv.key), (err) => {
+    ipfs.block.rm(key, (err) => {
       if (err) {
         throw err
       }
 
-      print('removed ' + argv.key)
+      print('removed ' + key)
     })
   }
 }

src/cli/commands/block/stat.js

@@ -1,22 +1,29 @@
 'use strict'
 
-const CID = require('cids')
-const print = require('../../utils').print
+const multibase = require('multibase')
+const { print } = require('../../utils')
+const { cidToString } = require('../../../utils/cid')
 
 module.exports = {
   command: 'stat <key>',
 
   describe: 'Print information of a raw IPFS block',
 
-  builder: {},
+  builder: {
+    'cid-base': {
+      describe: 'Number base to display CIDs in.',
+      type: 'string',
+      choices: multibase.names
+    }
+  },
 
-  handler (argv) {
-    argv.ipfs.block.stat(new CID(argv.key), (err, stats) => {
+  handler ({ ipfs, key, cidBase }) {
+    ipfs.block.stat(key, (err, stats) => {
       if (err) {
         throw err
       }
 
-      print('Key: ' + stats.key)
+      print('Key: ' + cidToString(stats.key, { base: cidBase }))
       print('Size: ' + stats.size)
     })
   }

src/cli/commands/ls.js

@@ -1,6 +1,8 @@
 'use strict'
 
-const utils = require('../utils')
+const multibase = require('multibase')
+const { print, rightpad } = require('../utils')
+const { cidToString } = require('../../utils/cid')
 
 module.exports = {
   command: 'ls <key>',
@@ -24,33 +26,41 @@ module.exports = {
       desc: 'Resolve linked objects to find out their types. (not implemented yet)',
       type: 'boolean',
       default: false // should be true when implemented
+    },
+    'cid-base': {
+      describe: 'Number base to display CIDs in.',
+      type: 'string',
+      choices: multibase.names
     }
   },
 
-  handler (argv) {
-    let path = argv.key
-    if (path.startsWith('/ipfs/')) {
-      path = path.replace('/ipfs/', '')
-    }
-
-    argv.ipfs.ls(path, { recursive: argv.recursive }, (err, links) => {
+  handler ({ ipfs, key, recursive, headers, cidBase }) {
+    ipfs.ls(key, { recursive }, (err, links) => {
       if (err) {
         throw err
       }
 
-      if (argv.headers) {
+      links = links.map(file => Object.assign(file, { hash: cidToString(file.hash, { base: cidBase }) }))
+
+      if (headers) {
         links = [{ hash: 'Hash', size: 'Size', name: 'Name' }].concat(links)
       }
 
       const multihashWidth = Math.max.apply(null, links.map((file) => file.hash.length))
       const sizeWidth = Math.max.apply(null, links.map((file) => String(file.size).length))
 
+      let pathParts = key.split('/')
+
+      if (key.startsWith('/ipfs/')) {
+        pathParts = pathParts.slice(2)
+      }
+
       links.forEach(link => {
         const fileName = link.type === 'dir' ? `${link.name || ''}/` : link.name
-        const padding = link.depth - path.split('/').length
-        utils.print(
-          utils.rightpad(link.hash, multihashWidth + 1) +
-          utils.rightpad(link.size || '', sizeWidth + 1) +
+        const padding = link.depth - pathParts.length
+        print(
+          rightpad(link.hash, multihashWidth + 1) +
+          rightpad(link.size || '', sizeWidth + 1) +
           '  '.repeat(padding) + fileName
         )
       })