feat: implement ipfs refs and refs local (#2004)

* feat: implement ipfs refs

* feat: refs support in http api

* feat: use ipld instead of unix-fs-exporter for refs

* test: add basic refs test

* feat: refs local

* feat: add refs.localPullStream && refs.localReadableStream

* chore: change Ref -> ref

* feat: make object.links work with CBOR

* feat: handle multiple refs. Better param handling

* fix: print refs errors to stderr

* chore: add comment to explain cli param parsing

* refactor: use streaming for refs local

* chore: update interface-ipfs-core package

* refactor: cleaner refs param handling

* fix: alias 'refs local' to 'refs-local'

* refactor: move links retrieval from object to refs

* chore: add missing packages

* refactor: use streaming for cli refs and refs local

* fix: add refs and refs local to command count

* fix: refs in browser

* fix: restore param parsing behaviour

* chore: update interface-ipfs-core

* chore: update http-client

* chore: update interface-ipfs-core

* fix: skip failing config.set test for now

* fix: skip failing config.set test in http-api

* chore: update interface-ipfs-core and ipfs-http-client

* chore: fix ipfs-http-client version

Author: dirkmc

package.json

@@ -72,7 +72,7 @@
     "execa": "^1.0.0",
     "form-data": "^2.3.3",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.99.1",
+    "interface-ipfs-core": "~0.101.1",
     "ipfsd-ctl": "~0.42.0",
     "libp2p-websocket-star": "~0.10.2",
     "ncp": "^2.0.0",
@@ -86,6 +86,7 @@
     "@hapi/hapi": "^18.3.1",
     "@hapi/joi": "^15.0.1",
     "async": "^2.6.1",
+    "base32.js": "~0.1.0",
     "bignumber.js": "^8.0.2",
     "binary-querystring": "~0.1.2",
     "bl": "^3.0.0",
@@ -112,7 +113,7 @@
     "ipfs-bitswap": "~0.23.0",
     "ipfs-block": "~0.8.0",
     "ipfs-block-service": "~0.15.1",
-    "ipfs-http-client": "^31.0.0",
+    "ipfs-http-client": "^31.0.2",
     "ipfs-http-response": "~0.2.1",
     "ipfs-mfs": "~0.10.2",
     "ipfs-multipart": "~0.1.0",
@@ -175,6 +176,7 @@
     "pull-sort": "^1.0.1",
     "pull-stream": "^3.6.9",
     "pull-stream-to-stream": "^1.3.4",
+    "pull-traverse": "^1.0.3",
     "readable-stream": "^3.1.1",
     "receptacle": "^1.3.2",
     "stream-to-pull-stream": "^1.7.3",

src/cli/bin.js

@@ -10,6 +10,7 @@ const mfs = require('ipfs-mfs/cli')
 const debug = require('debug')('ipfs:cli')
 const pkg = require('../../package.json')
 const parser = require('./parser')
+const commandAlias = require('./command-alias')
 
 async function main (args) {
   const oneWeek = 1000 * 60 * 60 * 24 * 7
@@ -22,6 +23,9 @@ async function main (args) {
 
   let getIpfs = null
 
+  // Apply command aliasing (eg `refs local` -> `refs-local`)
+  args = commandAlias(args)
+
   cli
     .parse(args)
     .then(({ data, argv }) => {

src/cli/command-alias.js

@@ -0,0 +1,35 @@
+'use strict'
+
+const aliases = {
+  // We need to be able to show help text for both the `refs` command and the
+  // `refs local` command, but with yargs `refs` cannot be both a command and
+  // a command directory. So alias `refs local` to `refs-local`
+  'refs-local': ['refs', 'local']
+}
+
+// Replace multi-word command with alias
+// eg replace `refs local` with `refs-local`
+module.exports = function (args) {
+  for (const [alias, original] of Object.entries(aliases)) {
+    if (arrayMatch(args, original)) {
+      return [alias, ...args.slice(original.length)]
+    }
+  }
+
+  return args
+}
+
+// eg arrayMatch([1, 2, 3], [1, 2]) => true
+function arrayMatch (arr, sub) {
+  if (sub.length > arr.length) {
+    return false
+  }
+
+  for (let i = 0; i < sub.length; i++) {
+    if (arr[i] !== sub[i]) {
+      return false
+    }
+  }
+
+  return true
+}

src/cli/commands/refs-local.js

@@ -0,0 +1,30 @@
+'use strict'
+
+const { print } = require('../utils')
+
+module.exports = {
+  command: 'refs-local',
+
+  describe: 'List all local references.',
+
+  handler ({ getIpfs, resolve }) {
+    resolve((async () => {
+      const ipfs = await getIpfs()
+
+      return new Promise((resolve, reject) => {
+        const stream = ipfs.refs.localReadableStream()
+
+        stream.on('error', reject)
+        stream.on('end', resolve)
+
+        stream.on('data', (ref) => {
+          if (ref.err) {
+            print(ref.err, true, true)
+          } else {
+            print(ref.ref)
+          }
+        })
+      })
+    })())
+  }
+}

src/cli/commands/refs.js

@@ -0,0 +1,65 @@
+'use strict'
+
+const { print } = require('../utils')
+
+module.exports = {
+  command: 'refs <key> [keys..]',
+
+  describe: 'List links (references) from an object',
+
+  builder: {
+    recursive: {
+      alias: 'r',
+      desc: 'Recursively list links of child nodes.',
+      type: 'boolean',
+      default: false
+    },
+    format: {
+      desc: 'Output edges with given format. Available tokens: <src> <dst> <linkname>.',
+      type: 'string',
+      default: '<dst>'
+    },
+    edges: {
+      alias: 'e',
+      desc: 'Output edge format: `<from> -> <to>`',
+      type: 'boolean',
+      default: false
+    },
+    unique: {
+      alias: 'u',
+      desc: 'Omit duplicate refs from output.',
+      type: 'boolean',
+      default: false
+    },
+    'max-depth': {
+      desc: 'Only for recursive refs, limits fetch and listing to the given depth.',
+      type: 'number'
+    }
+  },
+
+  handler ({ getIpfs, key, keys, recursive, format, edges, unique, maxDepth, resolve }) {
+    resolve((async () => {
+      if (maxDepth === 0) {
+        return
+      }
+
+      const ipfs = await getIpfs()
+      const k = [key].concat(keys)
+
+      return new Promise((resolve, reject) => {
+        const stream = ipfs.refsReadableStream(k, { recursive, format, edges, unique, maxDepth })
+
+        stream.on('error', reject)
+        stream.on('end', resolve)
+
+        stream.on('data', (ref) => {
+          if (ref.err) {
+            print(ref.err, true, true)
+          } else {
+            print(ref.ref)
+          }
+        })
+      })
+    })())
+  }
+}

src/cli/utils.js

@@ -81,7 +81,7 @@ exports.getRepoPath = () => {
 let visible = true
 exports.disablePrinting = () => { visible = false }
 
-exports.print = (msg, newline) => {
+exports.print = (msg, newline, isError = false) => {
   if (newline === undefined) {
     newline = true
   }
@@ -91,7 +91,8 @@ exports.print = (msg, newline) => {
       msg = ''
     }
     msg = newline ? msg + '\n' : msg
-    process.stdout.write(msg)
+    const outStream = isError ? process.stderr : process.stdout
+    outStream.write(msg)
   }
 }
 

src/core/components/files-regular/index.js

@@ -1,19 +1,28 @@
 'use strict'
 
-module.exports = self => ({
-  add: require('./add')(self),
-  addFromFs: require('./add-from-fs')(self),
-  addFromStream: require('./add-from-stream')(self),
-  addFromURL: require('./add-from-url')(self),
-  addPullStream: require('./add-pull-stream')(self),
-  addReadableStream: require('./add-readable-stream')(self),
-  cat: require('./cat')(self),
-  catPullStream: require('./cat-pull-stream')(self),
-  catReadableStream: require('./cat-readable-stream')(self),
-  get: require('./get')(self),
-  getPullStream: require('./get-pull-stream')(self),
-  getReadableStream: require('./get-readable-stream')(self),
-  ls: require('./ls')(self),
-  lsPullStream: require('./ls-pull-stream')(self),
-  lsReadableStream: require('./ls-readable-stream')(self)
-})
+module.exports = (self) => {
+  const filesRegular = {
+    add: require('./add')(self),
+    addFromFs: require('./add-from-fs')(self),
+    addFromStream: require('./add-from-stream')(self),
+    addFromURL: require('./add-from-url')(self),
+    addPullStream: require('./add-pull-stream')(self),
+    addReadableStream: require('./add-readable-stream')(self),
+    cat: require('./cat')(self),
+    catPullStream: require('./cat-pull-stream')(self),
+    catReadableStream: require('./cat-readable-stream')(self),
+    get: require('./get')(self),
+    getPullStream: require('./get-pull-stream')(self),
+    getReadableStream: require('./get-readable-stream')(self),
+    ls: require('./ls')(self),
+    lsPullStream: require('./ls-pull-stream')(self),
+    lsReadableStream: require('./ls-readable-stream')(self),
+    refs: require('./refs')(self),
+    refsReadableStream: require('./refs-readable-stream')(self),
+    refsPullStream: require('./refs-pull-stream')(self)
+  }
+  filesRegular.refs.local = require('./refs-local')(self)
+  filesRegular.refs.localReadableStream = require('./refs-local-readable-stream')(self)
+  filesRegular.refs.localPullStream = require('./refs-local-pull-stream')(self)
+  return filesRegular
+}

src/core/components/files-regular/refs-local-pull-stream.js

@@ -0,0 +1,34 @@
+'use strict'
+
+const CID = require('cids')
+const base32 = require('base32.js')
+const pull = require('pull-stream')
+const pullDefer = require('pull-defer')
+
+module.exports = function (self) {
+  return () => {
+    const deferred = pullDefer.source()
+
+    self._repo.blocks.query({ keysOnly: true }, (err, blocks) => {
+      if (err) {
+        return deferred.resolve(pull.error(err))
+      }
+
+      const refs = blocks.map(b => dsKeyToRef(b.key))
+      deferred.resolve(pull.values(refs))
+    })
+
+    return deferred
+  }
+}
+
+function dsKeyToRef (key) {
+  try {
+    // Block key is of the form /<base32 encoded string>
+    const decoder = new base32.Decoder()
+    const buff = Buffer.from(decoder.write(key.toString().slice(1)).finalize())
+    return { ref: new CID(buff).toString() }
+  } catch (err) {
+    return { err: `Could not convert block with key '${key}' to CID: ${err.message}` }
+  }
+}

src/core/components/files-regular/refs-local-readable-stream.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const toStream = require('pull-stream-to-stream')
+
+module.exports = function (self) {
+  return (ipfsPath, options) => {
+    return toStream.source(self.refs.localPullStream())
+  }
+}

src/core/components/files-regular/refs-local.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const promisify = require('promisify-es6')
+const pull = require('pull-stream')
+
+module.exports = function (self) {
+  return promisify((callback) => {
+    pull(
+      self.refs.localPullStream(),
+      pull.collect((err, values) => {
+        if (err) {
+          return callback(err)
+        }
+        callback(null, values)
+      })
+    )
+  })
+}