feat: --only-hash (#717)

* feat: only-hash option for add/addFromFS/addFromURL (#700)

* chore: update ipfsd-ctl

Author: daviddias

package.json

@@ -75,7 +75,7 @@
     "hapi": "^17.2.2",
     "interface-ipfs-core": "~0.55.1",
     "ipfs": "~0.28.0",
-    "ipfsd-ctl": "~0.29.1",
+    "ipfsd-ctl": "~0.30.1",
     "pre-commit": "^1.2.2",
     "socket.io": "^2.0.4",
     "socket.io-client": "^2.0.4",

src/files/add-pull-stream.js

@@ -8,6 +8,6 @@ module.exports = (send) => {
   return (options) => {
     options = options || {}
     options.converter = FileResultStreamConverter
-    return toPull(SendFilesStream(send, 'add')(options))
+    return toPull(SendFilesStream(send, 'add')({ qs: options }))
   }
 }

src/files/add.js

@@ -36,7 +36,7 @@ module.exports = (send) => {
 
     const files = [].concat(_files)
 
-    const stream = createAddStream(options)
+    const stream = createAddStream({ qs: options })
     const concat = ConcatStream((result) => callback(null, result))
     stream.once('error', callback)
     stream.pipe(concat)

src/files/write.js

@@ -33,7 +33,7 @@ module.exports = (send) => {
       converter: FileResultStreamConverter
     }
 
-    const stream = sendFilesStream(options)
+    const stream = sendFilesStream({ qs: options })
     const concat = concatStream((result) => callback(null, result))
     stream.once('error', callback)
     stream.pipe(concat)

src/util/fs-add.js

@@ -2,12 +2,11 @@
 
 const isNode = require('detect-node')
 const promisify = require('promisify-es6')
-const moduleConfig = require('../utils/module-config')
 const SendOneFile = require('../utils/send-one-file-multiple-results')
 const FileResultStreamConverter = require('../utils/file-result-stream-converter')
 
-module.exports = (arg) => {
-  const sendOneFile = SendOneFile(moduleConfig(arg), 'add')
+module.exports = (send) => {
+  const sendOneFile = SendOneFile(send, 'add')
 
   return promisify((path, opts, callback) => {
     if (typeof opts === 'function' &&

src/util/url-add.js

@@ -3,12 +3,11 @@
 const promisify = require('promisify-es6')
 const parseUrl = require('url').parse
 const request = require('../utils/request')
-const moduleConfig = require('../utils/module-config')
 const SendOneFile = require('../utils/send-one-file-multiple-results')
 const FileResultStreamConverter = require('../utils/file-result-stream-converter')
 
-module.exports = (arg) => {
-  const sendOneFile = SendOneFile(moduleConfig(arg), 'add')
+module.exports = (send) => {
+  const sendOneFile = SendOneFile(send, 'add')
 
   return promisify((url, opts, callback) => {
     if (typeof (opts) === 'function' &&

src/utils/send-files-stream.js

@@ -31,7 +31,7 @@ module.exports = (send, path) => {
     let ended = false
     let writing = false
 
-    options = options || {}
+    options = options ? Object.assign({}, options, options.qs) : {}
 
     const multipart = new Multipart()
 
@@ -42,7 +42,7 @@ module.exports = (send, path) => {
     retStream._write = (file, enc, _next) => {
       const next = once(_next)
       try {
-        const files = prepareFile(file, Object.assign({}, options, options.qs))
+        const files = prepareFile(file, options)
           .map((file) => Object.assign({headers: headers(file)}, file))
 
         writing = true
@@ -75,23 +75,10 @@ module.exports = (send, path) => {
 
     const qs = options.qs || {}
 
-    if (options['cid-version'] != null) {
-      qs['cid-version'] = options['cid-version']
-    } else if (options.cidVersion != null) {
-      qs['cid-version'] = options.cidVersion
-    }
-
-    if (options['raw-leaves'] != null) {
-      qs['raw-leaves'] = options['raw-leaves']
-    } else if (options.rawLeaves != null) {
-      qs['raw-leaves'] = options.rawLeaves
-    }
-
-    if (options.hash != null) {
-      qs.hash = options.hash
-    } else if (options.hashAlg != null) {
-      qs.hash = options.hashAlg
-    }
+    qs['cid-version'] = propOrProp(options, 'cid-version', 'cidVersion')
+    qs['raw-leaves'] = propOrProp(options, 'raw-leaves', 'rawLeaves')
+    qs['only-hash'] = propOrProp(options, 'only-hash', 'onlyHash')
+    qs.hash = propOrProp(options, 'hash', 'hashAlg')
 
     const args = {
       path: path,
@@ -158,3 +145,11 @@ module.exports = (send, path) => {
     return retStream
   }
 }
+
+function propOrProp (source, prop1, prop2) {
+  if (prop1 in source) {
+    return source[prop1]
+  } else if (prop2 in source) {
+    return source[prop2]
+  }
+}

test/files.spec.js

@@ -13,6 +13,7 @@ const CID = require('cids')
 
 const IPFSApi = require('../src')
 const f = require('./utils/factory')
+const expectTimeout = require('./utils/expect-timeout')
 
 const testfile = loadFixture('test/fixtures/testfile.txt')
 
@@ -102,6 +103,19 @@ describe('.files (the MFS API part)', function () {
     })
   })
 
+  it('files.add with only-hash=true', function () {
+    this.slow(10 * 1000)
+    const content = String(Math.random() + Date.now())
+
+    return ipfs.files.add(Buffer.from(content), { onlyHash: true })
+      .then(files => {
+        expect(files).to.have.length(1)
+
+        // 'ipfs.object.get(<hash>)' should timeout because content wasn't actually added
+        return expectTimeout(ipfs.object.get(files[0].hash), 4000)
+      })
+  })
+
   it('files.add with options', (done) => {
     ipfs.files.add(testfile, { pin: false }, (err, res) => {
       expect(err).to.not.exist()
@@ -113,6 +127,42 @@ describe('.files (the MFS API part)', function () {
     })
   })
 
+  it('files.add pins by default', (done) => {
+    const newContent = Buffer.from(String(Math.random()))
+
+    ipfs.pin.ls((err, pins) => {
+      expect(err).to.not.exist()
+      const initialPinCount = pins.length
+      ipfs.files.add(newContent, (err, res) => {
+        expect(err).to.not.exist()
+
+        ipfs.pin.ls((err, pins) => {
+          expect(err).to.not.exist()
+          expect(pins.length).to.eql(initialPinCount + 1)
+          done()
+        })
+      })
+    })
+  })
+
+  it('files.add with pin=false', (done) => {
+    const newContent = Buffer.from(String(Math.random()))
+
+    ipfs.pin.ls((err, pins) => {
+      expect(err).to.not.exist()
+      const initialPinCount = pins.length
+      ipfs.files.add(newContent, { pin: false }, (err, res) => {
+        expect(err).to.not.exist()
+
+        ipfs.pin.ls((err, pins) => {
+          expect(err).to.not.exist()
+          expect(pins.length).to.eql(initialPinCount)
+          done()
+        })
+      })
+    })
+  })
+
   HASH_ALGS.forEach((name) => {
     it(`files.add with hash=${name} and raw-leaves=false`, (done) => {
       const content = String(Math.random() + Date.now())

test/util.spec.js

@@ -9,9 +9,11 @@ chai.use(dirtyChai)
 const isNode = require('detect-node')
 const path = require('path')
 const fs = require('fs')
+const os = require('os')
 
 const IPFSApi = require('../src')
 const f = require('./utils/factory')
+const expectTimeout = require('./utils/expect-timeout')
 
 describe('.util', () => {
   if (!isNode) { return }
@@ -92,31 +94,66 @@ describe('.util', () => {
         done()
       })
     })
+
+    it('with only-hash=true', function () {
+      this.slow(10 * 1000)
+      const content = String(Math.random() + Date.now())
+      const filepath = path.join(os.tmpdir(), `${content}.txt`)
+      fs.writeFileSync(filepath, content)
+
+      return ipfs.util.addFromFs(filepath, { onlyHash: true })
+        .then(out => {
+          fs.unlinkSync(filepath)
+          return expectTimeout(ipfs.object.get(out[0].hash), 4000)
+        })
+    })
   })
 
-  it('.urlAdd http', function (done) {
-    this.timeout(20 * 1000)
+  describe('.urlAdd', () => {
+    it('http', function (done) {
+      this.timeout(20 * 1000)
 
-    ipfs.util.addFromURL('http://example.com/', (err, result) => {
-      expect(err).to.not.exist()
-      expect(result.length).to.equal(1)
-      done()
+      ipfs.util.addFromURL('http://example.com/', (err, result) => {
+        expect(err).to.not.exist()
+        expect(result.length).to.equal(1)
+        done()
+      })
     })
-  })
 
-  it('.urlAdd https', (done) => {
-    ipfs.util.addFromURL('https://example.com/', (err, result) => {
-      expect(err).to.not.exist()
-      expect(result.length).to.equal(1)
-      done()
+    it('https', function (done) {
+      this.timeout(20 * 1000)
+
+      ipfs.util.addFromURL('https://example.com/', (err, result) => {
+        expect(err).to.not.exist()
+        expect(result.length).to.equal(1)
+        done()
+      })
     })
-  })
 
-  it('.urlAdd http with redirection', (done) => {
-    ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
-      expect(err).to.not.exist()
-      expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
-      done()
+    it('http with redirection', function (done) {
+      this.timeout(20 * 1000)
+
+      ipfs.util.addFromURL('http://covers.openlibrary.org/book/id/969165.jpg', (err, result) => {
+        expect(err).to.not.exist()
+        expect(result[0].hash).to.equal('QmaL9zy7YUfvWmtD5ZXp42buP7P4xmZJWFkm78p8FJqgjg')
+        done()
+      })
+    })
+
+    it('.urlAdd http with redirection', (done) => {
+      ipfs.util.addFromURL('https://coverartarchive.org/release/6e2a1694-d8b9-466a-aa33-b1077b2333c1', (err, result) => {
+        expect(err).to.not.exist()
+        expect(result[0].hash).to.equal('QmSUdDvmXuq5YGrL4M3SEz7UZh5eT9WMuAsd9K34sambSj')
+        done()
+      })
+    })
+    
+    it('with only-hash=true', function () {
+      this.timeout(10 * 1000)
+      this.slow(10 * 1000)
+
+      return ipfs.util.addFromURL('http://www.randomtext.me/#/gibberish', { onlyHash: true })
+        .then(out => expectTimeout(ipfs.object.get(out[0].hash), 4000))
     })
   })
 })

test/utils/expect-timeout.js

@@ -0,0 +1,16 @@
+'use strict'
+
+/**
+ * Resolve if @param promise hangs for at least @param ms, throw otherwise
+ * @param  {Promise} promise promise that you expect to hang
+ * @param  {Number} ms       millis to wait
+ * @return {Promise}
+ */
+module.exports = (promise, ms) => {
+  return Promise.race([
+    promise.then((out) => {
+      throw new Error('Expected Promise to timeout but it was successful.')
+    }),
+    new Promise((resolve, reject) => setTimeout(resolve, ms))
+  ])
+}