test: add mfs interop tests

Author: achingbrain

package.json

@@ -39,7 +39,7 @@
     "async": "^2.6.1",
     "bl": "^2.0.1",
     "bs58": "^4.0.1",
-    "buffer-loader": "0.0.1",
+    "buffer-loader": "~0.0.1",
     "chai": "^4.1.2",
     "cross-env": "^5.2.0",
     "cids": "~0.5.3",
@@ -51,9 +51,10 @@
     "form-data": "^2.3.2",
     "go-ipfs-dep": "~0.4.17",
     "hat": "0.0.3",
-    "ipfs": "~0.31.1",
-    "ipfs-api": "^22.2.4",
-    "ipfsd-ctl": "~0.39.0",
+    "ipfs": "~0.31.6",
+    "ipfs-api": "^24.0.0",
+    "ipfsd-ctl": "~0.39.1",
+    "ipfs-unixfs": "~0.1.15",
     "left-pad": "^1.3.0",
     "libp2p-websocket-star-rendezvous": "~0.2.3",
     "lodash": "^4.17.10",

test/browser.js

@@ -6,3 +6,4 @@
 require('./kad-dht')
 require('./circuit')
 // require('./repo')
+require('./files')

test/files.js

@@ -0,0 +1,285 @@
+/* eslint-env mocha */
+'use strict'
+
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const expect = chai.expect
+chai.use(dirtyChai)
+const crypto = require('crypto')
+const UnixFs = require('ipfs-unixfs')
+const {
+  spawnInitAndStartGoDaemon,
+  spawnInitAndStartJsDaemon,
+  stopDaemon
+} = require('./utils/daemon')
+
+class ExpectedError extends Error {
+
+}
+
+function checkNodeTypes (daemon, file) {
+  return daemon.api.object.get(file.hash)
+    .then(node => {
+      const meta = UnixFs.unmarshal(node.data)
+
+      expect(meta.type).to.equal('file')
+      expect(node.links.length).to.equal(2)
+
+      return Promise.all(
+        node.links.map(link => daemon.api.object.get(link.toJSON().multihash).then(child => {
+          const childMeta = UnixFs.unmarshal(child.data)
+
+          expect(childMeta.type).to.equal('raw')
+        }))
+      )
+    })
+}
+
+function addFile (daemon, data) {
+  const fileName = 'test-file'
+
+  return daemon.api.files.write(`/${fileName}`, data, {
+    create: true
+  })
+    // cannot list file directly - https://github.com/ipfs/go-ipfs/issues/5044
+    .then(() => {
+      return daemon.api.files.ls('/', {
+        l: true
+      })
+    })
+    .then(files => {
+      return files.filter(file => file.name === fileName).pop()
+    })
+}
+
+const compare = (...ops) => {
+  expect(ops.length).to.be.above(1)
+
+  return Promise.all(
+    ops
+  )
+    .then(results => {
+      expect(results.length).to.equal(ops.length)
+
+      const result = results.pop()
+
+      results.forEach(res => expect(res).to.deep.equal(result))
+    })
+}
+
+const compareErrors = (...ops) => {
+  expect(ops.length).to.be.above(1)
+
+  return Promise.all(
+    // even if operations fail, their errors should be the same
+    ops.map(op => op.then(() => {
+      throw new ExpectedError('Expected operation to fail')
+    }).catch(error => {
+      if (error instanceof ExpectedError) {
+        throw error
+      }
+
+      return {
+        message: error.message,
+        code: error.code
+      }
+    }))
+  )
+    .then(results => {
+      expect(results.length).to.equal(ops.length)
+
+      const result = results.pop()
+
+      results.forEach(res => expect(res).to.deep.equal(result))
+    })
+}
+
+describe('files', function () {
+  this.timeout(50 * 1000)
+
+  let go
+  let js
+
+  before(() => {
+    return Promise.all([
+      spawnInitAndStartGoDaemon(),
+      spawnInitAndStartJsDaemon()
+    ])
+      .then(([goDaemon, jsDaemon]) => {
+        go = goDaemon
+        js = jsDaemon
+      })
+  })
+
+  after(() => {
+    return Promise.all([
+      stopDaemon(go),
+      stopDaemon(js)
+    ])
+  })
+
+  it('returns an error when reading non-existent files', () => {
+    const readNonExistentFile = (daemon) => {
+      return daemon.api.files.read(`/i-do-not-exist-${Math.random()}`)
+    }
+
+    return compareErrors(
+      readNonExistentFile(go),
+      readNonExistentFile(js)
+    )
+  })
+
+  it('returns an error when writing deeply nested files and the parents do not exist', () => {
+    const readNonExistentFile = (daemon) => {
+      return daemon.api.files.write(`/foo-${Math.random()}/bar-${Math.random()}/baz-${Math.random()}/i-do-not-exist-${Math.random()}`, Buffer.from([0, 1, 2, 3]))
+    }
+
+    return compareErrors(
+      readNonExistentFile(go),
+      readNonExistentFile(js)
+    )
+  })
+
+  it('uses raw nodes for leaf data', () => {
+    const data = crypto.randomBytes(1024 * 300)
+    const testLeavesAreRaw = (daemon) => {
+      return addFile(daemon, data)
+        .then(file => checkNodeTypes(daemon, file))
+    }
+
+    return compare(
+      testLeavesAreRaw(go),
+      testLeavesAreRaw(js)
+    )
+  })
+
+  it('errors when creating the same directory twice', () => {
+    const path = `/test-dir-${Math.random()}`
+
+    return compareErrors(
+      go.api.files.mkdir(path).then(() => go.api.files.mkdir(path)),
+      js.api.files.mkdir(path).then(() => js.api.files.mkdir(path))
+    )
+  })
+
+  it('does not error when creating the same directory twice and -p is passed', () => {
+    const path = `/test-dir-${Math.random()}`
+
+    return compare(
+      go.api.files.mkdir(path).then(() => go.api.files.mkdir(path, {p: true})),
+      js.api.files.mkdir(path).then(() => js.api.files.mkdir(path, {p: true}))
+    )
+  })
+
+  it('errors when creating the root directory', () => {
+    const path = '/'
+
+    return compareErrors(
+      go.api.files.mkdir(path).then(() => go.api.files.mkdir(path)),
+      js.api.files.mkdir(path).then(() => js.api.files.mkdir(path))
+    )
+  })
+
+  describe('has the same hashes for', () => {
+    const testHashesAreEqual = (daemon, data, options) => {
+      return daemon.api.files.add(data, options)
+        .then(files => files[0].hash)
+    }
+
+    const _writeData = (daemon, initialData, newData, options) => {
+      const fileName = `file-${Math.random()}.txt`
+
+      return daemon.api.files.write(`/${fileName}`, initialData, {
+        create: true
+      })
+        .then(() => daemon.api.files.ls('/', {
+          l: true
+        }))
+        .then(files => files.filter(file => file.name === fileName).pop().hash)
+    }
+
+    const appendData = (daemon, initialData, appendedData) => {
+      return _writeData(daemon, initialData, appendedData, {
+        offset: initialData.length
+      })
+    }
+
+    const overwriteData = (daemon, initialData, newData) => {
+      return _writeData(daemon, initialData, newData, {
+        offset: 0
+      })
+    }
+
+    it('empty files', () => {
+      const data = Buffer.alloc(0)
+
+      return compare(
+        testHashesAreEqual(go, data),
+        testHashesAreEqual(js, data)
+      )
+    })
+
+    it('small files', () => {
+      const data = Buffer.from([0x00, 0x01, 0x02])
+
+      return compare(
+        testHashesAreEqual(go, data),
+        testHashesAreEqual(js, data)
+      )
+    })
+
+    it('big files', () => {
+      const data = crypto.randomBytes(1024 * 3000)
+
+      return compare(
+        testHashesAreEqual(go, data),
+        testHashesAreEqual(js, data)
+      )
+    })
+
+    it('files that have had data appended', () => {
+      const initialData = crypto.randomBytes(1024 * 300)
+      const appendedData = crypto.randomBytes(1024 * 300)
+
+      return compare(
+        appendData(go, initialData, appendedData),
+        appendData(js, initialData, appendedData)
+      )
+    })
+
+    it('files that have had data overwritten', () => {
+      const bytes = 1024 * 300
+      const initialData = crypto.randomBytes(bytes)
+      const newData = crypto.randomBytes(bytes)
+
+      return compare(
+        overwriteData(go, initialData, newData),
+        overwriteData(js, initialData, newData)
+      )
+    })
+
+    it('small files with CIDv1', () => {
+      const data = Buffer.from([0x00, 0x01, 0x02])
+      const options = {
+        cidVersion: 1
+      }
+
+      return compare(
+        testHashesAreEqual(go, data, options),
+        testHashesAreEqual(js, data, options)
+      )
+    })
+
+    it('big files with CIDv1', () => {
+      const data = crypto.randomBytes(1024 * 3000)
+      const options = {
+        cidVersion: 1
+      }
+
+      return compare(
+        testHashesAreEqual(go, data, options),
+        testHashesAreEqual(js, data, options)
+      )
+    })
+  })
+})

test/node.js

@@ -7,3 +7,4 @@ require('./repo')
 require('./exchange-files')
 require('./kad-dht')
 require('./pin')
+require('./files')

test/utils/daemon.js

@@ -0,0 +1,56 @@
+'use strict'
+
+const os = require('os')
+const path = require('path')
+const hat = require('hat')
+const waterfall = require('async/waterfall')
+const DaemonFactory = require('ipfsd-ctl')
+const goDf = DaemonFactory.create()
+const jsDf = DaemonFactory.create({ type: 'js' })
+
+const spawnInitAndStartDaemon = (factory) => {
+  const dir = path.join(os.tmpdir(), hat())
+  let instance
+
+  return new Promise((resolve, reject) => {
+    waterfall([
+      (cb) => factory.spawn({
+        repoPath: dir,
+        disposable: false,
+        initOptions: {
+          bits: 1024
+        }
+      }, cb),
+      (node, cb) => {
+        instance = node
+        instance.init(cb)
+      },
+      (cb) => instance.start((error) => cb(error, instance))
+    ], (error) => {
+      if (error) {
+        return reject(error)
+      }
+
+      resolve(instance)
+    })
+  })
+}
+
+const stopDaemon = (daemon) => {
+  return new Promise((resolve, reject) => {
+    daemon.stop((error) => {
+      if (error) {
+        return reject(error)
+      }
+
+      resolve()
+    })
+  })
+}
+
+module.exports = {
+  spawnInitAndStartDaemon,
+  spawnInitAndStartGoDaemon: () => spawnInitAndStartDaemon(goDf),
+  spawnInitAndStartJsDaemon: () => spawnInitAndStartDaemon(jsDf),
+  stopDaemon
+}