feat: add support for File DOM API to files-regular (#2013)

* feat: add support for File DOM API to files-regular

* chore: use correct interface tests

* chore: use correct interface tests

* chore: fix package declaration cause npm is dumb

* chore: fix deps

* chore: fix deps

* chore: fix deps

* fix: use correct input

* chore: add correct http-client

* fix: use ipfs-utils

* chore: fix semver lint

Author: hugomrdias

package.json

@@ -72,12 +72,12 @@
     "execa": "^1.0.0",
     "form-data": "^2.3.3",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.101.1",
     "ipfsd-ctl": "~0.42.0",
     "libp2p-websocket-star": "~0.10.2",
     "ncp": "^2.0.0",
     "qs": "^6.5.2",
     "rimraf": "^2.6.2",
+    "interface-ipfs-core": "~0.102.0",
     "sinon": "^7.3.1",
     "stream-to-promise": "^2.2.0"
   },
@@ -109,11 +109,11 @@
     "glob": "^7.1.3",
     "hapi-pino": "^6.0.0",
     "human-to-milliseconds": "^1.0.0",
-    "interface-datastore": "~0.6.0",
+    "interface-datastore": "~0.7.0",
     "ipfs-bitswap": "~0.23.0",
     "ipfs-block": "~0.8.0",
     "ipfs-block-service": "~0.15.1",
-    "ipfs-http-client": "^31.0.2",
+    "ipfs-http-client": "^31.1.0",
     "ipfs-http-response": "~0.2.1",
     "ipfs-mfs": "~0.10.2",
     "ipfs-multipart": "~0.1.0",
@@ -134,8 +134,10 @@
     "is-pull-stream": "~0.0.0",
     "is-stream": "^2.0.0",
     "iso-url": "~0.4.6",
+    "ipfs-utils": "~0.0.3",
     "just-flatten-it": "^2.1.0",
     "just-safe-set": "^2.1.0",
+    "kind-of": "^6.0.2",
     "libp2p": "~0.25.0",
     "libp2p-bootstrap": "~0.9.3",
     "libp2p-crypto": "~0.16.0",

src/core/components/files-regular/add-pull-stream.js

@@ -1,13 +1,20 @@
 'use strict'
 
 const importer = require('ipfs-unixfs-importer')
-const pull = require('pull-stream')
+const kindOf = require('kind-of')
+const CID = require('cids')
+const pullValues = require('pull-stream/sources/values')
+const pullMap = require('pull-stream/throughs/map')
+const pullAsyncMap = require('pull-stream/throughs/async-map')
+const pullFlatten = require('pull-stream/throughs/flatten')
+const pull = require('pull-stream/pull')
 const toPull = require('stream-to-pull-stream')
 const waterfall = require('async/waterfall')
 const isStream = require('is-stream')
-const isSource = require('is-pull-stream').isSource
-const CID = require('cids')
+const { isSource } = require('is-pull-stream')
 const { parseChunkerString } = require('./utils')
+const streamFromFileReader = require('ipfs-utils/src/streams/stream-from-filereader')
+const { supportsFileReader } = require('ipfs-utils/src/supports')
 
 const WRAPPER = 'wrapper/'
 
@@ -52,9 +59,12 @@ function normalizeContent (content, opts) {
   }
 
   return content.map((data) => {
+    if (supportsFileReader && kindOf(data) === 'file') {
+      data = { path: '', content: toPull.source(streamFromFileReader(data)) }
+    }
     // Buffer input
     if (Buffer.isBuffer(data)) {
-      data = { path: '', content: pull.values([data]) }
+      data = { path: '', content: pullValues([data]) }
     }
 
     // Readable stream input
@@ -68,7 +78,7 @@ function normalizeContent (content, opts) {
 
     if (data && data.content && typeof data.content !== 'function') {
       if (Buffer.isBuffer(data.content)) {
-        data.content = pull.values([data.content])
+        data.content = pullValues([data.content])
       }
 
       if (isStream.readable(data.content)) {
@@ -124,7 +134,7 @@ module.exports = function (self) {
     try {
       chunkerOptions = parseChunkerString(options.chunker)
     } catch (err) {
-      return pull.map(() => { throw err })
+      return pullMap(() => { throw err })
     }
     const opts = Object.assign({}, {
       shardSplitThreshold: self._options.EXPERIMENTAL.sharding
@@ -147,12 +157,12 @@ module.exports = function (self) {
 
     opts.progress = progress
     return pull(
-      pull.map(content => normalizeContent(content, opts)),
-      pull.flatten(),
+      pullMap(content => normalizeContent(content, opts)),
+      pullFlatten(),
       importer(self._ipld, opts),
-      pull.asyncMap((file, cb) => prepareFile(file, self, opts, cb)),
-      pull.map(file => preloadFile(file, self, opts)),
-      pull.asyncMap((file, cb) => pinFile(file, self, opts, cb))
+      pullAsyncMap((file, cb) => prepareFile(file, self, opts, cb)),
+      pullMap(file => preloadFile(file, self, opts)),
+      pullAsyncMap((file, cb) => pinFile(file, self, opts, cb))
     )
   }
 }

src/core/components/files-regular/add.js

@@ -3,8 +3,8 @@
 const promisify = require('promisify-es6')
 const pull = require('pull-stream')
 const sort = require('pull-sort')
-const isStream = require('is-stream')
 const isSource = require('is-pull-stream').isSource
+const validateAddInput = require('ipfs-utils/src/files/add-input-validation')
 
 module.exports = function (self) {
   const add = promisify((data, options, callback) => {
@@ -15,23 +15,10 @@ module.exports = function (self) {
 
     options = options || {}
 
-    // Buffer, pull stream or Node.js stream
-    const isBufferOrStream = obj => Buffer.isBuffer(obj) || isStream.readable(obj) || isSource(obj)
-    // An object like { content?, path? }, where content isBufferOrStream and path isString
-    const isContentObject = obj => {
-      if (typeof obj !== 'object') return false
-      // path is optional if content is present
-      if (obj.content) return isBufferOrStream(obj.content)
-      // path must be a non-empty string if no content
-      return Boolean(obj.path) && typeof obj.path === 'string'
-    }
-    // An input atom: a buffer, stream or content object
-    const isInput = obj => isBufferOrStream(obj) || isContentObject(obj)
-    // All is ok if data isInput or data is an array of isInput
-    const ok = isInput(data) || (Array.isArray(data) && data.every(isInput))
-
-    if (!ok) {
-      return callback(new Error('invalid input: expected buffer, readable stream, pull stream, object or array of objects'))
+    try {
+      validateAddInput(data)
+    } catch (err) {
+      return callback(err)
     }
 
     pull(

src/core/components/files-regular/utils.js

@@ -1,8 +1,9 @@
 'use strict'
 
 const CID = require('cids')
+const { Buffer } = require('buffer')
 
-exports.normalizePath = (path) => {
+const normalizePath = (path) => {
   if (Buffer.isBuffer(path)) {
     return new CID(path).toString()
   }
@@ -30,7 +31,7 @@ exports.normalizePath = (path) => {
  *
  * @return {Object}   Chunker options for DAGBuilder
  */
-exports.parseChunkerString = (chunker) => {
+const parseChunkerString = (chunker) => {
   if (!chunker) {
     return {
       chunker: 'fixed'
@@ -67,7 +68,7 @@ exports.parseChunkerString = (chunker) => {
  *
  * @return {Object}   rabin chunker options
  */
-function parseRabinString (chunker) {
+const parseRabinString = (chunker) => {
   const options = {}
   const parts = chunker.split('-')
   switch (parts.length) {
@@ -89,11 +90,18 @@ function parseRabinString (chunker) {
   return options
 }
 
-function parseChunkSize (str, name) {
+const parseChunkSize = (str, name) => {
   let size = parseInt(str)
   if (isNaN(size)) {
     throw new Error(`Chunker parameter ${name} must be an integer`)
   }
 
   return size
 }
+
+module.exports = {
+  normalizePath,
+  parseChunkSize,
+  parseRabinString,
+  parseChunkerString
+}