feat: use it-tar (#2758)

`it-tar` now supports packing \o/

Author: Alan Shaw

package.json

@@ -123,6 +123,7 @@
     "it-glob": "0.0.7",
     "it-last": "^1.0.1",
     "it-pipe": "^1.1.0",
+    "it-tar": "^1.2.1",
     "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "jsondiffpatch": "~0.3.11",
@@ -165,7 +166,6 @@
     "semver": "^7.1.2",
     "stream-to-it": "^0.2.0",
     "streaming-iterables": "^4.1.1",
-    "tar-stream": "^2.0.0",
     "temp": "~0.9.0",
     "timeout-abort-controller": "^1.1.0",
     "update-notifier": "^4.0.0",

src/http/api/resources/files-regular.js

@@ -2,7 +2,7 @@
 
 const multipart = require('ipfs-multipart')
 const debug = require('debug')
-const tar = require('tar-stream')
+const tar = require('it-tar')
 const log = debug('ipfs:http-api:files')
 log.error = debug('ipfs:http-api:files:error')
 const toIterable = require('stream-to-it')
@@ -11,16 +11,20 @@ const Boom = require('@hapi/boom')
 const { PassThrough } = require('stream')
 const multibase = require('multibase')
 const isIpfs = require('is-ipfs')
-const { promisify } = require('util')
 const { cidToString } = require('../../../utils/cid')
 const { Format } = require('../../../core/components/refs')
 const pipe = require('it-pipe')
 const all = require('it-all')
-const concat = require('it-concat')
 const ndjson = require('iterable-ndjson')
 const { map } = require('streaming-iterables')
 const streamResponse = require('../../utils/stream-response')
 
+const toBuffer = async function * (source) {
+  for await (const chunk of source) {
+    yield chunk.slice()
+  }
+}
+
 function numberFromQuery (query, key) {
   if (query && query[key] !== undefined) {
     const value = parseInt(query[key], 10)
@@ -86,32 +90,24 @@ exports.get = {
     const { ipfs } = request.server.app
     const { key } = request.pre.args
 
-    const pack = tar.pack()
-    pack.entry = promisify(pack.entry.bind(pack))
+    return streamResponse(request, h, () => pipe(
+      ipfs.get(key),
+      async function * (source) {
+        for await (const file of source) {
+          const header = {
+            name: file.path
+          }
 
-    const streamFiles = async () => {
-      try {
-        for await (const file of ipfs.get(key)) {
           if (file.content) {
-            const content = await concat(file.content)
-            pack.entry({ name: file.path, size: file.size }, content.slice())
+            yield { header: { ...header, size: file.size }, body: toBuffer(file.content) }
           } else {
-            pack.entry({ name: file.path, type: 'directory' })
+            yield { header: { ...header, type: 'directory' } }
           }
         }
-        pack.finalize()
-      } catch (err) {
-        log.error(err)
-        pack.emit('error', err)
-        pack.destroy()
-      }
-    }
-
-    streamFiles()
-
-    // reply must be called right away so that tar-stream offloads its content
-    // otherwise it will block in large files
-    return h.response(pack).header('X-Stream-Output', '1')
+      },
+      tar.pack(),
+      toBuffer
+    ))
   }
 }