perf: switch out pull-block for bl (#12)

Author: achingbrain

.aegir.js

@@ -0,0 +1,7 @@
+'use strict'
+
+module.exports = {
+  karma: {
+    browserNoActivityTimeout: 100 * 1000
+  }
+}

package.json

@@ -37,7 +37,7 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs-unixfs-importer#readme",
   "devDependencies": {
-    "aegir": "^17.0.0",
+    "aegir": "^18.0.2",
     "chai": "^4.2.0",
     "detect-node": "^2.0.4",
     "dirty-chai": "^2.0.1",
@@ -52,6 +52,7 @@
   "dependencies": {
     "async": "^2.6.1",
     "async-iterator-to-pull-stream": "^1.1.0",
+    "bl": "^2.1.2",
     "cids": "~0.5.5",
     "deep-extend": "~0.6.0",
     "hamt-sharding": "~0.0.2",
@@ -60,7 +61,6 @@
     "left-pad": "^1.3.0",
     "multihashing-async": "~0.5.1",
     "pull-batch": "^1.0.0",
-    "pull-block": "^1.4.0",
     "pull-pair": "^1.1.0",
     "pull-paramap": "^1.2.2",
     "pull-pause": "0.0.2",

src/chunker/fixed-size.js

@@ -1,8 +1,50 @@
 'use strict'
 
-const pullBlock = require('pull-block')
+const BufferList = require('bl')
+const through = require('pull-through')
 
 module.exports = (options) => {
   let maxSize = (typeof options === 'number') ? options : options.maxChunkSize
-  return pullBlock(maxSize, { zeroPadding: false, emitEmpty: true })
+  let bl = new BufferList()
+  let currentLength = 0
+  let emitted = false
+
+  return through(
+    function onData (buffer) {
+      bl.append(buffer)
+
+      currentLength += buffer.length
+
+      while (currentLength >= maxSize) {
+        this.queue(bl.slice(0, maxSize))
+
+        emitted = true
+
+        // throw away consumed bytes
+        if (maxSize === bl.length) {
+          bl = new BufferList()
+          currentLength = 0
+        } else {
+          const newBl = new BufferList()
+          newBl.append(bl.shallowSlice(maxSize))
+          bl = newBl
+
+          // update our offset
+          currentLength -= maxSize
+        }
+      }
+    },
+    function onEnd () {
+      if (currentLength) {
+        this.queue(bl.slice(0, currentLength))
+        emitted = true
+      }
+
+      if (!emitted) {
+        this.queue(Buffer.alloc(0))
+      }
+
+      this.queue(null)
+    }
+  )
 }

test/chunker-fixed-size.spec.js

@@ -66,7 +66,7 @@ describe('chunker: fixed size', function () {
     const KiB256 = 262144
 
     pull(
-      values(rawFile),
+      values([rawFile]),
       chunker(KiB256),
       collect((err, chunks) => {
         expect(err).to.not.exist()
@@ -85,7 +85,7 @@ describe('chunker: fixed size', function () {
     let file = Buffer.concat([rawFile, Buffer.from('hello')])
 
     pull(
-      values(file),
+      values([file]),
       chunker(KiB256),
       collect((err, chunks) => {
         expect(err).to.not.exist()

test/import-export.spec.js

@@ -59,7 +59,7 @@ describe('import and export', function () {
         const path = strategy + '-big.dat'
 
         pull(
-          values([{ path: path, content: values(bigFile) }]),
+          values([{ path: path, content: values([bigFile]) }]),
           importer(ipld, importerOptions),
           map((file) => {
             expect(file.path).to.eql(path)