import data using a stream from createImportStream

Author: Matt Zumwalt

index.js

@@ -1,22 +1,16 @@
 var level = require('level')
 var hypercore = require('hypercore')
+var createImportStream = require('./lib/import.js')
 
 module.exports = Jawn
-function Jawn () {
-  var self = this
-  self.db = level('data.jawn')
-  self.core = hypercore(self.db)
+
+function Jawn (opts) {
+  if (!opts) opts = {}
+  var db = opts.db || level('data.jawn')
+  this.core = opts.core || hypercore(db)
+  this.db = this.core.db
 }
 
-Jawn.prototype.import = function (stream) {
-  var ws = this.core.createWriteStream()
-  stream.on('data', function (line) {
-    ws.write(line)
-  })
-  stream.on('end', function () {
-    ws.end(function () {
-      console.log(ws.id.toString('hex'))
-    })
-  })
-  return ws
+Jawn.prototype.createImportStream = function (opts) {
+  return createImportStream(this, opts)
 }

lib/import.js

@@ -0,0 +1,32 @@
+var pump = require('pump')
+var through = require('through2')
+
+module.exports = importStream
+
+function importStream (jawn, opts) {
+  if (!opts) opts = {}
+  var writeStream = jawn.core.createWriteStream(opts)
+  var stream = pump(parseInputStream(opts), writeStream, function done (err) {
+    stream.id = writeStream.id
+    console.log(err)
+  })
+  return stream
+}
+
+// Transform input CSV, JSON, etc to json objects that will be written as blocks in our hypercore feed
+// TODO: Implement this! See https://github.com/CfABrigadePhiladelphia/jawn/issues/32
+// This is the functionality that parse-input-stream aims to support.
+// If you can get that module to work, feel free to use it!
+function parseInputStream (opts) {
+  var transformStream = through(write, end)
+  return transformStream
+
+  function write (buffer, encoding, next) {
+    // transform input CSV, JSON, etc. here ...
+    next()
+  }
+
+  function end (done) {
+    done()
+  }
+}

package.json

@@ -21,14 +21,18 @@
   },
   "homepage": "https://github.com/CfABrigadePhiladelphia/jawn",
   "dependencies": {
-    "hypercore": "~1.8.0",
     "fs": "0.0.2",
+    "hypercore": "~1.8.0",
+    "level": "~1.4.0",
+    "parse-input-stream": "^1.0.1",
     "path": "~0.12.7",
+    "pump": "^1.0.1",
     "tape": "~4.5.1",
-    "level": "~1.4.0"
+    "through2": "^2.0.1"
   },
   "devDependencies": {
     "standard": "^6.0.5",
-    "tape": "^4.4.0"
+    "tape": "^4.4.0",
+    "memdb": "^1.3.1"
   }
 }

test.csv

@@ -1,29 +1,39 @@
 var test = require('tape')
 var Jawn = require('../')
-var fs = require('fs')
+var memdb = require('memdb')
 
-var j = new Jawn()
+test('import json to jawn', function (t) {
+  var jawn = freshJawn()
+  var importStream = jawn.createImportStream({'format': 'json'})
+  // Imitate the stream that would come from reading sample.csv
+  // importStream should parse the CSV correctly, identifying the first row as headers
+  // This is the same as doing
+  //    var data = fs.createReadStream('./test/data/sample.csv')
+  //    data.pipe(importStream)
+  // except the writes are being performed synchronously/inline so we can call importStream.end() after writing the contents into it.
+  importStream.write('{foo: "bar", name: "josie"}')
+  importStream.write('{foo: "baz", name: "eloise"}')
+  importStream.write('{foo: "baz", name: "francoise"}')
 
-test('import csv to jawn', function (t) {
-  var data = fs.createReadStream('./test/sample/sample.csv')
-  var ws = j.import(data)
+  // This should be expecting JSON objects, not strings.
+  // temporarily expecting strings in order to hand off the code as-is
   var expected = [
-              {'Type of Experience': 'Writing software in any programming language', 'Little/No Experience': 1, 'Some Experience': 5, 'Very Familiar': 4},
-              {'Type of Experience': 'Frontend Web Development', 'Little/No Experience': 4, 'Some Experience': 3, 'Very Familiar': 3},
-              {'Type of Experience': 'Server-side (backend) Web Development', 'Little/No Experience': 4, 'Some Experience': 4, 'Very Familiar': 2},
-              {'Type of Experience': 'Using Git to track changes and share code (add, commit, push, pull)', 'Little/No Experience': 2, 'Some Experience': 5, 'Very Familiar': 3}
+    '{foo: "bar", name: "josie"}',
+    '{foo: "baz", name: "eloise"}',
+    '{foo: "baz", name: "francoise"}'
   ]
-  ws.on('end', function () {
-    var feedId = ws.id.toString('hex')
-    var rs = j.core.createReadStream(feedId)
 
+  importStream.end(function () {
+    var feedId = importStream.id.toString('hex')
+    var rs = jawn.core.createReadStream(feedId)
     rs.on('data', function (block) {
       t.same(block.toString(), expected.shift(), 'block matches imported line')
     })
-
-    var blocks = j.core.get(feedId).blocks
-
-    t.same(blocks, 4, 'correct number of blocks returned')
+    t.same(jawn.core.get(feedId).blocks, 3, 'correct number of blocks returned')
     t.end()
   })
 })
+
+function freshJawn () {
+  return new Jawn({db: memdb()})
+}

test/sample/sample.csv renamed to test/data/sample.csv

@@ -1,6 +1,26 @@
 var test = require('tape')
 var Jawn = require('../index')
-test('jawn import', function (t) {
-  t.equal(new Jawn().import('some data'), 'Hello World')
+var memdb = require('memdb')
+var hypercore = require('hypercore')
+
+test('jawn constructor defaults', function (t) {
+  var jawn = new Jawn({db: memdb()})
+  t.true(jawn.core instanceof hypercore, 'core is an instance of Hypercore')
+  // t.true(jawn.db instanceof leveldb, 'db is an instance of leveldb')
+  t.equal(jawn.db, jawn.core.db, 'jawn db matches core db')
+  t.end()
+})
+
+test('jawn constructor set db', function (t) {
+  var sampledb = memdb()
+  var jawn = new Jawn({db: sampledb})
+  t.equal(jawn.core.db, sampledb, 'initializes hypercore with the provided db')
+  t.end()
+})
+
+test('jawn constructor set core', function (t) {
+  var samplecore = hypercore(memdb())
+  var jawn = new Jawn({core: samplecore})
+  t.equal(jawn.core, samplecore, 'uses the provided instance of hypercore')
   t.end()
 })