cli: refactor --batch from file to accept large inputs

allowing to use it for dump imports

Also being more tolerant on the output, to accept the output of a --all dump

Author: maxlath

README.md

@@ -88,16 +88,38 @@ lev --batch '[
 ```
 or from a file
 ```sh
+# there should be one entry per line
+# either as valid JSON
 echo '[
 {"type":"del","key":"father"},
 {"type":"put","key":"name","value":"Yuri Irsenovich Kim"},
 {"type":"put","key":"dob","value":"16 February 1941"},
 {"type":"put","key":"spouse","value":"Kim Young-sook"},
 {"type":"put","key":"occupation","value":"Clown"}
 ]' > ops.json
+# or as newline-delimited JSON
+echo '
+{"type":"del","key":"father"}
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"}
+{"type":"put","key":"dob","value":"16 February 1941"}
+{"type":"put","key":"spouse","value":"Kim Young-sook"}
+{"type":"put","key":"occupation","value":"Clown"}
+' > ops.json
 lev --batch ./ops.json
 ```
 
+If the type is omitted, defaults to `put`, which allows to use the command to do imports/exports, in combination with [`--all`](#--all):
+```sh
+lev --all > leveldb.export
+lev /tmp/my-new-db --batch leveldb.export
+```
+If it's a large export, you can compress it on the fly
+```sh
+lev --all | gzip -9 > leveldb.export.gz
+gzip -dk leveldb.export.gz
+lev /tmp/my-new-db --batch leveldb.export
+```
+
 ## --keys
 List all the keys in the current range. Will tabularize the output by default (see `--line`).
 ```sh
@@ -117,6 +139,11 @@ Emit as a new-line delimited stream of json.
 ```sh
 lev --all
 ```
+It can be used to create an export of the database, to be imported with [`--batch`](#--batch)
+```sh
+lev --all > leveldb.export
+lev /tmp/my-new-db --batch leveldb.export
+```
 
 ## --start <key-pattern>
 Specify the start of the current range. You can also use `gt` or `gte`.

lib/batch_from_file.js

@@ -0,0 +1,52 @@
+var fs = require('fs');
+var path = require('path');
+var split = require('split');
+var printAndExit = require('./print_and_exit');
+
+module.exports = function(db, file) {
+  var batch = [];
+  var total = 0;
+
+  fs.createReadStream(path.resolve(file))
+  .pipe(split())
+  .on('data', function(line) {
+    if (line[0] !== '{') return
+    line = line.replace(/,$/, '')
+
+    //
+    // add each line to the batch
+    //
+    var entry = JSON.parse(line);
+    entry.type = entry.type || 'put';
+    batch.push(entry)
+
+    //
+    // run operations by batches of 10000 entries
+    //
+    if (batch.length >= 10000) {
+      var stream = this;
+      stream.pause();
+      db.batch(batch, function(err, val) {
+        if (err) return printAndExit(err, val);
+        total += batch.length
+        console.log('ops run:', total)
+        batch = [];
+        stream.resume();
+      })
+    }
+
+  })
+  .on('close', function() {
+    if (batch.length > 0) {
+      db.batch(batch, function(err, val) {
+        if (err) return printAndExit(err, val);
+        total += batch.length;
+        console.log('total ops:', total)
+      });
+    }
+    else {
+      console.log('total ops:', total)
+    }
+  })
+}
+

lib/cli.js

@@ -94,7 +94,8 @@ module.exports = function(db, args) {
   }
   else if (args.batch) {
     if (args.batch[0] !== '[') {
-      args.batch = require('fs').readFileSync(args.batch).toString()
+      require('./batch_from_file')(db, args.batch)
+      return
     }
     var batch = JSON.parse(args.batch);
     db.batch(batch, printAndExit);

package.json

@@ -41,6 +41,7 @@
     "minimist": "^1.1.0",
     "multilevel": "^7.3.0",
     "rc": "^0.5.4",
+    "split": "^1.0.1",
     "tabulate": "^1.0.0",
     "xtend": "^4.0.0"
   }