refactoring CLI commands and documenting

README.md

@@ -1,85 +1,261 @@
-# STATUS
+# lev
 
-[![development sponsored by voltra.co](https://img.shields.io/badge/Development%20sponsored%20by-Voltra.co-yellow.svg)](https://voltra.co/)
-
-# SYNOPSIS
 A simple and convenient commandline tool and REPL for [`leveldb`](http://leveldb.org/).
 
-# FEATURES
-- REPL with colorized tab-completion and zsh/fish style key suggestions
-- REPL automatically saves and reloads REPL history
+## Features
+* [REPL](#repl-commands)
+  * with colorized tab-completion and zsh/fish style key suggestions
+  * automatically saves and reloads REPL history
+* [CLI](#cli-commands)
 
-# SCREENSHOT
 ![screenshot](/docs/screenshot.png)
 
-# INSTALLATION
+## Summary
+
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+
+- [Installation](#installation)
+- [REPL commands](#repl-commands)
+  - [GET &lt;key&gt;](#get-ltkeygt)
+  - [PUT &lt;key&gt; &lt;value&gt;](#put-ltkeygt-ltvaluegt)
+  - [DEL &lt;key&gt;](#del-ltkeygt)
+  - [LS](#ls)
+  - [START &lt;key-pattern&gt;](#start-ltkey-patterngt)
+  - [END &lt;key-pattern&gt;](#end-ltkey-patterngt)
+  - [LIMIT &lt;number&gt;](#limit-ltnumbergt)
+  - [REVERSE](#reverse)
+- [CLI commands](#cli-commands)
+  - [--get &lt;key&gt;](#--get-ltkeygt)
+  - [--put &lt;key&gt;](#--put-ltkeygt)
+  - [--del &lt;key&gt;](#--del-ltkeygt)
+  - [--batch &lt;operations&gt;](#--batch-ltoperationsgt)
+  - [--keys](#--keys)
+  - [--values](#--values)
+  - [--all](#--all)
+  - [--start &lt;key-pattern&gt;](#--start-ltkey-patterngt)
+  - [--end &lt;key-pattern&gt;](#--end-ltkey-patterngt)
+  - [--match &lt;key-pattern&gt;](#--match-ltkey-patterngt)
+  - [--limit &lt;number&gt;](#--limit-ltnumbergt)
+  - [--reverse](#--reverse)
+  - [--line](#--line)
+  - [--length](#--length)
+  - [--valueEncoding &lt;string&gt;](#--valueencoding-ltstringgt)
+  - [--location &lt;string&gt;](#--location-ltstringgt)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+
+## Installation
 
 ```
-$ npm install -g lev
+$ npm install -g lev@https://github.com/maxlath/lev#master
 ```
 
-# BASIC USAGE
+## REPL commands
 
-```
+Start the REPL
+```sh
+# in the current directory
+$ lev
+# somewhere else
 $ lev path/to/db
 ```
 
-# REPL COMMANDS
 Use upper or lower case for the following commands.
 
-## GET &lt;key&gt;
+### GET &lt;key&gt;
 Get a key from the database.
 
-## PUT &lt;key&gt; &lt;value&gt;
+### PUT &lt;key&gt; &lt;value&gt;
 Put a value into the database. If you have `keyEncoding` or `valueEncoding`
 set to `json`, these values will be parsed from strings into `json`.
 
-## DEL &lt;key&gt;
+### DEL &lt;key&gt;
 Delete a key from the database.
 
-## LS
+### LS
 Get all the keys in the current range.
 
-## START &lt;key-pattern&gt;
+### START &lt;key-pattern&gt;
 Defines the start of the current range. You can also use `GT` or `GTE`.
 
-## END &lt;key-pattern&gt;
+### END &lt;key-pattern&gt;
 Defines the end of the current range. You can also use `LT` or `LTE`.
 
-## LIMIT &lt;number&gt;
+### LIMIT &lt;number&gt;
 Limit the number of records in the current range (defaults to 5000).
 
-## REVERSE
+### REVERSE
 Reverse the records in the current range.
 
-# CLI COMMANDS
-These all match the parameters used with
-[`levelup`](https://github.com/rvagg/node-levelup). The default encoding
-for the database is set to `json`.
+## CLI commands
+These all match the parameters used with [`levelup`](https://github.com/rvagg/node-levelup). The default encoding for the database is set to `json`.
 
-## --start &lt;key-pattern&gt;
-Specify the start of the current range. You can also use `gt` or `gte`.
+### --get &lt;key&gt;
+Get a value
+```sh
+lev --get foo
+```
 
-## --end &lt;key-pattern&gt;
-Specify the end of the current range. You can also use `lt` and `lte`.
+### --put &lt;key&gt;
+Put a value
+```sh
+lev --put foo --value bar
+```
+
+### --del &lt;key&gt;
+Delete a value
+```sh
+lev --del foo
+```
+
+### --batch &lt;operations&gt;
+Put or delete several values, using [`levelup` batch syntax](https://github.com/Level/levelup#dbbatcharray-options-callback-array-form)
+```sh
+lev --batch '[
+{"type":"del","key":"father"},
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"},
+{"type":"put","key":"dob","value":"16 February 1941"},
+{"type":"put","key":"spouse","value":"Kim Young-sook"},
+{"type":"put","key":"occupation","value":"Clown"}
+]'
+```
+or from a file
+```sh
+# there should be one entry per line
+# either as valid JSON
+echo '[
+{"type":"del","key":"father"},
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"},
+{"type":"put","key":"dob","value":"16 February 1941"},
+{"type":"put","key":"spouse","value":"Kim Young-sook"},
+{"type":"put","key":"occupation","value":"Clown"}
+]' > ops.json
+# or as newline-delimited JSON
+echo '
+{"type":"del","key":"father"}
+{"type":"put","key":"name","value":"Yuri Irsenovich Kim"}
+{"type":"put","key":"dob","value":"16 February 1941"}
+{"type":"put","key":"spouse","value":"Kim Young-sook"}
+{"type":"put","key":"occupation","value":"Clown"}
+' > ops.json
+lev --batch ./ops.json
+```
+
+If the type is omitted, defaults to `put`, which allows to use the command to do imports/exports, in combination with [`--all`](#--all):
+```sh
+lev --all > leveldb.export
+lev /tmp/my-new-db --batch leveldb.export
+```
+If it's a large export, you can compress it on the fly
+```sh
+lev --all | gzip -9 > leveldb.export.gz
+gzip -dk leveldb.export.gz
+lev /tmp/my-new-db --batch leveldb.export
+```
+
+### --keys
+List all the keys in the current range. Will tabularize the output by default (see `--line`).
+```sh
+lev --keys
+```
 
-## --values
-Only list the all of the values in the current range.
+### --values
+List all the values in the current range.
 Emit as a new-line delimited stream of json.
+```sh
+lev --values
+```
+
+### --all
+List all the keys and values in the current range.
+Emit as a new-line delimited stream of json.
+```sh
+lev --all
+```
+It can be used to create an export of the database, to be imported with [`--batch`](#--batch)
+```sh
+lev --all > leveldb.export
+lev /tmp/my-new-db --batch leveldb.export
+```
 
-## --keys
-Only list all of the keys in the current range. Will tabularize the output.
+### --start &lt;key-pattern&gt;
+Specify the start of the current range. You can also use `gt` or `gte`.
+```sh
+# output all keys after 'foo'
+lev --keys --start 'foo'
+# which is equivalent to
+lev --keys --gte 'foo'
+# the same for values
+lev --values --start 'foo'
+```
+
+### --end &lt;key-pattern&gt;
+Specify the end of the current range. You can also use `lt` and `lte`.
+```sh
+# output all keys before 'fooz'
+lev --keys --end 'fooz'
+# which is equivalent to
+lev --keys --lte 'fooz'
+# the same for values
+lev --values --end 'fooz'
+# output all keys between 'foo' and 'fooz'
+lev --keys --start 'foo' --end 'fooz'
+```
 
-## --keyEncoding &lt;string&gt;
-Specify the encoding for the keys.
+### --match &lt;key-pattern&gt;
+Filter keys or values by a pattern applied on the key
+```sh
+lev  --keys --match 'f*'
+lev  --values --match 'f*'
+lev  --all --match 'f*'
+# Equivalent to
+lev --match 'f*'
+```
 
-## --valueEncoding &lt;string&gt;
-Specify the encoding for the values.
+See [`minimatch` doc](https://github.com/isaacs/minimatch#readme) for patterns
 
-## --limit &lt;number&gt;
+### --limit &lt;number&gt;
 Limit the number of records emitted in the current range.
+```sh
+lev --keys --limit 10
+lev --values --start 'foo' --end 'fooz' --limit 100
+lev --match 'f*' --limit 10
+```
 
-## --reverse
+### --reverse
 Reverse the stream.
+```sh
+lev --keys --reverse
+lev --keys --start 'foo' --end 'fooz' --limit 100 --reverse
+```
+
+### --line
+Output one key per line (instead of the default tabularized output)
+```sh
+lev --keys --line
+```
 
+### --length
+Output the length of the current range
+```sh
+# Count all the key/value pairs in the database
+lev --length
+# Counts the keys and values between 'foo' and 'fooz'
+lev --start 'foo' --end 'fooz' --length
+```
 
+### --valueEncoding &lt;string&gt;
+Specify the encoding for the values (Defaults to 'json').
+```sh
+lev --values --valueEncoding buffer
+```
+
+### --location &lt;string&gt;
+Specify the path to the LevelDB to use. Defaults to the current directory.
+```sh
+lev --location /tmp/test-db --keys
+# Equivalent to
+lev /tmp/test-db --keys
+```

index.js

@@ -21,13 +21,13 @@ module.exports = function(args) {
   //
   var cliCommands = [
     'keys', 'values', 'get', 'match',
-    'put', 'del', 'createReadStream', 'batch'
+    'put', 'del', 'all', 'batch', 'length'
   ];
-  
+
   var cliMode = Object.keys(args).some(function(cmd) {
     return cliCommands.indexOf(cmd) > -1;
   });
- 
+
   if (cliMode) {
     return cli(db, args);
   }

lib/batch_from_file.js

@@ -0,0 +1,52 @@
+var fs = require('fs');
+var path = require('path');
+var split = require('split');
+var printAndExit = require('./print_and_exit');
+
+module.exports = function(db, file) {
+  var batch = [];
+  var total = 0;
+
+  fs.createReadStream(path.resolve(file))
+  .pipe(split())
+  .on('data', function(line) {
+    if (line[0] !== '{') return
+    line = line.replace(/,$/, '')
+
+    //
+    // add each line to the batch
+    //
+    var entry = JSON.parse(line);
+    entry.type = entry.type || 'put';
+    batch.push(entry)
+
+    //
+    // run operations by batches of 10000 entries
+    //
+    if (batch.length >= 10000) {
+      var stream = this;
+      stream.pause();
+      db.batch(batch, function(err, val) {
+        if (err) return printAndExit(err, val);
+        total += batch.length
+        console.log('ops run:', total)
+        batch = [];
+        stream.resume();
+      })
+    }
+
+  })
+  .on('close', function() {
+    if (batch.length > 0) {
+      db.batch(batch, function(err, val) {
+        if (err) return printAndExit(err, val);
+        total += batch.length;
+        console.log('total ops:', total)
+      });
+    }
+    else {
+      console.log('total ops:', total)
+    }
+  })
+}
+

lib/cache.js

@@ -1,6 +1,6 @@
 /*
  *
- * cahce.js
+ * cache.js
  * save the keys from the readstream into an
  * array so that they can be autocompleted and suggested.
  *