Add support for file streaming

Author: Anil Maurya

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fast_jsonparser (0.1.0)
+    fast_jsonparser (0.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -11,6 +11,7 @@ GEM
     rake (13.0.1)
     rake-compiler (1.1.0)
       rake
+    yajl-ruby (1.4.1)
 
 PLATFORMS
   ruby
@@ -22,6 +23,7 @@ DEPENDENCIES
   oj
   rake (~> 13.0)
   rake-compiler
+  yajl-ruby
 
 BUNDLED WITH
    2.0.1

README.md

@@ -38,6 +38,17 @@ FastJsonparser.parse(json) is 96% faster than JSON.parse(json)
 # standard JSON   13.025884   0.031287  13.057171 ( 13.264931) ~ 96% more
 ```
 
+3. Streaming json from file
+```
+FastJsonparser.load_many(src) is 153% faster than Yajl::Parser.new.parse(File.new(src, 'r'))
+```
+[Benchmark result](https://github.com/anilmaurya/fast_jsonparser/blob/master/benchmark/stream_benchmark.rb)
+```
+#                   user     system      total        real
+# FastJsonparser  3.844446   0.141822   3.986268 (  3.884655)
+# YAJL            9.699621   0.110060   9.809681 (  9.826104) ~ 150% more
+```
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -78,7 +89,27 @@ FastJsonparser.parse('{"one": 1, "two": 2}')
 
 ```
 
-3. Raise FastJsonparser::ParseError when invalid JSON provided for parsing
+3. Streaming JSON from file
+
+File with multiple json can be stream with `load_many` method
+
+Example: logs.json with following content
+```
+{"time": "17/May/2015:08:05:32 +0000", "remote_ip": "93.180.71.3", "remote_user": "-"}
+{"time": "17/May/2015:08:05:23 +0000", "remote_ip": "93.180.71.3", "remote_user": "-"}
+{"time": "17/May/2015:08:05:24 +0000", "remote_ip": "80.91.33.133", "remote_user": "-"}
+```
+
+`load_many` accepts file_path & block as arguments
+```
+> FastJsonparser.load_many(file_path) { |obj| p obj[:time]}
+"17/May/2015:08:05:32 +0000"
+"17/May/2015:08:05:23 +0000"
+"17/May/2015:08:05:24 +0000"
+```
+
+
+4. Raise FastJsonparser::ParseError when invalid JSON provided for parsing
 
 ```
 FastJsonparser.parse("123: 1") # FastJsonparser::ParseError (parse error)

benchmark/nginx_json_logs.json

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+require 'benchmark'
+require 'json'
+require './lib/fast_jsonparser/fast_jsonparser'
+require 'yajl'
+
+def run_report(rep, src)
+  n = 30
+  rep.report("FastJsonparser") do
+    n.times do
+      FastJsonparser.load_many(src) {}
+    end
+  end
+  rep.report("YAJL") do
+    n.times do
+      parser = Yajl::Parser.new
+      parser.parse(File.new(src, 'r')) {}
+    end
+  end
+end
+
+Benchmark.bm do |rep|
+  run_report(rep, './benchmark/nginx_json_logs.json')
+end
+
+
+# nginx_json_logs.json 12M
+#                   user     system      total        real
+# FastJsonparser  3.844446   0.141822   3.986268 (  3.884655)
+# YAJL            9.699621   0.110060   9.809681 (  9.826104) ~ 150% more

benchmark/stream_benchmark.rb

@@ -93,6 +93,27 @@ static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg)
     return Qnil;
 }
 
+static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg)
+{
+    Check_Type(arg, T_STRING);
+
+    dom::parser parser;
+    auto [docs, error] = parser.load_many(RSTRING_PTR(arg));
+    if (error == SUCCESS)
+    {
+        for (dom::element doc : docs)
+        {
+            if (rb_block_given_p())
+            {
+                rb_yield(make_ruby_object(doc));
+            }
+        }
+        return Qnil;
+    }
+    rb_raise(rb_eFastJsonparserParseError, "parse error");
+    return Qnil;
+}
+
 extern "C"
 {
 
@@ -102,5 +123,6 @@ extern "C"
         rb_eFastJsonparserParseError = rb_define_class_under(rb_mFastJsonparser, "ParseError", rb_eStandardError);
         rb_define_module_function(rb_mFastJsonparser, "parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 1);
         rb_define_module_function(rb_mFastJsonparser, "load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 1);
+        rb_define_module_function(rb_mFastJsonparser, "load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 1);
     }
 }