Skip to content

Commit 66e3a15

Browse files
anilmauryaanilmaurya
authored
Merge pull request #15 from casperisfine/nested-use
Handle concurrent use of the parser
2 parents 20f4bc7 + 37c08ad commit 66e3a15

File tree

3 files changed

+64
-92
lines changed

3 files changed

+64
-92
lines changed

‎ext/fast_jsonparser/fast_jsonparser.cpp‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,38 @@
55
VALUE rb_eFastJsonparserUnknownError, rb_eFastJsonparserParseError;
66

77
using namespace simdjson;
8-
static dom::parser parser;
8+
9+
typedef struct {
10+
dom::parser *parser;
11+
} parser_t;
12+
13+
static void Parser_delete(void *ptr) {
14+
parser_t *data = (parser_t*) ptr;
15+
delete data->parser;
16+
}
17+
18+
static size_t Parser_memsize(const void *parser) {
19+
return sizeof(dom::parser); // TODO: low priority, figure the real size, e.g. internal buffers etc.
20+
}
21+
22+
static const rb_data_type_t parser_data_type = {
23+
"Parser",
24+
{ 0, Parser_delete, Parser_memsize, },
25+
0, 0, RUBY_TYPED_FREE_IMMEDIATELY
26+
};
27+
28+
static VALUE parser_allocate(VALUE klass) {
29+
parser_t *data;
30+
VALUE obj = TypedData_Make_Struct(klass, parser_t, &parser_data_type, data);
31+
data->parser = new dom::parser;
32+
return obj;
33+
}
34+
35+
static inline dom::parser * get_parser(VALUE self) {
36+
parser_t *data;
37+
TypedData_Get_Struct(self, parser_t, &parser_data_type, data);
38+
return data->parser;
39+
}
940

1041
// Convert tape to Ruby's Object
1142
static VALUE make_ruby_object(dom::element element, bool symbolize_keys)
@@ -71,8 +102,9 @@ static VALUE make_ruby_object(dom::element element, bool symbolize_keys)
71102
static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg, VALUE symbolize_keys)
72103
{
73104
Check_Type(arg, T_STRING);
105+
dom::parser *parser = get_parser(self);
74106

75-
auto [doc, error] = parser.parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
107+
auto [doc, error] = parser->parse(RSTRING_PTR(arg), RSTRING_LEN(arg));
76108
if (error != SUCCESS)
77109
{
78110
rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
@@ -83,8 +115,9 @@ static VALUE rb_fast_jsonparser_parse(VALUE self, VALUE arg, VALUE symbolize_key
83115
static VALUE rb_fast_jsonparser_load(VALUE self, VALUE arg, VALUE symbolize_keys)
84116
{
85117
Check_Type(arg, T_STRING);
118+
dom::parser *parser = get_parser(self);
86119

87-
auto [doc, error] = parser.load(RSTRING_PTR(arg));
120+
auto [doc, error] = parser->load(RSTRING_PTR(arg));
88121
if (error != SUCCESS)
89122
{
90123
rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
@@ -96,9 +129,10 @@ static VALUE rb_fast_jsonparser_load_many(VALUE self, VALUE arg, VALUE symbolize
96129
{
97130
Check_Type(arg, T_STRING);
98131
Check_Type(batch_size, T_FIXNUM);
132+
dom::parser *parser = get_parser(self);
99133

100134
try {
101-
auto [docs, error] = parser.load_many(RSTRING_PTR(arg), FIX2INT(batch_size));
135+
auto [docs, error] = parser->load_many(RSTRING_PTR(arg), FIX2INT(batch_size));
102136
if (error != SUCCESS)
103137
{
104138
rb_raise(rb_eFastJsonparserParseError, "%s", error_message(error));
@@ -123,10 +157,12 @@ extern "C"
123157
void Init_fast_jsonparser(void)
124158
{
125159
VALUE rb_mFastJsonparser = rb_const_get(rb_cObject, rb_intern("FastJsonparser"));
160+
VALUE rb_cFastJsonparserNative = rb_const_get(rb_mFastJsonparser, rb_intern("Native"));
126161

127-
rb_define_module_function(rb_mFastJsonparser, "_parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 2);
128-
rb_define_module_function(rb_mFastJsonparser, "_load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 2);
129-
rb_define_module_function(rb_mFastJsonparser, "_load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 3);
162+
rb_define_alloc_func(rb_cFastJsonparserNative, parser_allocate);
163+
rb_define_method(rb_cFastJsonparserNative, "_parse", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_parse), 2);
164+
rb_define_method(rb_cFastJsonparserNative, "_load", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load), 2);
165+
rb_define_method(rb_cFastJsonparserNative, "_load_many", reinterpret_cast<VALUE (*)(...)>(rb_fast_jsonparser_load_many), 3);
130166

131167
rb_eFastJsonparserParseError = rb_const_get(rb_mFastJsonparser, rb_intern("ParseError"));
132168
rb_global_variable(&rb_eFastJsonparserParseError);

‎lib/fast_jsonparser.rb‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,15 @@ module FastJsonparser
1212

1313
class << self
1414
def parse(source, symbolize_keys: true)
15-
_parse(source, symbolize_keys)
15+
parser._parse(source, symbolize_keys)
1616
end
1717

1818
def load(source, symbolize_keys: true)
19-
_load(source, symbolize_keys)
19+
parser._load(source, symbolize_keys)
2020
end
2121

2222
def load_many(source, symbolize_keys: true, batch_size: DEFAULT_BATCH_SIZE, &block)
23-
_load_many(source, symbolize_keys, batch_size, &block)
23+
Native.new._load_many(source, symbolize_keys, batch_size, &block)
2424
rescue UnknownError => error
2525
case error.message
2626
when "This parser can't support a document that big"
@@ -30,7 +30,17 @@ def load_many(source, symbolize_keys: true, batch_size: DEFAULT_BATCH_SIZE, &blo
3030
end
3131
end
3232

33-
require "fast_jsonparser/fast_jsonparser" # loads cpp extension
34-
private :_parse, :_load, :_load_many
33+
private
34+
35+
def parser
36+
@parser ||= Native.new
37+
end
38+
end
39+
40+
class Native
3541
end
42+
43+
require "fast_jsonparser/fast_jsonparser" # loads cpp extension
44+
45+
private_constant :Native
3646
end

‎test/fast_jsonparser_test.rb‎

Lines changed: 6 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -70,85 +70,11 @@ def test_load_many_batch_size
7070
end
7171
end
7272

73-
def test_compat_forward_slash_escape
74-
assert_compat('"\/"', "/")
75-
end
76-
77-
def test_compat_backward_slash_escape
78-
assert_compat('"\\\\"', '\\')
79-
end
80-
81-
def test_compat_illegal_escape
82-
refute_compat('["Illegal backslash escape: \x15"]', ["Illegal backslash escape: x15"], :raises)
83-
end
84-
85-
def test_compat_utf8
86-
assert_compat('"École"', "École")
87-
end
88-
89-
def test_compat_NaN_and_Infinity
90-
assert_compat('[NaN]', :raises)
91-
assert_compat('[Infinity]', :raises)
92-
assert_compat('[-Infinity]', :raises)
93-
end
94-
95-
def test_compat_hex_numbers
96-
assert_compat('[0x42]', :raises)
97-
end
98-
99-
def test_compat_weird_keys
100-
assert_compat('{[1]:2}', :raises)
101-
assert_compat('{{1:2}:2}', :raises)
102-
assert_compat('{null:2}', :raises)
103-
end
104-
105-
def test_compat_trailing_commas
106-
assert_compat('{1:2,}', :raises)
107-
assert_compat('{,,,,}', :raises)
108-
assert_compat('[1,]', :raises)
109-
assert_compat('[,,,,]', :raises)
110-
end
111-
112-
def test_compat_trailing_comments
113-
assert_compat('{} // comment', :raises)
114-
refute_compat('{} /* comment */', {}, :raises)
115-
assert_compat('{1:/*comment*/2}', :raises)
116-
refute_compat('{"a":/*comment*/"b"}', { "a" => "b" }, :raises)
117-
end
118-
119-
def test_compat_float_precision
120-
assert_compat '1.3553e142', 1.3553e142
121-
assert_compat '1.3553E142', 1.3553E142
122-
end
123-
124-
def test_compat_big_floats
125-
assert_compat '100000000000000000000000000000000000000000.0', 100000000000000000000000000000000000000000.0
126-
end
127-
128-
def test_compat_big_integers
129-
assert_compat '18446744073709551615', 18446744073709551615
130-
refute_compat '18446744073709551616', 18446744073709551616, :raises
131-
132-
assert_compat '-9223372036854775808', -9223372036854775808
133-
refute_compat '-9223372036854775809', -9223372036854775809, :raises
134-
end
135-
136-
private
137-
138-
def assert_compat(source, expected)
139-
assert_equal expected, parse(JSON, source), "This test is invalid"
140-
assert_equal expected, parse(FastJsonparser, source), "#{source.inspect} is not parsed the same by JSON and FastJsonparser"
141-
end
142-
143-
def refute_compat(source, expected, got)
144-
refute_equal expected, got
145-
assert_equal expected, parse(JSON, source)
146-
assert_equal got, parse(FastJsonparser, source)
147-
end
148-
149-
def parse(parser, source)
150-
parser.parse(source)
151-
rescue => error
152-
:raises
73+
def test_nested_usage
74+
FastJsonparser.load_many('./benchmark/nginx_json_logs.json') do
75+
test_json_load_from_file_is_working
76+
test_json_parse_is_working
77+
break
78+
end
15379
end
15480
end

0 commit comments

Comments
 (0)