Properly handle UTF-8 keys and values

Author: byroot

ext/fast_jsonparser/fast_jsonparser.cpp

@@ -27,7 +27,7 @@ static VALUE make_ruby_object(dom::element element)
             for (dom::key_value_pair field : dom::object(element))
             {
                 std::string_view view(field.key);
-                VALUE k = rb_intern(view.data());
+                VALUE k = rb_intern_str(rb_utf8_str_new(view.data(), view.size()));
                 VALUE v = make_ruby_object(field.value);
                 rb_hash_aset(hash, ID2SYM(k), v);
             }
@@ -48,7 +48,7 @@ static VALUE make_ruby_object(dom::element element)
         case dom::element_type::STRING:
         {
             std::string_view view(element);
-            return rb_str_new(view.data(), view.size());
+            return rb_utf8_str_new(view.data(), view.size());
         }
         case dom::element_type::BOOL:
         {

test/fast_jsonparser_test.rb

@@ -7,6 +7,17 @@ def test_that_it_has_a_version_number
     refute_nil ::FastJsonparser::VERSION
   end
 
+  def test_string_encoding
+    result = FastJsonparser.parse('"École"')
+    assert_equal Encoding::UTF_8, result.encoding
+  end
+
+  def test_symbols_encoding
+    hash = FastJsonparser.parse('{"École": 1}')
+    assert_includes hash, :"École"
+    assert_equal Encoding::UTF_8, hash.keys.first.encoding
+  end
+
   def test_json_load_from_file_is_working
     result = FastJsonparser.load("./benchmark/graduation.json")
     assert_equal result[:meta].length, 1