[ruby/json] Handle non-string keys returning immediate values via to_s

We can't directly call `RBASIC_CLASS` as the return value of
`to_s` may be an immediate.

ruby/json@12dc394d11

Author: byroot

ext/json/generator/generator.c

@@ -789,6 +789,21 @@ struct hash_foreach_arg {
     int iter;
 };
 
+static VALUE
+convert_string_subclass(VALUE key)
+{
+    VALUE key_to_s = rb_funcall(key, i_to_s, 0);
+
+    if (RB_UNLIKELY(rb_type(key_to_s) != T_STRING)) {
+        VALUE cname = rb_obj_class(key);
+        rb_raise(rb_eTypeError,
+                 "can't convert %"PRIsVALUE" to %s (%"PRIsVALUE"#%s gives %"PRIsVALUE")",
+                 cname, "String", cname, "to_s", rb_obj_class(key_to_s));
+    }
+
+    return key_to_s;
+}
+
 static int
 json_object_i(VALUE key, VALUE val, VALUE _arg)
 {
@@ -817,7 +832,7 @@ json_object_i(VALUE key, VALUE val, VALUE _arg)
             if (RB_LIKELY(RBASIC_CLASS(key) == rb_cString)) {
                 key_to_s = key;
             } else {
-                key_to_s = rb_funcall(key, i_to_s, 0);
+                key_to_s = convert_string_subclass(key);
             }
             break;
         case T_SYMBOL:

test/json/json_generator_test.rb

@@ -605,6 +605,22 @@ def test_string_subclass_with_to_s
     assert_equal '{"JSONGeneratorTest::StringWithToS#to_s":1}', JSON.generate(StringWithToS.new => 1)
   end
 
+  def test_string_subclass_with_broken_to_s
+    klass = Class.new(String) do
+      def to_s
+        false
+      end
+    end
+    s = klass.new("test")
+    assert_equal '["test"]', JSON.generate([s])
+
+    omit("Can't figure out how to match behavior in java code") if RUBY_PLATFORM == "java"
+
+    assert_raise TypeError do
+      JSON.generate(s => 1)
+    end
+  end
+
   if defined?(JSON::Ext::Generator) and RUBY_PLATFORM != "java"
     def test_valid_utf8_in_different_encoding
       utf8_string = "€™"