[GR-19220] Implement rb_enc_strlen (#2707)

PullRequest: truffleruby/3464

Author: eregon

CHANGELOG.md

@@ -28,6 +28,7 @@ Compatibility:
 * Fix `Regexp.new` to coerce non-String arguments (#2705, @andrykonchin).
 * Fix `Kernel#sprintf` formatting for `%c` when used non-ASCII encoding (#2369, @andrykonchin).
 * Fix `Kernel#sprintf` argument casting for `%c` (@andrykonchin).
+* Implement the `rb_enc_strlen` function for use by native extensions (@nirvdrum).
 
 Performance:
 

lib/cext/ABI_version.txt

@@ -1 +1 @@
-8
+9

spec/ruby/optional/capi/encoding_spec.rb

@@ -63,6 +63,48 @@
     end
   end
 
+  describe "rb_enc_strlen" do
+    before :each do
+      @str = 'こにちわ' # Each codepoint in this string is 3 bytes in UTF-8
+    end
+
+    it "returns the correct string length for the encoding" do
+      @s.rb_enc_strlen(@str, @str.bytesize, Encoding::UTF_8).should == 4
+      @s.rb_enc_strlen(@str, @str.bytesize, Encoding::BINARY).should == 12
+    end
+
+    it "returns the string length based on a fixed-width encoding's character length, even if the encoding is incompatible" do
+      @s.rb_enc_strlen(@str, @str.bytesize, Encoding::UTF_16BE).should == 6
+      @s.rb_enc_strlen(@str, @str.bytesize, Encoding::UTF_16LE).should == 6
+      @s.rb_enc_strlen(@str, @str.bytesize, Encoding::UTF_32BE).should == 3
+      @s.rb_enc_strlen(@str, @str.bytesize, Encoding::UTF_32LE).should == 3
+    end
+
+    it "does not consider strings to be NUL-terminated" do
+      s = "abc\0def"
+      @s.rb_enc_strlen(s, s.bytesize, Encoding::US_ASCII).should == 7
+      @s.rb_enc_strlen(s, s.bytesize, Encoding::UTF_8).should == 7
+    end
+
+    describe "handles broken strings" do
+      it "combines valid character and invalid character counts in UTF-8" do
+        # The result is 3 because `rb_enc_strlen` counts the first valid character and then adds
+        # the byte count for the invalid character that follows for 1 + 2.
+        @s.rb_enc_strlen(@str, 5, Encoding::UTF_8).should == 3
+      end
+
+      it "combines valid character and invalid character counts in UTF-16" do
+        @s.rb_enc_strlen(@str, 5, Encoding::UTF_16BE).should == 3
+      end
+
+      it "rounds up for fixed-width encodings" do
+        @s.rb_enc_strlen(@str, 7, Encoding::UTF_32BE).should == 2
+        @s.rb_enc_strlen(@str, 7, Encoding::UTF_32LE).should == 2
+        @s.rb_enc_strlen(@str, 5, Encoding::BINARY).should == 5
+      end
+    end
+  end
+
   describe "rb_enc_find" do
     it "returns the encoding of an Encoding" do
       @s.rb_enc_find("UTF-8").should == "UTF-8"

spec/ruby/optional/capi/ext/encoding_spec.c

@@ -301,6 +301,14 @@ static VALUE encoding_spec_rb_enc_codelen(VALUE self, VALUE code, VALUE encoding
   return INT2FIX(rb_enc_codelen(c, enc));
 }
 
+static VALUE encoding_spec_rb_enc_strlen(VALUE self, VALUE str, VALUE length, VALUE encoding) {
+  int l = FIX2INT(length);
+  char *p = RSTRING_PTR(str);
+  char *e = p + l;
+
+  return LONG2FIX(rb_enc_strlen(p, e, rb_to_encoding(encoding)));
+}
+
 void Init_encoding_spec(void) {
   VALUE cls;
   native_rb_encoding_pointer = (rb_encoding**) malloc(sizeof(rb_encoding*));
@@ -335,6 +343,7 @@ void Init_encoding_spec(void) {
   rb_define_method(cls, "rb_enc_compatible", encoding_spec_rb_enc_compatible, 2);
   rb_define_method(cls, "rb_enc_copy", encoding_spec_rb_enc_copy, 2);
   rb_define_method(cls, "rb_enc_codelen", encoding_spec_rb_enc_codelen, 2);
+  rb_define_method(cls, "rb_enc_strlen", encoding_spec_rb_enc_strlen, 3);
   rb_define_method(cls, "rb_enc_find", encoding_spec_rb_enc_find, 1);
   rb_define_method(cls, "rb_enc_find_index", encoding_spec_rb_enc_find_index, 1);
   rb_define_method(cls, "rb_enc_isalnum", encoding_spec_rb_enc_isalnum, 2);

src/main/c/cext/encoding.c

@@ -246,6 +246,17 @@ int rb_enc_precise_mbclen(const char *p, const char *e, rb_encoding *enc) {
   return polyglot_as_i32(RUBY_CEXT_INVOKE_NO_WRAP("rb_enc_precise_mbclen", rb_tr_temporary_native_string(p, length, enc)));
 }
 
+long rb_enc_strlen(const char *p, const char *e, rb_encoding *enc) {
+  long length = e - p;
+  int minlen = rb_enc_mbminlen(enc);
+
+  if (minlen == rb_enc_mbmaxlen(enc)) {
+    return length / minlen + !!(length % minlen);
+  }
+
+  return polyglot_as_i64(RUBY_CEXT_INVOKE_NO_WRAP("rb_enc_strlen", rb_tr_temporary_native_string(p, length, enc)));
+}
+
 int rb_enc_dummy_p(rb_encoding *enc) {
   return polyglot_as_i32(RUBY_INVOKE_NO_WRAP(rb_enc_from_encoding(enc), "dummy?"));
 }
@@ -433,5 +444,3 @@ int enc_is_unicode(const OnigEncodingType *enc) {
   const char *name = rb_enc_name(enc);
   return !strncmp(name,"UTF", 3);
 }
-
-

src/main/java/org/truffleruby/cext/CExtNodes.java

@@ -1481,6 +1481,21 @@ protected int rbEncPreciseMbclen(Object string,
         }
     }
 
+    @CoreMethod(names = "rb_enc_strlen", onSingleton = true, required = 1)
+    public abstract static class RbEncStrlen extends CoreMethodArrayArgumentsNode {
+
+        @Specialization(guards = "strings.isRubyString(string)", limit = "1")
+        protected int rbEncStrlen(Object string,
+                @Cached RubyStringLibrary strings,
+                @Cached TruffleString.CodePointLengthNode codePointLengthNode) {
+            var tstring = strings.getTString(string);
+            var tencoding = strings.getTEncoding(string);
+
+            return codePointLengthNode.execute(tstring, tencoding);
+        }
+
+    }
+
     @CoreMethod(names = "rb_enc_left_char_head", onSingleton = true, required = 3, lowerFixnum = 3)
     public abstract static class RbEncLeftCharHeadNode extends CoreMethodArrayArgumentsNode {