[GR-34937] Adopt TruffleString in TruffleRuby and replace Rope

PullRequest: truffleruby/3432

Author: eregon

bench/micro/file/write.rb

@@ -10,8 +10,8 @@
 
 kilobyte = 'x' * 1024
 
-if defined?(Truffle::Ropes.flatten_rope)
-  kilobyte = Truffle::Ropes.flatten_rope(kilobyte)
+if defined?(Truffle::Debug.flatten_string)
+  kilobyte = Truffle::Debug.flatten_string(kilobyte)
 end
 
 benchmark 'core-write-kilobyte' do
@@ -20,8 +20,8 @@
 
 gigabyte = 'x' * 1024 * 1024 * 1024
 
-if defined?(Truffle::Ropes.flatten_rope)
-  gigabyte = Truffle::Ropes.flatten_rope(gigabyte)
+if defined?(Truffle::Debug.flatten_string)
+  gigabyte = Truffle::Debug.flatten_string(gigabyte)
 end
 
 benchmark 'core-write-gigabyte' do

bench/micro/string/flatten.rb

@@ -9,7 +9,7 @@
 side = 512 * 1024
 big_string = ("a".b * side + "é".b + "z".b * side)[1...-1]
 result = big_string.byteslice(4, 8)
-# Truffle::Ropes.debug_print_rope(big_string, false)
+# Truffle::Debug.tstring_to_debug_string(big_string)
 
 benchmark "core-string-many-substrings-of-large-substring" do
   i = 0

bench/micro/string/substring.rb

@@ -0,0 +1,37 @@
+# TruffleString in TruffleRuby
+
+TruffleRuby uses `TruffleString` to represent Ruby Strings, but wraps them in either a RubyString or a ImmutableRubyString object.
+
+## Encodings Compatibility
+
+The notion of encodings compatibility is mostly the same between Ruby and TruffleString but differs in one point:
+* An empty Ruby String is always considered compatible with any other Ruby String of any encoding.
+* TruffleString does not consider whether a string is empty or not, and only look at their encodings and code range.
+
+As a result, to use TruffleString equality nodes, one needs to:
+1. Compute the compatible encoding with `NegotiateCompatibleStringEncodingNode` or `Primitive.encoding_ensure_compatible_str`.
+2. Check if both sides are empty, and if so return true before using TruffleString equality nodes.
+
+`StringHelperNodes.StringEqualInternalNode` is a good example showing what is needed.
+
+An example which would throw without empty checks is comparing an empty ISO-2022-JP (a dummy, non-ascii-compatible, fixed-width encoding) string with an empty US-ASCII string:
+
+```bash
+$ jt ruby -e '"".force_encoding("ISO-2022-JP") == ""'
+the given string is not compatible to the expected encoding "ISO_2022_JP", did you forget to convert it? (java.lang.IllegalArgumentException)
+```
+
+## Logical vs Physical Byte Offsets
+
+We categorize a byte offset into a `TruffleString` as either *logical* or *physical*.
+A physical byte offset includes the offset from the `InternalByteArray` (`InternalByteArray#getOffset()`).
+A logical byte offset does not include that and is the semantic byte offset from the start of the string.
+Physical offsets are quite difficult to use and they are error-prone as they can be passed by mistake to a method taking a logical offset.
+So avoid physical offsets as much as possible, and therefore avoid `InternalByteArray#getArray()`.
+
+## Tests
+
+This is a good set of tests to run when touching String code:
+```
+jt test integration strict-encoding-checks
+```

doc/contributor/truffle-string.md

@@ -1 +1 @@
-7
+8

lib/cext/ABI_check.txt

@@ -692,10 +692,6 @@ def rb_thread_alone
     Thread.list.count == 1 ? 1 : 0
   end
 
-  def rb_intern(str)
-    str.intern
-  end
-
   def rb_int_positive_pow(a, b)
     a ** b
   end
@@ -809,14 +805,6 @@ def rb_enc_get_index(obj)
     enc
   end
 
-  def rb_intern_str(string)
-    string.intern
-  end
-
-  def rb_intern3(string, enc)
-    string.force_encoding(enc).intern
-  end
-
   def rb_str_append(str, to_append)
     Primitive.string_append(str, to_append)
   end
@@ -1766,7 +1754,7 @@ def rb_gv_get(name)
   end
 
   def rb_reg_match(re, str)
-    result = str ? Truffle::RegexpOperations.match(re, str, 0) : nil
+    result = Truffle::RegexpOperations.match(re, str, 0)
     Primitive.regexp_last_match_set(rb_get_special_vars(), result)
 
     result.begin(0) if result

lib/truffle/truffle/cext.rb

@@ -159,10 +159,10 @@
   end
 
   it "does not match leading periods in filenames with wildcards by default" do
-    File.send(@method, '*', '.profile').should == false
-    File.send(@method, '*', 'home/.profile').should == true
-    File.send(@method, '*/*', 'home/.profile').should == true
-    File.send(@method, '*/*', 'dave/.profile', File::FNM_PATHNAME).should == false
+    File.should_not.send(@method, '*', '.profile')
+    File.should.send(@method, '*', 'home/.profile')
+    File.should.send(@method, '*/*', 'home/.profile')
+    File.should_not.send(@method, '*/*', 'dave/.profile', File::FNM_PATHNAME)
   end
 
   it "matches patterns with leading periods to dotfiles by default" do

spec/ruby/core/file/shared/fnmatch.rb

@@ -17,6 +17,11 @@
     Regexp.send(@method, str).should == '\+\[\]\('
   end
 
+  it "works for broken strings" do
+    Regexp.send(@method, "a.\x85b.".force_encoding("US-ASCII")).should =="a\\.\x85b\\.".force_encoding("US-ASCII")
+    Regexp.send(@method, "a.\x80".force_encoding("UTF-8")).should == "a\\.\x80".force_encoding("UTF-8")
+  end
+
   it "sets the encoding of the result to US-ASCII if there are only US-ASCII characters present in the input String" do
     str = "abc".force_encoding("euc-jp")
     Regexp.send(@method, str).encoding.should == Encoding::US_ASCII

spec/ruby/core/regexp/shared/quote.rb

@@ -10,6 +10,7 @@
     "hello".capitalize.should == "Hello"
     "HELLO".capitalize.should == "Hello"
     "123ABC".capitalize.should == "123abc"
+    "abcdef"[1...-1].capitalize.should == "Bcde"
   end
 
   describe "full Unicode case mapping" do
@@ -37,7 +38,7 @@
     end
 
     it "handles non-ASCII substrings properly" do
-      "garçon"[1..-1].capitalize(:ascii).should == "Arçon"
+      "garçon"[1...-1].capitalize(:ascii).should == "Arço"
     end
   end
 

spec/ruby/core/string/capitalize_spec.rb

@@ -21,6 +21,10 @@
     r.should == s
   end
 
+  it "does not remove partial bytes, only full characters" do
+    "\xe3\x81\x82".delete_prefix("\xe3").should == "\xe3\x81\x82"
+  end
+
   it "doesn't set $~" do
     $~ = nil