[ruby/prism] Fix an incompatibility with the parser translator

Earlopain · kddnewton · commit 7cbaa3b9298b · 2025-01-11T19:09:05.000-05:00
The offset cache contains an entry for each byte so it can't be accessed via the string length. Adds tests for all variants except for this: ``` "fo o" "ba ’" ``` For some reason, this still has the wrong offset. ruby/prism@a651126458
diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb
@@ -1091,7 +1091,7 @@ def visit_interpolated_string_node(node)
                 start_offset = node.content_loc.start_offset
 
                 node.unescaped.lines.map do |line|
-                  end_offset = start_offset + line.length
+                  end_offset = start_offset + line.bytesize
                   offsets = srange_offsets(start_offset, end_offset)
                   start_offset = end_offset
 
@@ -1692,7 +1692,7 @@ def visit_string_node(node)
                 start_offset = node.content_loc.start_offset
 
                 [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line|
-                  end_offset = start_offset + content_line.length
+                  end_offset = start_offset + content_line.bytesize
                   offsets = srange_offsets(start_offset, end_offset)
                   start_offset = end_offset
 
@@ -1747,7 +1747,7 @@ def visit_symbol_node(node)
               start_offset = node.value_loc.start_offset
 
               node.value.lines.map do |line|
-                end_offset = start_offset + line.length
+                end_offset = start_offset + line.bytesize
                 offsets = srange_offsets(start_offset, end_offset)
                 start_offset = end_offset
 
@@ -1890,7 +1890,7 @@ def visit_x_string_node(node)
               start_offset = node.content_loc.start_offset
 
               node.unescaped.lines.map do |line|
-                end_offset = start_offset + line.length
+                end_offset = start_offset + line.bytesize
                 offsets = srange_offsets(start_offset, end_offset)
                 start_offset = end_offset
 
diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
@@ -343,7 +343,7 @@ def to_a
                     adjustment = 0
                   end
 
-                  end_offset = start_offset + adjusted_line.length + adjustment
+                  end_offset = start_offset + adjusted_line.bytesize + adjustment
                   tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]]
                   start_offset = end_offset
                 end
diff --git a/test/prism/fixtures/dstring.txt b/test/prism/fixtures/dstring.txt
@@ -27,3 +27,6 @@ foo\\\\
 "
 foo\\\\\
 "
+
+"
+’"
diff --git a/test/prism/fixtures/dsym_str.txt b/test/prism/fixtures/dsym_str.txt
@@ -1,2 +1,5 @@
 :"foo
   bar"
+
+:"
+’"
diff --git a/test/prism/fixtures/xstring.txt b/test/prism/fixtures/xstring.txt
@@ -11,3 +11,6 @@
 ``
 
 %x{}
+
+`
+’`
diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb
@@ -46,6 +46,7 @@ class RubyParserTest < TestCase
     # https://github.com/seattlerb/ruby_parser/issues/344
     failures = [
       "alias.txt",
+      "dsym_str.txt",
       "dos_endings.txt",
       "heredocs_with_ignored_newlines.txt",
       "method_calls.txt",

Original file line number	Diff line number	Diff line change
`@@ -27,3 +27,6 @@ foo\\\\`
`27`	`27`	`"`
`28`	`28`	`foo\\\\\`
`29`	`29`	`"`
	`30`	`+`
	`31`	`+"`
	`32`	`+’"`
-Original file line number
+Diff line change
@@ @@ -1,2 +1,5 @@ @@
 :"foo
   bar"
++
 +:"
 +’"