Merge pull request rails#53655 from martinemde/martinemde/byteslice-erb-tokenize

Fix multibyte character tokenization bug in ERB::Util

Author: byroot

activesupport/CHANGELOG.md

@@ -1,3 +1,7 @@
+*   Fix a bug in `ERB::Util.tokenize` that causes incorrect tokenization when ERB tags are preceeded by multibyte characters.
+
+    *Martin Emde*
+
 *   Add `ActiveSupport::Testing::NotificationAssertions` module to help with testing `ActiveSupport::Notifications`.
 
     *Nicholas La Roux*, *Yishu See*, *Sean Doyle*

activesupport/lib/active_support/core_ext/erb/util.rb

@@ -174,7 +174,7 @@ def self.tokenize(source) # :nodoc:
 
         case source.matched
         when start_re
-          tokens << [:TEXT, source.string[pos, len]] if len > 0
+          tokens << [:TEXT, source.string.byteslice(pos, len)] if len > 0
           tokens << [:OPEN, source.matched]
           if source.scan(/(.*?)(?=#{finish_re}|\z)/m)
             tokens << [:CODE, source.matched] unless source.matched.empty?
@@ -183,7 +183,7 @@ def self.tokenize(source) # :nodoc:
             raise NotImplementedError
           end
         when finish_re
-          tokens << [:CODE, source.string[pos, len]] if len > 0
+          tokens << [:CODE, source.string.byteslice(pos, len)] if len > 0
           tokens << [:CLOSE, source.matched]
         else
           raise NotImplementedError, source.matched

activesupport/test/core_ext/erb_util_test.rb

@@ -135,6 +135,24 @@ def test_text_end
       ], actual_tokens
     end
 
+    def test_multibyte_characters_start
+      source = "こんにちは<%= name %>"
+      actual_tokens = tokenize source
+      assert_equal [[:TEXT, "こんにちは"],
+                    [:OPEN, "<%="],
+                    [:CODE, " name "],
+                    [:CLOSE, "%>"],
+      ], actual_tokens
+    end
+
+    def test_multibyte_characters_end
+      source = " 'こんにちは' %>"
+      actual_tokens = tokenize source
+      assert_equal [[:CODE, " 'こんにちは' "],
+                    [:CLOSE, "%>"],
+      ], actual_tokens
+    end
+
     def tokenize(source)
       ERB::Util.tokenize source
     end