In ERB::Util.tokenize, return :PLAIN when line has no ERB

Previous behavior was to raise. Allows caller to decide what to do
with lines that are plain. Error highlighter assumes PLAIN lines
must be code otherwise they wouldn't be in the backtrace.

Author: martinemde

actionview/lib/action_view/template/handlers/erb.rb

@@ -152,8 +152,9 @@ def find_offset(compiled, source_tokens, error_column)
 
         def offset_source_tokens(source_tokens)
           source_offset = 0
-          with_offset = source_tokens.filter_map do |(name, str)|
-            result = [name, str, source_offset] if name == :CODE || name == :TEXT
+          with_offset = source_tokens.filter_map do |name, str|
+            result = [:CODE, str, source_offset] if name == :CODE || name == :PLAIN
+            result = [:TEXT, str, source_offset] if name == :TEXT
             source_offset += str.bytesize
             result
           end

activesupport/CHANGELOG.md

@@ -1,3 +1,7 @@
+*   Alter `ERB::Util.tokenize` to return :PLAIN token with full input string when string doesn't contain ERB tags.
+
+    *Martin Emde*
+
 *   Fix a bug in `ERB::Util.tokenize` that causes incorrect tokenization when ERB tags are preceeded by multibyte characters.
 
     *Martin Emde*

activesupport/lib/active_support/core_ext/erb/util.rb

@@ -169,7 +169,7 @@ def self.tokenize(source) # :nodoc:
       while !source.eos?
         pos = source.pos
         source.scan_until(/(?:#{start_re}|#{finish_re})/)
-        raise NotImplementedError if source.matched.nil?
+        return [[:PLAIN, source.string]] unless source.matched?
         len = source.pos - source.matched.bytesize - pos
 
         case source.matched

activesupport/test/core_ext/erb_util_test.rb

@@ -135,6 +135,14 @@ def test_text_end
       ], actual_tokens
     end
 
+    # This happens when a template is multiline and no
+    # ERB tags are used on the current line.
+    def test_plain_without_tags
+      source = " @post.title\n"
+      actual_tokens = tokenize source
+      assert_equal [[:PLAIN, " @post.title"]], actual_tokens
+    end
+
     def test_multibyte_characters_start
       source = "こんにちは<%= name %>"
       actual_tokens = tokenize source