Remove line and convert text tokens to char lists

Author: José Valim

lib/eex/lib/eex/compiler.ex

@@ -17,8 +17,8 @@ defmodule EEx.Compiler do
 
   # Generates the buffers by handling each expression from the tokenizer
 
-  defp generate_buffer([{ :text, _line, chars }|t], buffer, scope, state) do
-    buffer = state.engine.handle_text(buffer, chars)
+  defp generate_buffer([{ :text, chars }|t], buffer, scope, state) do
+    buffer = state.engine.handle_text(buffer, String.from_char_list!(chars))
     generate_buffer(t, buffer, scope, state)
   end
 

lib/eex/lib/eex/tokenizer.ex

@@ -16,39 +16,39 @@ defmodule EEx.Tokenizer do
   end
 
   def tokenize(list, line) do
-    Enum.reverse(tokenize(list, line, line, [], []))
+    Enum.reverse(tokenize(list, line, [], []))
   end
 
-  defp tokenize('<%%' ++ t, current_line, line, buffer, acc) do
+  defp tokenize('<%%' ++ t, line, buffer, acc) do
     { buffer, new_line, rest } = tokenize_expr t, line, [?%, ?<|buffer]
-    tokenize rest, current_line, new_line, [?>, ?%|buffer], acc
+    tokenize rest, new_line, [?>, ?%|buffer], acc
   end
 
-  defp tokenize('<%#' ++ t, current_line, line, buffer, acc) do
+  defp tokenize('<%#' ++ t, line, buffer, acc) do
     { _, new_line, rest } = tokenize_expr t, line, []
-    tokenize rest, current_line, new_line, buffer, acc
+    tokenize rest, new_line, buffer, acc
   end
 
-  defp tokenize('<%' ++ t, current_line, line, buffer, acc) do
+  defp tokenize('<%' ++ t, line, buffer, acc) do
     { marker, t } = retrieve_marker(t)
     { expr, new_line, rest } = tokenize_expr t, line, []
 
     token = token_name(expr)
-    acc   = tokenize_text(current_line, buffer, acc)
+    acc   = tokenize_text(buffer, acc)
     final = { token, line, marker, Enum.reverse(expr) }
-    tokenize rest, new_line, new_line, [], [final | acc]
+    tokenize rest, new_line, [], [final | acc]
   end
 
-  defp tokenize('\n' ++ t, current_line, line, buffer, acc) do
-    tokenize t, current_line, line + 1, [?\n|buffer], acc
+  defp tokenize('\n' ++ t, line, buffer, acc) do
+    tokenize t, line + 1, [?\n|buffer], acc
   end
 
-  defp tokenize([h|t], current_line, line, buffer, acc) do
-    tokenize t, current_line, line, [h|buffer], acc
+  defp tokenize([h|t], line, buffer, acc) do
+    tokenize t, line, [h|buffer], acc
   end
 
-  defp tokenize([], current_line, _line, buffer, acc) do
-    tokenize_text(current_line, buffer, acc)
+  defp tokenize([], _line, buffer, acc) do
+    tokenize_text(buffer, acc)
   end
 
   # Retrieve marker for <%
@@ -149,11 +149,11 @@ defmodule EEx.Tokenizer do
   # Tokenize the buffered text by appending
   # it to the given accumulator.
 
-  defp tokenize_text(_line, [], acc) do
+  defp tokenize_text([], acc) do
     acc
   end
 
-  defp tokenize_text(line, buffer, acc) do
-    [{ :text, line, String.from_char_list!(Enum.reverse(buffer)) } | acc]
+  defp tokenize_text(buffer, acc) do
+    [{ :text, Enum.reverse(buffer) } | acc]
   end
 end

lib/eex/test/eex/tokenizer_test.exs

@@ -5,23 +5,23 @@ defmodule EEx.TokenizerTest do
   require EEx.Tokenizer, as: T
 
   test "simple chars lists" do
-    assert T.tokenize('foo', 1) == [ { :text, 1, "foo" } ]
+    assert T.tokenize('foo', 1) == [ { :text, 'foo' } ]
   end
 
   test "simple strings" do
-    assert T.tokenize("foo", 1) == [ { :text, 1, "foo" } ]
+    assert T.tokenize("foo", 1) == [ { :text, 'foo' } ]
   end
 
   test "strings with embedded code" do
-    assert T.tokenize('foo <% bar %>', 1) == [ { :text, 1, "foo " }, { :expr, 1, "", ' bar ' } ]
+    assert T.tokenize('foo <% bar %>', 1) == [ { :text, 'foo ' }, { :expr, 1, "", ' bar ' } ]
   end
 
   test "strings with embedded equals code" do
-    assert T.tokenize('foo <%= bar %>', 1) == [ { :text, 1, "foo " }, { :expr, 1, "=", ' bar ' } ]
+    assert T.tokenize('foo <%= bar %>', 1) == [ { :text, 'foo ' }, { :expr, 1, "=", ' bar ' } ]
   end
 
   test "strings with more than one line" do
-    assert T.tokenize('foo\n<%= bar %>', 1) == [ { :text, 1, "foo\n" }, { :expr, 2, "=", ' bar ' } ]
+    assert T.tokenize('foo\n<%= bar %>', 1) == [ { :text, 'foo\n' }, { :expr, 2, "=", ' bar ' } ]
   end
 
   test "strings with more than one line and expression with more than one line" do
@@ -33,66 +33,66 @@ baz %>
 '''
 
     assert T.tokenize(string, 1) == [
-      {:text, 1, "foo "},
+      {:text, 'foo '},
       {:expr, 1, "=", ' bar\n\nbaz '},
-      {:text, 3, "\n"},
+      {:text, '\n'},
       {:expr, 4, "", ' foo '},
-      {:text, 4, "\n"}
+      {:text, '\n'}
     ] 
   end
 
   test "quotation" do
     assert T.tokenize('foo <%% true %>', 1) == [
-      { :text, 1, "foo <% true %>" }
+      { :text, 'foo <% true %>' }
     ]
   end
 
   test "quotation with do/end" do
     assert T.tokenize('foo <%% true do %>bar<%% end %>', 1) == [
-      { :text, 1, "foo <% true do %>bar<% end %>" }
+      { :text, 'foo <% true do %>bar<% end %>' }
     ]
   end
 
   test "comments" do
     assert T.tokenize('foo <%# true %>', 1) == [
-      { :text, 1, "foo " }
+      { :text, 'foo ' }
     ]
   end
 
   test "comments with do/end" do
     assert T.tokenize('foo <%# true do %>bar<%# end %>', 1) == [
-      { :text, 1, "foo bar" }
+      { :text, 'foo bar' }
     ]
   end
 
   test "strings with embedded do end" do
     assert T.tokenize('foo <% if true do %>bar<% end %>', 1) == [
-      { :text, 1, "foo " },
+      { :text, 'foo ' },
       { :start_expr, 1, "", ' if true do ' },
-      { :text, 1, "bar" },
+      { :text, 'bar' },
       { :end_expr, 1, "", ' end ' }
     ]
   end
 
   test "strings with embedded -> end" do
     assert T.tokenize('foo <% cond do %><% false -> %>bar<% true -> %>baz<% end %>', 1) == [
-      { :text, 1, "foo " },
+      { :text, 'foo ' },
       { :start_expr, 1, "", ' cond do ' },
       { :middle_expr, 1, "", ' false -> ' },
-      { :text, 1, "bar" },
+      { :text, 'bar' },
       { :middle_expr, 1, "", ' true -> ' },
-      { :text, 1, "baz" },
+      { :text, 'baz' },
       { :end_expr, 1, "", ' end ' }
     ]
   end
 
   test "strings with embedded keywords blocks" do
     assert T.tokenize('foo <% if true do %>bar<% else %>baz<% end %>', 1) == [
-      { :text, 1, "foo " },
+      { :text, 'foo ' },
       { :start_expr, 1, "", ' if true do ' },
-      { :text, 1, "bar" },
+      { :text, 'bar' },
       { :middle_expr, 1, "", ' else ' },
-      { :text, 1, "baz" },
+      { :text, 'baz' },
       { :end_expr, 1, "", ' end ' }
     ]
   end