Update bidi/line break character checks according to UX#55

Author: josevalim

lib/elixir/src/elixir_interpolation.erl

@@ -89,9 +89,12 @@ extract(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) ->
 
 extract_char(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) ->
   case unicode_util:gc(Rest) of
-    [Char | _] when ?bidi(Char) ->
+    [Char | _] when ?bidi(Char); ?break(Char) ->
       Token = io_lib:format("\\u~4.16.0B", [Char]),
-      Pre = "invalid bidirectional formatting character in string: ",
+      Pre = if
+        ?bidi(Char) -> "invalid bidirectional formatting character in string: ";
+        true -> "invalid line break character in string: "
+      end,
       Pos = io_lib:format(". If you want to use such character, use it in its escaped ~ts form instead", [Token]),
       {error, {?LOC(Line, Column), {Pre, Pos}, Token}};
 

lib/elixir/src/elixir_tokenizer.erl

@@ -198,8 +198,8 @@ tokenize([$0, $o, H | T], Line, Column, Scope, Tokens) when ?is_octal(H) ->
 
 tokenize([$# | String], Line, Column, Scope, Tokens) ->
   case tokenize_comment(String, [$#]) of
-    {error, Char} ->
-      error_comment(Char, [$# | String], Line, Column, Scope, Tokens);
+    {error, Char, Reason} ->
+      error_comment(Char, Reason, [$# | String], Line, Column, Scope, Tokens);
     {Rest, Comment} ->
       preserve_comments(Line, Column, Tokens, Comment, Rest, Scope),
       tokenize(Rest, Line, Column, Scope, reset_eol(Tokens))
@@ -748,8 +748,8 @@ tokenize_dot(T, Line, Column, DotInfo, Scope, Tokens) ->
   case strip_horizontal_space(T, 0) of
     {[$# | R], _} ->
       case tokenize_comment(R, [$#]) of
-        {error, Char} ->
-          error_comment(Char, [$# | R], Line, Column, Scope, Tokens);
+        {error, Char, Reason} ->
+          error_comment(Char, Reason, [$# | R], Line, Column, Scope, Tokens);
 
         {Rest, Comment} ->
           preserve_comments(Line, Column, Tokens, Comment, Rest, Scope),
@@ -1315,16 +1315,17 @@ tokenize_comment("\r\n" ++ _ = Rest, Acc) ->
 tokenize_comment("\n" ++ _ = Rest, Acc) ->
   {Rest, lists:reverse(Acc)};
 tokenize_comment([H | _Rest], _) when ?bidi(H) ->
-  {error, H};
+  {error, H, "invalid bidirectional formatting character in comment: "};
+tokenize_comment([H | _Rest], _) when ?break(H) ->
+  {error, H, "invalid line break character in comment: "};
 tokenize_comment([H | Rest], Acc) ->
   tokenize_comment(Rest, [H | Acc]);
 tokenize_comment([], Acc) ->
   {[], lists:reverse(Acc)}.
 
-error_comment(H, Comment, Line, Column, Scope, Tokens) ->
-  Token = io_lib:format("\\u~4.16.0B", [H]),
-  Reason = {?LOC(Line, Column), "invalid bidirectional formatting character in comment: ", Token},
-  error(Reason, Comment, Scope, Tokens).
+error_comment(Char, Reason, Comment, Line, Column, Scope, Tokens) ->
+  Token = io_lib:format("\\u~4.16.0B", [Char]),
+  error({?LOC(Line, Column), Reason, Token}, Comment, Scope, Tokens).
 
 preserve_comments(Line, Column, Tokens, Comment, Rest, Scope) ->
   case Scope#elixir_tokenizer.preserve_comments of

lib/elixir/src/elixir_tokenizer.hrl

@@ -33,3 +33,11 @@
                  C =:= 16#2068;
                  C =:= 16#202C;
                  C =:= 16#2069).
+
+%% Unsupported newlines
+%% https://www.unicode.org/reports/tr55/
+-define(break(C), C =:= 16#000B;
+                  C =:= 16#000C;
+                  C =:= 16#0085;
+                  C =:= 16#2028;
+                  C =:= 16#2029).

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -982,6 +982,34 @@ defmodule Kernel.ParserTest do
       )
     end
 
+    test "invalid newline in source" do
+      assert_syntax_error(
+        ["nofile:1:1:", ~s/invalid line break character in comment: \\u2028/],
+        ~c"# This is a \u2028"
+      )
+
+      assert_syntax_error(
+        ["nofile:1:5:", "invalid line break character in comment: \\u2028"],
+        ~c"foo. # This is a \u2028"
+      )
+
+      assert_syntax_error(
+        [
+          "nofile:1:12:",
+          "invalid line break character in string: \\u2028. If you want to use such character, use it in its escaped \\u2028 form instead"
+        ],
+        ~c"\"this is a \u2028\""
+      )
+
+      assert_syntax_error(
+        [
+          "nofile:1:13:",
+          "invalid line break character in string: \\u2028. If you want to use such character, use it in its escaped \\u2028 form instead"
+        ],
+        ~c"\"this is a \\\u2028\""
+      )
+    end
+
     test "reserved tokens" do
       assert_syntax_error(["nofile:1:1:", "reserved token: __aliases__"], ~c"__aliases__")
       assert_syntax_error(["nofile:1:1:", "reserved token: __block__"], ~c"__block__")