Add delimiter meta to demote calls with quoted identifier

Author: jonatanklosko

lib/elixir/src/elixir_parser.yrl

@@ -890,8 +890,15 @@ build_dot_container(Dot, Left, Right, Extra) ->
 
 build_dot(Dot, Left, {_, Location, _} = Right) ->
   Meta = meta_from_token(Dot),
-  IdentifierLocation = meta_from_location(Location),
-  {'.', Meta, IdentifierLocation, [Left, extract_identifier(Right)]}.
+  IdentifierMeta0 = meta_from_location(Location),
+  IdentifierMeta1 =
+    case Location of
+      {_Line, _Column, {_Unencoded, Delimiter}} when Delimiter =/= nil ->
+        delimiter(<<Delimiter>>) ++ IdentifierMeta0;
+      _ ->
+        IdentifierMeta0
+    end,
+  {'.', Meta, IdentifierMeta1, [Left, extract_identifier(Right)]}.
 
 extract_identifier({Kind, _, Identifier}) when
     Kind == identifier; Kind == bracket_identifier; Kind == paren_identifier;
@@ -916,17 +923,17 @@ build_no_parens_do_block(Expr, Args, {BlockMeta, Block}) ->
 build_no_parens(Expr, Args) ->
   build_call(Expr, Args).
 
-build_identifier({'.', Meta, IdentifierLocation, DotArgs}) ->
-  {{'.', Meta, DotArgs}, [{no_parens, true} | IdentifierLocation], []};
+build_identifier({'.', Meta, IdentifierMeta, DotArgs}) ->
+  {{'.', Meta, DotArgs}, [{no_parens, true} | IdentifierMeta], []};
 
 build_identifier({'.', Meta, _} = Dot) ->
   {Dot, [{no_parens, true} | Meta], []};
 
 build_identifier({_, Location, Identifier}) ->
   {Identifier, meta_from_location(Location), nil}.
 
-build_call({'.', Meta, IdentifierLocation, DotArgs}, Args) ->
-  {{'.', Meta, DotArgs}, IdentifierLocation, Args};
+build_call({'.', Meta, IdentifierMeta, DotArgs}, Args) ->
+  {{'.', Meta, DotArgs}, IdentifierMeta, Args};
 
 build_call({'.', Meta, _} = Dot, Args) ->
   {Dot, Meta, Args};

lib/elixir/src/elixir_tokenizer.erl

@@ -540,7 +540,7 @@ tokenize([$: | String] = Original, Line, Column, Scope, Tokens) ->
     {_Kind, Unencoded, Atom, Rest, Length, Ascii, _Special} ->
       NewScope = maybe_warn_for_ambiguous_bang_before_equals(atom, Unencoded, Rest, Line, Column, Scope),
       TrackedScope = track_ascii(Ascii, NewScope),
-      Token = {atom, {Line, Column, Unencoded}, Atom},
+      Token = {atom, {Line, Column, {Unencoded, nil}}, Atom},
       tokenize(Rest, Line, Column + 1 + Length, TrackedScope, [Token | Tokens]);
     empty when Scope#elixir_tokenizer.cursor_completion == false ->
       unexpected_token(Original, Line, Column, Scope, Tokens);
@@ -651,7 +651,7 @@ tokenize(String, Line, Column, OriginalScope, Tokens) ->
 
       case Rest of
         [$: | T] when ?is_space(hd(T)) ->
-          Token = {kw_identifier, {Line, Column, Unencoded}, Atom},
+          Token = {kw_identifier, {Line, Column, {Unencoded, nil}}, Atom},
           tokenize(T, Line, Column + Length + 1, Scope, [Token | Tokens]);
 
         [$: | T] when hd(T) =/= $: ->
@@ -671,7 +671,7 @@ tokenize(String, Line, Column, OriginalScope, Tokens) ->
 
         _ when Kind == identifier ->
           NewScope = maybe_warn_for_ambiguous_bang_before_equals(identifier, Unencoded, Rest, Line, Column, Scope),
-          Token = check_call_identifier(Line, Column, Unencoded, Atom, Rest),
+          Token = check_call_identifier(Line, Column, Unencoded, nil, Atom, Rest),
           tokenize(Rest, Line, Column + Length, NewScope, [Token | Tokens]);
 
         _ ->
@@ -918,7 +918,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, Scope, Tokens) when ?i
 
       case unsafe_to_atom(UnescapedPart, Line, Column, NewScope) of
         {ok, Atom} ->
-          Token = check_call_identifier(Line, Column, Part, Atom, Rest),
+          Token = check_call_identifier(Line, Column, Part, $", Atom, Rest),
           TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens),
           tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]);
 
@@ -937,7 +937,7 @@ handle_dot([$. | Rest], Line, Column, DotInfo, Scope, Tokens) ->
   tokenize(Rest, Line, Column, Scope, TokensSoFar).
 
 handle_call_identifier(Rest, Line, Column, DotInfo, Length, UnencodedOp, Scope, Tokens) ->
-  Token = check_call_identifier(Line, Column, UnencodedOp, list_to_atom(UnencodedOp), Rest),
+  Token = check_call_identifier(Line, Column, UnencodedOp, nil, list_to_atom(UnencodedOp), Rest),
   TokensSoFar = add_token_with_eol({'.', DotInfo}, Tokens),
   tokenize(Rest, Line, Column + Length, Scope, [Token | TokensSoFar]).
 
@@ -1324,18 +1324,18 @@ tokenize_alias(Rest, Line, Column, Unencoded, Atom, Length, Ascii, Special, Scop
       error(Reason, Unencoded ++ Rest, Scope, Tokens);
 
     true ->
-      AliasesToken = {alias, {Line, Column, Unencoded}, Atom},
+      AliasesToken = {alias, {Line, Column, {Unencoded, nil}}, Atom},
       tokenize(Rest, Line, Column + Length, Scope, [AliasesToken | Tokens])
   end.
 
 %% Check if it is a call identifier (paren | bracket | do)
 
-check_call_identifier(Line, Column, Unencoded, Atom, [$( | _]) ->
-  {paren_identifier, {Line, Column, Unencoded}, Atom};
-check_call_identifier(Line, Column, Unencoded, Atom, [$[ | _]) ->
-  {bracket_identifier, {Line, Column, Unencoded}, Atom};
-check_call_identifier(Line, Column, Unencoded, Atom, _Rest) ->
-  {identifier, {Line, Column, Unencoded}, Atom}.
+check_call_identifier(Line, Column, Unencoded, Delimiter, Atom, [$( | _]) ->
+  {paren_identifier, {Line, Column, {Unencoded, Delimiter}}, Atom};
+check_call_identifier(Line, Column, Unencoded, Delimiter, Atom, [$[ | _]) ->
+  {bracket_identifier, {Line, Column, {Unencoded, Delimiter}}, Atom};
+check_call_identifier(Line, Column, Unencoded, Delimiter, Atom, _Rest) ->
+  {identifier, {Line, Column, {Unencoded, Delimiter}}, Atom}.
 
 add_token_with_eol({unary_op, _, _} = Left, T) -> [Left | T];
 add_token_with_eol(Left, [{eol, _} | T]) -> [Left | T];

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -128,7 +128,17 @@ defmodule Kernel.ParserTest do
     end
 
     test "handles graphemes inside quoted identifiers" do
-      assert {{:., _, [{:foo, _, nil}, :"➡️"]}, _, []} = Code.string_to_quoted!(~s|foo."➡️"|)
+      assert {
+               {:., _, [{:foo, _, nil}, :"➡️"]},
+               [no_parens: true, delimiter: ~S["], line: 1],
+               []
+             } = Code.string_to_quoted!(~S|foo."➡️"|, token_metadata: true)
+
+      assert {
+               {:., _, [{:foo, _, nil}, :"➡️"]},
+               [closing: [line: 1], delimiter: ~S["], line: 1],
+               []
+             } = Code.string_to_quoted!(~S|foo."➡️"()|, token_metadata: true)
     end
   end
 

lib/elixir/unicode/security.ex

@@ -40,7 +40,7 @@ defmodule String.Tokenizer.Security do
   ]
 
   defp check_token_for_confusability(
-         {kind, {_line, _column, [_ | _] = name} = info, _},
+         {kind, {_line, _column, {[_ | _] = name, _delimiter}} = info, _},
          skeletons
        )
        when kind in @identifiers do
@@ -50,7 +50,7 @@ defmodule String.Tokenizer.Security do
       {_, _, ^name} ->
         {:ok, skeletons}
 
-      {line, _, previous_name} when name != previous_name ->
+      {line, _, {previous_name, _delimiter}} when name != previous_name ->
         {:warn,
          "confusable identifier: '#{name}' looks like '#{previous_name}' on line #{line}, " <>
            "but they are written using different characters" <> dir_compare(name, previous_name)}