Fix delimiter metadata for single quote atoms and remote calls

Author: jonatanklosko

lib/elixir/src/elixir_parser.yrl

@@ -284,9 +284,9 @@ access_expr -> list_heredoc : build_list_heredoc('$1').
 access_expr -> bitstring : '$1'.
 access_expr -> sigil : build_sigil('$1').
 access_expr -> atom : handle_literal(?exprs('$1'), '$1').
-access_expr -> atom_quoted : handle_literal(?exprs('$1'), '$1', delimiter(<<$">>)).
-access_expr -> atom_safe : build_quoted_atom('$1', true, delimiter(<<$">>)).
-access_expr -> atom_unsafe : build_quoted_atom('$1', false, delimiter(<<$">>)).
+access_expr -> atom_quoted : handle_literal(?exprs('$1'), '$1', atom_delimiter('$1')).
+access_expr -> atom_safe : build_quoted_atom('$1', true, atom_delimiter('$1')).
+access_expr -> atom_unsafe : build_quoted_atom('$1', false, atom_delimiter('$1')).
 access_expr -> dot_alias : '$1'.
 access_expr -> parens_call : '$1'.
 
@@ -1033,6 +1033,12 @@ build_quoted_atom({_, Location, Args}, Safe, ExtraMeta) ->
 binary_to_atom_op(true)  -> binary_to_existing_atom;
 binary_to_atom_op(false) -> binary_to_atom.
 
+atom_delimiter({_Kind, {_Line, _Column, Delimiter}, _Args}) ->
+  case ?token_metadata() of
+    true -> [{delimiter, <<Delimiter>>}];
+    false -> []
+  end.
+
 charlist_parts(Parts) ->
   [charlist_part(Part) || Part <- Parts].
 charlist_part(Binary) when is_binary(Binary) ->

lib/elixir/src/elixir_tokenizer.erl

@@ -511,7 +511,7 @@ tokenize([$:, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H)
         {ok, [Part]} when is_binary(Part) ->
           case unsafe_to_atom(Part, Line, Column, Scope) of
             {ok, Atom} ->
-              Token = {atom_quoted, {Line, Column, nil}, Atom},
+              Token = {atom_quoted, {Line, Column, H}, Atom},
               tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]);
 
             {error, Reason} ->
@@ -523,7 +523,7 @@ tokenize([$:, H | T] = Original, Line, Column, Scope, Tokens) when ?is_quote(H)
             true  -> atom_safe;
             false -> atom_unsafe
           end,
-          Token = {Key, {Line, Column, nil}, Unescaped},
+          Token = {Key, {Line, Column, H}, Unescaped},
           tokenize(Rest, NewLine, NewColumn, NewScope, [Token | Tokens]);
 
         {error, Reason} ->
@@ -919,7 +919,7 @@ handle_dot([$., H | T] = Original, Line, Column, DotInfo, Scope, Tokens) when ?i
       case unsafe_to_atom(UnescapedPart, Line, Column, NewScope) of
         {ok, Atom} ->
           Token = check_call_identifier(Line, Column, Part, Atom, Rest),
-          DotInfo1 = setelement(3, DotInfo, $"),
+          DotInfo1 = setelement(3, DotInfo, H),
           TokensSoFar = add_token_with_eol({'.', DotInfo1}, Tokens),
           tokenize(Rest, NewLine, NewColumn, NewScope, [Token | TokensSoFar]);
 

lib/elixir/test/elixir/kernel/parser_test.exs

@@ -128,17 +128,29 @@ defmodule Kernel.ParserTest do
     end
 
     test "handles graphemes inside quoted identifiers" do
+      string_to_quoted =
+        fn code ->
+          Code.string_to_quoted!(code,
+            token_metadata: true,
+            literal_encoder: &{:ok, {:__block__, &2, [&1]}}
+          )
+        end
+
       assert {
                {:., _, [{:foo, _, nil}, :"➡️"]},
                [no_parens: true, delimiter: ~S["], line: 1],
                []
-             } = Code.string_to_quoted!(~S|foo."➡️"|, token_metadata: true)
+             } = string_to_quoted.(~S|foo."➡️"|)
 
       assert {
                {:., _, [{:foo, _, nil}, :"➡️"]},
-               [closing: [line: 1], delimiter: ~S["], line: 1],
+               [no_parens: true, delimiter: ~S['], line: 1],
                []
-             } = Code.string_to_quoted!(~S|foo."➡️"()|, token_metadata: true)
+             } = string_to_quoted.(~S|foo.'➡️'|)
+
+      assert {:__block__, [delimiter: ~S["], line: 1], [:"➡️"]} = string_to_quoted.(~S|:"➡️"|)
+
+      assert {:__block__, [delimiter: ~S['], line: 1], [:"➡️"]} = string_to_quoted.(~S|:'➡️'|)
     end
   end