Support token_metadata in container_cursor_to_quoted

Author: josevalim

lib/elixir/lib/code/fragment.ex

@@ -806,6 +806,11 @@ defmodule Code.Fragment do
     * `:columns` - when `true`, attach a `:column` key to the quoted
       metadata. Defaults to `false`.
 
+    * `:token_metadata` - when `true`, includes token-related
+      metadata in the expression AST, such as metadata for `do` and `end`
+      tokens, for closing tokens, end of expressions, as well as delimiters
+      for sigils. See `t:Macro.metadata/0`. Defaults to `false`.
+
   """
   @doc since: "1.13.0"
   @spec container_cursor_to_quoted(List.Chars.t(), keyword()) ::
@@ -815,12 +820,14 @@ defmodule Code.Fragment do
     line = Keyword.get(opts, :line, 1)
     column = Keyword.get(opts, :column, 1)
     columns = Keyword.get(opts, :columns, false)
+    token_metadata = Keyword.get(opts, :token_metadata, false)
+
     fragment = to_charlist(fragment)
     tokenizer_opts = [file: file, cursor_completion: true, columns: columns]
 
     case :elixir_tokenizer.tokenize(fragment, line, column, tokenizer_opts) do
       {:ok, _, _, _warnings, tokens} ->
-        :elixir.tokens_to_quoted(tokens, file, columns: columns)
+        :elixir.tokens_to_quoted(tokens, file, columns: columns, token_metadata: token_metadata)
 
       {:error, {line, column, {prefix, suffix}, token}, _rest, _warnings, _so_far} ->
         location = [line: line, column: column]

lib/elixir/lib/macro.ex

@@ -1772,9 +1772,7 @@ defmodule Macro do
   def quoted_literal?({:{}, _, args}), do: quoted_literal?(args)
   def quoted_literal?({left, right}), do: quoted_literal?(left) and quoted_literal?(right)
   def quoted_literal?(list) when is_list(list), do: Enum.all?(list, &quoted_literal?/1)
-
-  def quoted_literal?(term),
-    do: is_atom(term) or is_number(term) or is_binary(term)
+  def quoted_literal?(term), do: is_atom(term) or is_number(term) or is_binary(term)
 
   @doc """
   Receives an AST node and expands it until it can no longer

lib/elixir/test/elixir/code_fragment_test.exs

@@ -908,9 +908,12 @@ defmodule CodeFragmentTest do
       assert cc2q("foo(bar do :done end |>") == s2q("foo(__cursor__())")
     end
 
-    test "returns column information" do
+    test "options" do
       assert cc2q("foo(", columns: true) == s2q("foo(__cursor__())", columns: true)
       assert cc2q("foo(123,", columns: true) == s2q("foo(123,__cursor__())", columns: true)
+
+      assert cc2q("foo(", token_metadata: true) == s2q("foo(__cursor__())", token_metadata: true)
+      assert cc2q("foo(123,", token_metadata: true) == s2q("foo(123,__cursor__())", token_metadata: true)
     end
   end
 end