Add Code.Fragment.container_cursor_to_quoted/2 (#11315)

Author: josevalim

lib/eex/lib/eex/tokenizer.ex

@@ -71,14 +71,14 @@ defmodule EEx.Tokenizer do
       {:ok, expr, new_line, new_column, rest} ->
         {key, expr} =
           case :elixir_tokenizer.tokenize(expr, 1, file: "eex", check_terminators: false) do
-            {:ok, warnings, tokens} ->
-              Enum.each(warnings, fn {line, file, msg} ->
+            {:ok, _line, _column, warnings, tokens} ->
+              Enum.each(Enum.reverse(warnings), fn {line, file, msg} ->
                 :elixir_errors.erl_warn(line, file, msg)
               end)
 
               token_key(tokens, expr)
 
-            {:error, _, _, _} ->
+            {:error, _, _, _, _} ->
               {:expr, expr}
           end
 

lib/elixir/lib/code/fragment.ex

@@ -104,8 +104,7 @@ defmodule Code.Fragment do
 
   The current algorithm only considers the last line of the input. This means
   it will also show suggestions inside strings, heredocs, etc, which is
-  intentional as it helps with doctests, references, and more. Other functions
-  may be added in the future that consider the tree-structure of the code.
+  intentional as it helps with doctests, references, and more.
   """
   @doc since: "1.13.0"
   @spec cursor_context(List.Chars.t(), keyword()) ::
@@ -405,10 +404,10 @@ defmodule Code.Fragment do
 
   defp operator(rest, count, acc, _call_op?) do
     case :elixir_tokenizer.tokenize(acc, 1, 1, []) do
-      {:ok, _, [{:atom, _, _}]} ->
+      {:ok, _, _, _, [{:atom, _, _}]} ->
         {{:unquoted_atom, tl(acc)}, count}
 
-      {:ok, _, [{_, _, op}]} ->
+      {:ok, _, _, _, [{_, _, op}]} ->
         {rest, dot_count} = strip_spaces(rest, count)
 
         cond do
@@ -720,4 +719,116 @@ defmodule Code.Fragment do
 
   defp enum_reverse_at([h | t], n, acc) when n > 0, do: enum_reverse_at(t, n - 1, [h | acc])
   defp enum_reverse_at(rest, _, acc), do: {acc, rest}
+
+  @doc """
+  Receives a code fragment and returns a quoted expression
+  with a cursor at the nearest argument position.
+
+  A container is any Elixir expression starting with `(`,
+  `{`, and `[`. This includes function calls, tuples, lists,
+  maps, and so on. For example, take this code, which would
+  be given as input:
+
+      max(some_value,
+
+  This function will return the AST equivalent to:
+
+      max(some_value, __cursor__())
+
+  In other words, this function is capable of closing any open
+  brackets and insert the cursor position. Any content at the
+  cursor position that is after a comma or an opening bracket
+  is discarded. For example, if this is given as input:
+
+      max(some_value, another_val
+
+  It will return the same AST:
+
+      max(some_value, __cursor__())
+
+  Similarly, if only this is given:
+
+      max(some_va
+
+  Then it returns:
+
+      max(__cursor__())
+
+  Calls without parenthesis are also supported, as we assume the
+  brackets are implicit.
+
+  Operators and anonymous functions are not containers, and therefore
+  will be discarded. The following will all return the same AST:
+
+      max(some_value,
+      max(some_value, fn x -> x end
+      max(some_value, 1 + another_val
+      max(some_value, 1 |> some_fun() |> another_fun
+
+  On the other hand, tuples, lists, maps, etc all retain the
+  cursor position:
+
+      max(some_value, [1, 2,
+
+  Returns the following AST:
+
+      max(some_value, [1, 2, __cursor__()])
+
+  Keyword lists (and do-end blocks) are also retained. The following:
+
+      if(some_value, do:
+      if(some_value, do: :token
+      if(some_value, do: 1 + val
+
+  all return:
+
+      if(some_value, do: __cursor__())
+
+  The AST returned by this function is not safe to evaluate but
+  it can be analyzed and expanded.
+
+  ## Examples
+
+      iex> Code.Fragment.container_cursor_to_quoted("max(some_value, ")
+      {:ok, {:max, [line: 1], [{:some_value, [line: 1], nil}, {:__cursor__, [line: 1], []}]}}
+
+  ## Options
+
+    * `:file` - the filename to be reported in case of parsing errors.
+      Defaults to `"nofile"`.
+
+    * `:line` - the starting line of the string being parsed.
+      Defaults to 1.
+
+    * `:column` - the starting column of the string being parsed.
+      Defaults to 1.
+
+    * `:columns` - when `true`, attach a `:column` key to the quoted
+      metadata. Defaults to `false`.
+
+  """
+  @doc since: "1.13.0"
+  @spec container_cursor_to_quoted(List.Chars.t(), keyword()) ::
+          {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}}
+  def container_cursor_to_quoted(fragment, opts \\ []) do
+    file = Keyword.get(opts, :file, "nofile")
+    line = Keyword.get(opts, :line, 1)
+    column = Keyword.get(opts, :column, 1)
+    columns = Keyword.get(opts, :columns, false)
+    fragment = to_charlist(fragment)
+    tokenizer_opts = [file: file, cursor_completion: true, columns: columns]
+
+    case :elixir_tokenizer.tokenize(fragment, line, column, tokenizer_opts) do
+      {:ok, _, _, _warnings, tokens} ->
+        :elixir.tokens_to_quoted(tokens, file, columns: columns)
+
+      {:error, {line, column, {prefix, suffix}, token}, _rest, _warnings, _so_far} ->
+        location = [line: line, column: column]
+        {:error, {location, {to_string(prefix), to_string(suffix)}, to_string(token)}}
+
+      {:error, {line, column, error, token}, _rest, _warnings, _so_far} ->
+        location = [line: line, column: column]
+        {:error, {location, to_string(error), to_string(token)}}
+    end
+  end
 end

lib/elixir/src/elixir.erl

@@ -348,13 +348,13 @@ quoted_to_erl(Quoted, Env, Scope) ->
 
 string_to_tokens(String, StartLine, StartColumn, File, Opts) when is_integer(StartLine), is_binary(File) ->
   case elixir_tokenizer:tokenize(String, StartLine, StartColumn, [{file, File} | Opts]) of
-    {ok, Warnings, Tokens} ->
-      [elixir_errors:erl_warn(L, F, M) || {L, F, M} <- Warnings],
+    {ok, _Line, _Column, Warnings, Tokens} ->
+      [elixir_errors:erl_warn(L, F, M) || {L, F, M} <- lists:reverse(Warnings)],
       {ok, Tokens};
-    {error, {Line, Column, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _SoFar} ->
+    {error, {Line, Column, {ErrorPrefix, ErrorSuffix}, Token}, _Rest, _Warnings, _SoFar} ->
       Location = [{line, Line}, {column, Column}],
       {error, {Location, {to_binary(ErrorPrefix), to_binary(ErrorSuffix)}, to_binary(Token)}};
-    {error, {Line, Column, Error, Token}, _Rest, _SoFar} ->
+    {error, {Line, Column, Error, Token}, _Rest, _Warnings, _SoFar} ->
       Location = [{line, Line}, {column, Column}],
       {error, {Location, to_binary(Error), to_binary(Token)}}
   end.

lib/elixir/src/elixir.hrl

@@ -28,6 +28,7 @@
   file=(<<"nofile">>),
   terminators=[],
   unescape=true,
+  cursor_completion=false,
   existing_atoms_only=false,
   static_atoms_encoder=nil,
   preserve_comments=nil,

lib/elixir/src/elixir_interpolation.erl

@@ -9,72 +9,78 @@ unescape_tokens/1, unescape_map/1]).
 
 %% Extract string interpolations
 
-extract(Line, Column, Raw, Interpol, String, Last) ->
-  %% Ignore whatever is in the scope and enable terminator checking.
-  Scope = Raw#elixir_tokenizer{terminators=[]},
-  extract(Line, Column, Scope, Interpol, String, [], [], Last).
+extract(Line, Column, Scope, Interpol, String, Last) ->
+  extract(String, [], [], Line, Column, Scope, Interpol, Last).
 
 %% Terminators
 
-extract(Line, Column, _Scope, _Interpol, [], _Buffer, _Output, Last) ->
+extract([], _Buffer, _Output, Line, Column, #elixir_tokenizer{cursor_completion=false}, _Interpol, Last) ->
   {error, {string, Line, Column, io_lib:format("missing terminator: ~ts", [[Last]]), []}};
 
-extract(Line, Column, _Scope, _Interpol, [Last | Rest], Buffer, Output, Last) ->
-  finish_extraction(Line, Column + 1, Buffer, Output, Rest);
+extract([], Buffer, Output, Line, Column, Scope, _Interpol, _Last) ->
+  finish_extraction([], Buffer, Output, Line, Column, Scope);
+
+extract([Last | Rest], Buffer, Output, Line, Column, Scope, _Interpol, Last) ->
+  finish_extraction(Rest, Buffer, Output, Line, Column + 1, Scope);
 
 %% Going through the string
 
-extract(Line, _Column, Scope, Interpol, [$\\, $\r, $\n | Rest], Buffer, Output, Last) ->
-  extract_nl(Line, Scope, Interpol, Rest, [$\n, $\r, $\\ | Buffer], Output, Last);
+extract([$\\, $\r, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) ->
+  extract_nl(Rest, [$\n, $\r, $\\ | Buffer], Output, Line, Scope, Interpol, Last);
 
-extract(Line, _Column, Scope, Interpol, [$\\, $\n | Rest], Buffer, Output, Last) ->
-  extract_nl(Line, Scope, Interpol, Rest, [$\n, $\\ | Buffer], Output, Last);
+extract([$\\, $\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) ->
+  extract_nl(Rest, [$\n, $\\ | Buffer], Output, Line, Scope, Interpol, Last);
 
-extract(Line, _Column, Scope, Interpol, [$\n | Rest], Buffer, Output, Last) ->
-  extract_nl(Line, Scope, Interpol, Rest, [$\n | Buffer], Output, Last);
+extract([$\n | Rest], Buffer, Output, Line, _Column, Scope, Interpol, Last) ->
+  extract_nl(Rest, [$\n | Buffer], Output, Line, Scope, Interpol, Last);
 
-extract(Line, Column, Scope, Interpol, [$\\, Last | Rest], Buffer, Output, Last) ->
-  extract(Line, Column+2, Scope, Interpol, Rest, [Last | Buffer], Output, Last);
+extract([$\\, Last | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) ->
+  extract(Rest, [Last | Buffer], Output, Line, Column+2, Scope, Interpol, Last);
 
-extract(Line, Column, Scope, true, [$\\, $#, ${ | Rest], Buffer, Output, Last) ->
-  extract(Line, Column+1, Scope, true, Rest, [${, $#, $\\ | Buffer], Output, Last);
+extract([$\\, $#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) ->
+  extract(Rest, [${, $#, $\\ | Buffer], Output, Line, Column+1, Scope, true, Last);
 
-extract(Line, Column, Scope, true, [$#, ${ | Rest], Buffer, Output, Last) ->
-  Output1 = build_string(Line, Buffer, Output),
-  case elixir_tokenizer:tokenize(Rest, Line, Column + 2, Scope) of
-    {error, {EndLine, EndColumn, _, "}"}, [$} | NewRest], Tokens} ->
-      Output2 = build_interpol(Line, Column, EndLine, EndColumn, Tokens, Output1),
-      extract(EndLine, EndColumn + 1, Scope, true, NewRest, [], Output2, Last);
-    {error, Reason, _, _} ->
+extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) ->
+  Output1 = build_string(Buffer, Output),
+  case elixir_tokenizer:tokenize(Rest, Line, Column + 2, Scope#elixir_tokenizer{terminators=[]}) of
+    {error, {EndLine, EndColumn, _, "}"}, [$} | NewRest], Warnings, Tokens} ->
+      NewScope = Scope#elixir_tokenizer{warnings=Warnings},
+      Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens), Output1),
+      extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last);
+    {error, Reason, _, _, _} ->
       {error, Reason};
-    {ok, _, _} ->
+    {ok, EndLine, EndColumn, Warnings, Tokens} when Scope#elixir_tokenizer.cursor_completion /= false ->
+      NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=terminators},
+      Output2 = build_interpol(Line, Column, EndLine, EndColumn, Tokens, Output1),
+      extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last);
+    {ok, _, _, _, _} ->
       {error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}}
   end;
 
-extract(Line, Column, Scope, Interpol, [$\\, Char | Rest], Buffer, Output, Last) ->
-  extract(Line, Column + 2, Scope, Interpol, Rest, [Char, $\\ | Buffer], Output, Last);
+extract([$\\, Char | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last) ->
+  extract(Rest, [Char, $\\ | Buffer], Output, Line, Column + 2, Scope, Interpol, Last);
 
 %% Catch all clause
 
-extract(Line, Column, Scope, Interpol, [Char1, Char2 | Rest], Buffer, Output, Last)
+extract([Char1, Char2 | Rest], Buffer, Output, Line, Column, Scope, Interpol, Last)
     when Char1 =< 255, Char2 =< 255 ->
-  extract(Line, Column + 1, Scope, Interpol, [Char2 | Rest], [Char1 | Buffer], Output, Last);
+  extract([Char2 | Rest], [Char1 | Buffer], Output, Line, Column + 1, Scope, Interpol, Last);
 
-extract(Line, Column, Scope, Interpol, Rest, Buffer, Output, Last) ->
+extract(Rest, Buffer, Output, Line, Column, Scope, Interpol, Last) ->
   [Char | NewRest] = unicode_util:gc(Rest),
-  extract(Line, Column + 1, Scope, Interpol, NewRest, [Char | Buffer], Output, Last).
+  extract(NewRest, [Char | Buffer], Output, Line, Column + 1, Scope, Interpol, Last).
 
 %% Handle newlines. Heredocs require special attention
 
-extract_nl(Line, Scope, Interpol, Rest, Buffer, Output, [H,H,H] = Last) ->
+extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, [H,H,H] = Last) ->
   case strip_horizontal_space(Rest, Buffer, 1) of
     {[H,H,H|NewRest], _NewBuffer, Column} ->
-      finish_extraction(Line + 1, Column + 3, Buffer, Output, NewRest);
+      finish_extraction(NewRest, Buffer, Output, Line + 1, Column + 3, Scope);
     {NewRest, NewBuffer, Column} ->
-      extract(Line + 1, Column, Scope, Interpol, NewRest, NewBuffer, Output, Last)
+      extract(NewRest, NewBuffer, Output, Line + 1, Column, Scope, Interpol, Last)
   end;
-extract_nl(Line, Scope, Interpol, Rest, Buffer, Output, Last) ->
-  extract(Line + 1, 1, Scope, Interpol, Rest, Buffer, Output, Last).
+extract_nl(Rest, Buffer, Output, Line, Scope, Interpol, Last) ->
+  extract(Rest, Buffer, Output, Line + 1, 1, Scope, Interpol, Last).
 
 strip_horizontal_space([H | T], Buffer, Counter) when H =:= $\s; H =:= $\t ->
   strip_horizontal_space(T, [H | Buffer], Counter + 1);
@@ -243,16 +249,16 @@ unescape_map(E)  -> E.
 
 % Extract Helpers
 
-finish_extraction(Line, Column, Buffer, Output, Remaining) ->
-  Final = case build_string(Line, Buffer, Output) of
+finish_extraction(Remaining, Buffer, Output, Line, Column, Scope) ->
+  Final = case build_string(Buffer, Output) of
     [] -> [[]];
     F  -> F
   end,
 
-  {Line, Column, lists:reverse(Final), Remaining}.
+  {Line, Column, lists:reverse(Final), Remaining, Scope}.
 
-build_string(_Line, [], Output) -> Output;
-build_string(_Line, Buffer, Output) -> [lists:reverse(Buffer) | Output].
+build_string([], Output) -> Output;
+build_string(Buffer, Output) -> [lists:reverse(Buffer) | Output].
 
 build_interpol(Line, Column, EndLine, EndColumn, Buffer, Output) ->
-  [{{Line, Column, nil}, {EndLine, EndColumn, nil}, lists:reverse(Buffer)} | Output].
+  [{{Line, Column, nil}, {EndLine, EndColumn, nil}, Buffer} | Output].