Relative tokenizer

lib/elixir/lib/code/fragment.ex

@@ -1138,12 +1138,13 @@ defmodule Code.Fragment do
           {:ok, Macro.t()} | {:error, {location :: keyword, binary | {binary, binary}, binary}}
   def container_cursor_to_quoted(fragment, opts \\ []) do
     {trailing_fragment, opts} = Keyword.pop(opts, :trailing_fragment)
-    opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder])
+    opts = Keyword.take(opts, [:columns, :token_metadata, :literal_encoder, :mode])
     opts = [check_terminators: {:cursor, []}, emit_warnings: false] ++ opts
 
     file = Keyword.get(opts, :file, "nofile")
     line = Keyword.get(opts, :line, 1)
     column = Keyword.get(opts, :column, 1)
+    mode = Keyword.get(opts, :mode, :absolute)
 
     case :elixir_tokenizer.tokenize(to_charlist(fragment), line, column, opts) do
       {:ok, line, column, _warnings, rev_tokens, rev_terminators}
@@ -1158,7 +1159,7 @@ defmodule Code.Fragment do
             _ -> {rev_tokens, rev_terminators}
           end
 
-        tokens = reverse_tokens(line, column, rev_tokens, rev_terminators)
+        tokens = reverse_tokens(line, column, rev_tokens, rev_terminators, mode)
         :elixir.tokens_to_quoted(tokens, file, opts)
 
       {:ok, line, column, _warnings, rev_tokens, rev_terminators} ->
@@ -1171,7 +1172,7 @@ defmodule Code.Fragment do
                {:error, {meta, _, ~c"end"}, _rest, _warnings, trailing_rev_tokens} <-
                  :elixir_tokenizer.tokenize(to_charlist(trailing_fragment), line, column, opts) do
             trailing_tokens =
-              reverse_tokens(meta[:line], meta[:column], trailing_rev_tokens, after_start)
+              reverse_tokens(meta[:line], meta[:column], trailing_rev_tokens, after_start, mode)
 
             # If the cursor has its own line, then we do not trim new lines trailing tokens.
             # Otherwise we want to drop any newline so we drop the next tokens after eol.
@@ -1183,7 +1184,7 @@ defmodule Code.Fragment do
 
             Enum.reverse(rev_tokens, drop_tokens(trailing_tokens, 0))
           else
-            _ -> reverse_tokens(line, column, rev_tokens, rev_terminators)
+            _ -> reverse_tokens(line, column, rev_tokens, rev_terminators, mode)
           end
 
         :elixir.tokens_to_quoted(tokens, file, opts)
@@ -1193,12 +1194,19 @@ defmodule Code.Fragment do
     end
   end
 
-  defp reverse_tokens(line, column, tokens, terminators) do
+  defp reverse_tokens(line, column, tokens, terminators, mode) do
     {terminators, _} =
-      Enum.map_reduce(terminators, column, fn {start, _, _}, column ->
+      Enum.map_reduce(terminators, {column, 1}, fn {start, _, _, _}, {column, prev_length} ->
         atom = :elixir_tokenizer.terminator(start)
 
-        {{atom, {line, column, nil}}, column + length(Atom.to_charlist(atom))}
+        meta =
+          case mode do
+            :relative -> {0, prev_length, nil}
+            _ -> {line, column, nil}
+          end
+
+        length = length(Atom.to_charlist(atom))
+        {{atom, meta}, {column + length, length}}
       end)
 
     Enum.reverse(tokens, terminators)

lib/elixir/src/elixir.hrl

@@ -77,5 +77,7 @@
   indentation=0,
   column=1,
   mismatch_hints=[],
-  warnings=[]
+  warnings=[],
+  mode=absolute,
+  prev_pos={1, 1}
 }).

lib/elixir/src/elixir_interpolation.erl

@@ -62,14 +62,28 @@ extract([$#, ${ | Rest], Buffer, Output, Line, Column, Scope, true, Last) ->
       NewScope = Scope#elixir_tokenizer{warnings=Warnings},
       {line, EndLine} = lists:keyfind(line, 1, Location),
       {column, EndColumn} = lists:keyfind(column, 1, Location),
-      Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens), Output1),
+
+      {Line1, Column1, EndLine1, EndColumn1} = case Scope of
+        #elixir_tokenizer{mode=relative, prev_pos={PrevLine, PrevColumn}} ->
+          {Line - PrevLine, Column - PrevColumn, EndLine - PrevLine, EndColumn - PrevColumn};
+        _ ->
+          {Line, Column, EndLine, EndColumn}
+      end,
+
+      Output2 = build_interpol(Line1, Column1, EndLine1, EndColumn1, lists:reverse(Tokens), Output1),
       extract(NewRest, [], Output2, EndLine, EndColumn + 1, NewScope, true, Last);
     {error, Reason, _, _, _} ->
       {error, Reason};
     {ok, EndLine, EndColumn, Warnings, Tokens, Terminators} when Scope#elixir_tokenizer.cursor_completion /= false ->
       NewScope = Scope#elixir_tokenizer{warnings=Warnings, cursor_completion=noprune},
-      {CursorTerminators, _} = cursor_complete(EndLine, EndColumn, Terminators),
-      Output2 = build_interpol(Line, Column, EndLine, EndColumn, lists:reverse(Tokens, CursorTerminators), Output1),
+      {CursorTerminators, _} = cursor_complete(EndLine, EndColumn, Terminators, NewScope),
+      {Line1, Column1, EndLine1, EndColumn1} = case Scope of
+        #elixir_tokenizer{mode=relative, prev_pos={PrevLine, PrevColumn}} ->
+          {Line - PrevLine, Column - PrevColumn, EndLine - PrevLine, EndColumn - PrevColumn};
+        _ ->
+          {Line, Column, EndLine, EndColumn}
+      end,
+      Output2 = build_interpol(Line1, Column1, EndLine1, EndColumn1, lists:reverse(Tokens, CursorTerminators), Output1),
       extract([], [], Output2, EndLine, EndColumn, NewScope, true, Last);
     {ok, _, _, _, _, _} ->
       {error, {string, Line, Column, "missing interpolation terminator: \"}\"", []}}
@@ -122,13 +136,19 @@ strip_horizontal_space([H | T], Buffer, Counter) when H =:= $\s; H =:= $\t ->
 strip_horizontal_space(T, Buffer, Counter) ->
   {T, Buffer, Counter}.
 
-cursor_complete(Line, Column, Terminators) ->
+cursor_complete(Line, Column, Terminators, Scope) ->
   lists:mapfoldl(
-    fun({Start, _, _}, AccColumn) ->
+    fun({Start, _, _, _StartPos}, {AccColumn, PrevLength}) ->
       End = elixir_tokenizer:terminator(Start),
-      {{End, {Line, AccColumn, nil}}, AccColumn + length(erlang:atom_to_list(End))}
+      Meta = case Scope of
+        #elixir_tokenizer{mode=relative} ->
+          {0, PrevLength, nil};
+        _ -> {Line, AccColumn, nil}
+      end,
+      Length = length(erlang:atom_to_list(End)),
+      {{End, Meta}, {AccColumn + Length, Length}}
     end,
-    Column,
+    {Column, 1},
     Terminators
   ).