Fix column in :elixir_tokenizer for sigils with modifiers (#7529)

This fixes cases where :elixir_tokenizer does not account for sigil modifiers
when determining the column for the next token.

Signed-off-by: José Valim <[email protected]>

Author: rrrene

lib/elixir/src/elixir_tokenizer.erl

@@ -182,7 +182,8 @@ tokenize([$~, S, H, H, H | T] = Original, Line, Column, Scope, Tokens) when ?is_
     {ok, NewLine, NewColumn, Parts, Rest} ->
       {Final, Modifiers} = collect_modifiers(Rest, []),
       Token = {sigil, {Line, Column, nil}, S, Parts, Modifiers, <<H, H, H>>},
-      tokenize(Final, NewLine, NewColumn, Scope, [Token | Tokens]);
+      NewColumn2 = NewColumn + length(Modifiers),
+      tokenize(Final, NewLine, NewColumn2, Scope, [Token | Tokens]);
     {error, Reason} ->
       {error, Reason, Original, Tokens}
   end;
@@ -192,7 +193,8 @@ tokenize([$~, S, H | T] = Original, Line, Column, Scope, Tokens) when ?is_sigil(
     {NewLine, NewColumn, Parts, Rest} ->
       {Final, Modifiers} = collect_modifiers(Rest, []),
       Token = {sigil, {Line, Column, nil}, S, Parts, Modifiers, <<H>>},
-      tokenize(Final, NewLine, NewColumn, Scope, [Token | Tokens]);
+      NewColumn2 = NewColumn + length(Modifiers),
+      tokenize(Final, NewLine, NewColumn2, Scope, [Token | Tokens]);
     {error, Reason} ->
       Sigil = [$~, S, H],
       interpolation_error(Reason, Original, Tokens, " (for sigil ~ts starting at line ~B)", [Sigil, Line])

lib/elixir/test/erlang/tokenizer_test.erl

@@ -208,6 +208,12 @@ sigil_terminator_test() ->
   [{sigil, {1, 1, nil}, 114, [<<"foo">>], [], <<"/">>}] = tokenize("~r/foo/"),
   [{sigil, {1, 1, nil}, 114, [<<"foo">>], [], <<"[">>}] = tokenize("~r[foo]"),
   [{sigil, {1, 1, nil}, 114, [<<"foo">>], [], <<"\"">>}] = tokenize("~r\"foo\""),
+  [{sigil, {1, 1, nil}, 114, [<<"foo">>], [], <<"/">>},
+   {comp_op, {1, 9, nil}, '=='},
+   {identifier, {1, 12, nil}, bar}] = tokenize("~r/foo/ == bar"),
+  [{sigil, {1, 1, nil}, 114, [<<"foo">>], "iu", <<"/">>},
+   {comp_op, {1, 11, nil}, '=='},
+   {identifier, {1, 14, nil}, bar}] = tokenize("~r/foo/iu == bar"),
   [{sigil, {1, 1, nil}, 83, [<<"sigil heredoc\n">>], [], <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),
   [{sigil, {1, 1, nil}, 83, [<<"sigil heredoc\n">>], [], <<"'''">>}] = tokenize("~S'''\nsigil heredoc\n'''").