From f5d44566afeef55b8b918e4235d9a01c108fa460 Mon Sep 17 00:00:00 2001 From: Lukasz Samson Date: Mon, 18 Aug 2025 21:59:21 +0200 Subject: [PATCH] Handle identifier followed by dual_op across line continuation --- lib/elixir/src/elixir_tokenizer.erl | 21 ++++++++++++++++++++- lib/elixir/test/erlang/tokenizer_test.erl | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 3c38062ec17..3d8ccb69a6b 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -723,7 +723,26 @@ unexpected_token([T | Rest], Line, Column, Scope, Tokens) -> tokenize_eol(Rest, Line, Scope, Tokens) -> {StrippedRest, Column} = strip_horizontal_space(Rest, Scope#elixir_tokenizer.column), IndentedScope = Scope#elixir_tokenizer{indentation=Column-1}, - tokenize(StrippedRest, Line + 1, Column, IndentedScope, Tokens). + %% If we arrived here via a line continuation (\\\n or \\\r\n), there is no EOL token added. + %% In such case, if the previous token is an identifier and the next non-space token + %% is a dual operator sign (+ or -) followed by a non-marker, we should mirror the + %% same-line space-sensitive behavior and convert the identifier to op_identifier. + %% This makes: foo\\\n-1 and foo\\\n +1 behave like foo -1 and foo +1. + case {Tokens, StrippedRest} of + {[{identifier, _, _} = H | TokensTail], [Sign, NotMarker | T]} -> + case previous_was_eol(Tokens) of + %% Only apply across line continuations (no previous EOL recorded) + nil when ?dual_op(Sign), not(?is_space(NotMarker)), NotMarker =/= Sign, NotMarker =/= $/, NotMarker =/= $> -> + DualOpToken = {dual_op, {Line + 1, Column, nil}, list_to_atom([Sign])}, + Rest2 = [NotMarker | T], + tokenize(Rest2, Line + 1, Column + 1, IndentedScope, + [DualOpToken, setelement(1, H, op_identifier) | TokensTail]); + _ -> + tokenize(StrippedRest, Line + 1, Column, IndentedScope, Tokens) + end; + _ -> + tokenize(StrippedRest, Line + 1, Column, IndentedScope, Tokens) + end. strip_horizontal_space([H | T], Counter) when ?is_horizontal_space(H) -> strip_horizontal_space(T, Counter + 1); diff --git a/lib/elixir/test/erlang/tokenizer_test.erl b/lib/elixir/test/erlang/tokenizer_test.erl index b383a74b1c3..b9f128f0fdb 100644 --- a/lib/elixir/test/erlang/tokenizer_test.erl +++ b/lib/elixir/test/erlang/tokenizer_test.erl @@ -184,6 +184,22 @@ space_test() -> {dual_op, {1, 6, nil}, '-'}, {int, {1, 7, 2}, "2"}] = tokenize("foo -2"). +op_identifier_line_continuation_test() -> + %% Backslash followed by LF + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {2, 1, nil}, '+'}, + {int, {2, 2, 1}, "1"}] = tokenize("foo\\\n+1"), + + %% Backslash followed by CRLF + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {2, 1, nil}, '+'}, + {int, {2, 2, 1}, "1"}] = tokenize("foo\\\r\n+1"), + + %% Backslash-LF then a space before '+' + [{op_identifier, {1, 1, _}, foo}, + {dual_op, {2, 2, nil}, '+'}, + {int, {2, 3, 1}, "1"}] = tokenize("foo\\\n +1"). + chars_test() -> [{char, {1, 1, "?a"}, 97}] = tokenize("?a"), [{char, {1, 1, "?c"}, 99}] = tokenize("?c"),