Handle line continuations for Ripper :on_sp events

Co-authored-by: Earlopain <[email protected]>

Author: eregon

lib/prism/lex_compat.rb

@@ -874,21 +874,44 @@ def add_on_sp_tokens(tokens)
         line, column = token.location
         start_offset = @source.line_to_byte_offset(line) + column
         if start_offset > prev_token_end
-          new_tokens << IgnoreStateToken.new([
-            [
-              @source.line(prev_token_end),
-              @source.column(prev_token_end),
-            ],
-            :on_sp,
-            @source.slice(prev_token_end, start_offset - prev_token_end),
-            prev_token_state
-          ])
+          sp_value = @source.slice(prev_token_end, start_offset - prev_token_end)
+          sp_line = @source.line(prev_token_end)
+          sp_column = @source.column(prev_token_end)
+          continuation_index = sp_value.byteindex("\\")
+          # ripper emits up to three on_sp tokens when line continuations are used
+          if continuation_index
+            next_whitespace_index = continuation_index + 1
+            next_whitespace_index += 1 if sp_value.byteslice(next_whitespace_index) == "\r"
+            first_whitespace = sp_value[0...continuation_index]
+            continuation = sp_value[continuation_index..next_whitespace_index]
+            second_whitespace = sp_value[next_whitespace_index+1..]
+            new_tokens << IgnoreStateToken.new([
+              [sp_line, sp_column],
+              :on_sp,
+              first_whitespace,
+              prev_token_state
+            ]) if first_whitespace != ""
+            new_tokens << IgnoreStateToken.new([
+              [sp_line, sp_column + continuation_index],
+              :on_sp,
+              continuation,
+              prev_token_state
+            ])
+            new_tokens << IgnoreStateToken.new([
+              [sp_line + 1, 0],
+              :on_sp,
+              second_whitespace,
+              prev_token_state
+            ]) if second_whitespace != ""
+          else
+            new_tokens << IgnoreStateToken.new([
+              [sp_line, @source.column(prev_token_end)],
+              :on_sp,
+              sp_value,
+              prev_token_state
+            ])
+          end
         end
-        new_tokens << token
-
-        prev_token_state = token.state
-        prev_token_end = start_offset + token.value.bytesize
-      end
 
       new_tokens
     end