Fix on_words_sep for ripper translator with newlines

Ripper emits a token each per line.

Author: Earlopain

lib/prism/lex_compat.rb

@@ -657,6 +657,16 @@ def result
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
           when :on_embexpr_end
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
+          when :on_words_sep
+            # Ripper emits one token each per line. This splits the
+            # string at newlines and emits one token for each match.
+            parts = value.scan(/(?=\s)[^\S\r\n]*(?:\r\n|\n)?/)
+            parts[0...-1].each do |whitespace|
+              tokens << Token.new([[lineno, column], event, whitespace, lex_state])
+              lineno += 1
+              column = 0
+            end
+            Token.new([[lineno, column], event, parts.last, lex_state])
           when :on_regexp_end
             # On regex end, Ripper scans and then sets end state, so the ripper
             # lexed output is begin, when it should be end. prism sets lex state

lib/prism/lex_ripper.rb

@@ -26,13 +26,6 @@ def result
             results << token
             previous = token
           end
-        when :on_words_sep
-          if previous[1] == :on_words_sep
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
         else
           results << token
           previous = token

test/prism/ruby/ripper_test.rb

@@ -62,18 +62,10 @@ class RipperTest < TestCase
     ]
 
     omitted_lex = [
-      "comments.txt",
-      "heredoc_percent_q_newline_delimiter.txt",
       "heredoc_with_escaped_newline_at_start.txt",
       "heredocs_with_fake_newlines.txt",
       "indented_file_end.txt",
-      "seattlerb/TestRubyParserShared.txt",
-      "seattlerb/class_comments.txt",
-      "seattlerb/module_comments.txt",
-      "seattlerb/parse_line_block_inline_comment_leading_newlines.txt",
-      "seattlerb/parse_line_block_inline_multiline_comment.txt",
       "spanning_heredoc_newlines.txt",
-      "strings.txt",
       "whitequark/dedenting_heredoc.txt",
       "whitequark/procarg0.txt",
     ]