Fix tests for heredoc tokens

Author: aycabta

test/test_rdoc_ruby_lex.rb

@@ -89,9 +89,11 @@ def x
       @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
       @TK::TkNL        .new( 5, 1,  5, "\n"),
       @TK::TkSPACE     .new( 6, 2,  0, '  '),
-      @TK::TkHEREDOC   .new( 8, 2,  2,
-                            %Q{<<E\nLine 1\nLine 2\nE}),
-      @TK::TkNL        .new(27, 5, 28, "\n"),
+
+      @TK::TkHEREDOCBEG.new( 8, 2,  2, '<<E'),
+      @TK::TkNL        .new(11, 2,  6, "\n"),
+      @TK::TkHEREDOC   .new(11, 2,  6, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(27, 5, 26, "E\n"),
       @TK::TkEND       .new(28, 6,  0, 'end'),
       @TK::TkNL        .new(31, 6, 28, "\n"),
     ]
@@ -147,10 +149,12 @@ def test_class_tokenize_heredoc_CR_NL
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOC   .new( 9, 1,  9,
-                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
-      @TK::TkSPACE     .new(44, 4, 45, "\r"),
-      @TK::TkNL        .new(45, 4, 46, "\n"),
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkSPACE     .new(18, 1, 18, "\r"),
+      @TK::TkNL        .new(19, 1, 19, "\n"),
+      @TK::TkHEREDOC   .new(19, 1, 19,
+                            %Q{Line 1\nLine 2\n}),
+      @TK::TkHEREDOCEND.new(45, 4, 36, "  STRING\n"),
     ]
 
     assert_equal expected, tokens
@@ -169,10 +173,12 @@ def test_class_tokenize_heredoc_call
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
-      @TK::TkDOT       .new(41, 4, 42, '.'),
-      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
-      @TK::TkNL        .new(47, 4, 48, "\n"),
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkDOT       .new(18, 1, 18, '.'),
+      @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'),
+      @TK::TkNL        .new(24, 1, 24, "\n"),
+      @TK::TkHEREDOC   .new(24, 1, 24, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(47, 4, 39, "  STRING\n"),
     ]
 
     assert_equal expected, tokens
@@ -191,9 +197,12 @@ def test_class_tokenize_heredoc_indent
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOC   .new( 9, 1,  9,
-                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
-      @TK::TkNL        .new(41, 4, 42, "\n"),
+
+
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkNL        .new(18, 1, 18, "\n"),
+      @TK::TkHEREDOC   .new(18, 1, 18, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(41, 4, 33, "  STRING\n")
     ]
 
     assert_equal expected, tokens
@@ -223,8 +232,10 @@ def test_class_tokenize_heredoc_percent_N
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
       @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
       @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
-      @TK::TkNL        .new(13, 3, 14, "\n"),
+      @TK::TkHEREDOCBEG.new( 4, 1,  4, '<<-U'),
+      @TK::TkNL        .new( 8, 1,  8, "\n"),
+      @TK::TkHEREDOC   .new( 8, 1,  8, "%N\n"),
+      @TK::TkHEREDOCEND.new(13, 3, 12, "U\n")
     ]
 
     assert_equal expected, tokens