Fix heredoc implementation

The here-documents in middle of line are replaced with string literal.
It's useful for IRB, but it doesn't fit for documentation.

For example,

def example
  puts [<<~TILDE, <<-HYPHEN, <<~'TILDE', <<-'HYPHEN', <<'AAA', <<"BBB",
<<-`CCC`]
    a
      b
      c
    d
  TILDE
    1
      2
      3
    4
  HYPHEN
    w
      x
      y
    z
  TILDE
    1
      2
      3
    4
  HYPHEN
aaa
aa
a
AAA
bbb
bb
b
BBB
echo 'a'
echo 'b'
echo 'c'
CCC
end

MRI processes this code and outputs correctly.
This is changed in documentation below:

def example
  puts [<<~TILDE, "    a
      b
      c
    d
  TILDE
    1
      2
      3
    4
", <<~'TILDE', "    w
      x
      y
    z
  TILDE
    1
      2
      3
    4
", "aaa
aa
a
", "bbb
bb
b
", "echo 'a'
echo 'b'
echo 'c'
"]
end

I think this is broken document.
This commit fixes it.

Author: aycabta

lib/rdoc/ruby_lex.rb

@@ -106,6 +106,8 @@ def initialize(content, options)
     @rests = []
     @seek = 0
 
+    @heredoc_queue = []
+
     @indent = 0
     @indent_stack = []
     @lex_state = :EXPR_BEG
@@ -462,21 +464,43 @@ def lex_init()
 
     @OP.def_rule("\n") do |op, io|
       print "\\n\n" if RDoc::RubyLex.debug?
-      case @lex_state
-      when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
-        @continue = true
-      else
-        @continue = false
-        @lex_state = :EXPR_BEG
-        until (@indent_stack.empty? ||
-               [TkLPAREN, TkLBRACK, TkLBRACE,
-                 TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-          @indent_stack.pop
+      unless @heredoc_queue.empty?
+        info = @heredoc_queue[0]
+        if !info[:started] # "\n"
+          info[:started] = true
+          ungetc "\n"
+        elsif info[:heredoc_end].nil? # heredoc body
+          tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
+          info[:heredoc_end] = heredoc_end
+          ungetc "\n"
+        else # heredoc end
+          @heredoc_queue.shift
+          @lex_state = :EXPR_BEG
+          tk = Token(TkHEREDOCEND, info[:heredoc_end])
+          if !@heredoc_queue.empty?
+            @heredoc_queue[0][:started] = true
+            ungetc "\n"
+          end
         end
       end
-      @current_readed = @readed
-      @here_readed.clear
-      Token(TkNL)
+      unless tk
+        case @lex_state
+        when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
+          @continue = true
+        else
+          @continue = false
+          @lex_state = :EXPR_BEG
+          until (@indent_stack.empty? ||
+                 [TkLPAREN, TkLBRACK, TkLBRACE,
+                   TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
+            @indent_stack.pop
+          end
+        end
+        @current_readed = @readed
+        @here_readed.clear
+        tk = Token(TkNL)
+      end
+      tk
     end
 
     @OP.def_rules("*", "**",
@@ -506,8 +530,8 @@ def lex_init()
       if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
          (@lex_state != :EXPR_ARG || @space_seen)
         c = peek(0)
-        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-")
-          tk = identify_here_document
+        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
+          tk = identify_here_document(op)
         end
       end
       unless tk
@@ -985,77 +1009,63 @@ def identify_identifier
     end
   end
 
-  def identify_here_document
+  def identify_here_document(op)
     ch = getc
+    start_token = op
     #    if lt = PERCENT_LTYPE[ch]
-    if ch == "-"
+    if ch == "-" or ch == "~"
+      start_token.concat ch
       ch = getc
       indent = true
     end
     if /['"`]/ =~ ch
+      start_token.concat ch
       user_quote = lt = ch
       quoted = ""
       while (c = getc) && c != lt
         quoted.concat c
       end
+      start_token.concat quoted
+      start_token.concat lt
     else
       user_quote = nil
       lt = '"'
       quoted = ch.dup
       while (c = getc) && c =~ /\w/
         quoted.concat c
       end
+      start_token.concat quoted
       ungetc
     end
 
-    ltback, @ltype = @ltype, lt
-    reserve = []
-    while ch = getc
-      reserve.push ch
-      if ch == "\\"
-        reserve.push ch = getc
-      elsif ch == "\n"
-        break
-      end
-    end
-
-    output_heredoc = reserve.join =~ /\A\r?\n\z/
+    @heredoc_queue << {
+      quoted: quoted,
+      lt: lt,
+      indent: indent,
+      started: false
+    }
+    @lex_state = :EXPR_BEG
+    Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
+  end
 
-    if output_heredoc then
-      doc = '<<'
-      doc << '-' if indent
-      doc << "#{user_quote}#{quoted}#{user_quote}\n"
-    else
-      doc = '"'
-    end
+  def identify_here_document_body(quoted, lt, indent)
+    ltback, @ltype = @ltype, lt
 
-    @current_readed = @readed
+    doc = ""
+    heredoc_end = nil
     while l = gets
       l = l.sub(/(:?\r)?\n\z/, "\n")
       if (indent ? l.strip : l.chomp) == quoted
+        heredoc_end = l
         break
       end
       doc << l
     end
+    raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
 
-    if output_heredoc then
-      raise Error, "Missing terminating #{quoted} for string" unless l
-
-      doc << l.chomp
-    else
-      doc << '"'
-    end
-
-    @current_readed = @here_readed
-    @here_readed.concat reserve
-    while ch = reserve.pop
-      ungetc ch
-    end
-
-    token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
     @ltype = ltback
-    @lex_state = :EXPR_END
-    Token(token_class, doc)
+    @lex_state = :EXPR_BEG
+    [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
   end
 
   def identify_quotation

lib/rdoc/ruby_token.rb

@@ -329,6 +329,8 @@ def Token(token, value = nil)
     [:TkCVAR,       TkId],
     [:TkIVAR,       TkId],
     [:TkCONSTANT,   TkId],
+    [:TkHEREDOCBEG, TkId],
+    [:TkHEREDOCEND, TkId],
 
     [:TkINTEGER,    TkVal],
     [:TkFLOAT,      TkVal],