Merge remote-tracking branch 'upstream/master' into support-encoding-keyword

aycabta · aycabta · commit e2d1e178d9fb · 2017-08-23T01:34:26.000+09:00
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
@@ -752,6 +752,9 @@ def parse_class container, single, tk, comment
 
     cls.line   = line_no
 
+    # after end modifiers
+    read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+
     cls
   end
 
@@ -1311,6 +1314,9 @@ def parse_method(container, single, tk, comment)
 
     meth.comment = comment
 
+    # after end modifiers
+    read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
+
     @stats.add_method meth
   end
 
@@ -1542,6 +1548,9 @@ def parse_module container, single, tk, comment
     mod.add_comment comment, @top_level
     parse_statements mod
 
+    # after end modifiers
+    read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+
     @stats.add_module mod
   end
 
@@ -1715,7 +1724,6 @@ def parse_statements(container, single = NORMAL, current_method = nil,
       when TkEND then
         nest -= 1
         if nest == 0 then
-          read_documentation_modifiers container, RDoc::CLASS_MODIFIERS
           container.ongoing_visibility = save_visibility
 
           parse_comment container, tk, comment unless comment.empty?
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
@@ -378,6 +378,10 @@ def token
       else
         tk = tk1
       end
+    elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then
+      tk1 = token
+      set_token_position tk.seek, tk.line_no, tk.char_no
+      tk = Token(tk1.class, tk.text + tk1.text)
     end
     #      Tracer.off
     tk
@@ -450,15 +454,18 @@ def lex_init()
                  proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
       |op, io|
       @ltype = "="
-      res = ''
-      nil until getc == "\n"
+      res = op
+      until (ch = getc) == "\n" do
+        res << ch
+      end
+      res << ch
 
       until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
         (ch = getc)
         res << ch
       end
 
-      gets # consume =end
+      res << gets # consume =end
 
       @ltype = nil
       Token(TkRD_COMMENT, res)
@@ -602,6 +609,7 @@ def lex_init()
           Token(TkQUESTION)
         else
           @lex_state = :EXPR_END
+          ch << getc if "\\" == ch
           Token(TkCHAR, "?#{ch}")
         end
       end
@@ -883,7 +891,8 @@ def lex_int2
         identify_quotation
       elsif peek(0) == '='
         getc
-        Token(TkOPASGN, :%)
+        @lex_state = :EXPR_BEG
+        Token(TkOPASGN, '%')
       elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
         identify_quotation
       else
@@ -985,7 +994,7 @@ def identify_identifier
 
     ungetc
 
-    if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
+    if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
       token.concat getc
     end
 
@@ -1046,12 +1055,7 @@ def identify_identifier
                   @indent_stack.push token_c
                 end
               else
-                if peek(0) == ':' and !peek_match?(/^::/)
-                  token.concat getc
-                  token_c = TkSYMBOL
-                else
-                  token_c = TkIDENTIFIER
-                end
+                token_c = TkIDENTIFIER
               end
 
             elsif DEINDENT_CLAUSE.include?(token)
@@ -1063,13 +1067,17 @@ def identify_identifier
             @lex_state = :EXPR_END
           end
         end
+        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
+          token.concat getc
+          token_c = TkSYMBOL
+        end
         return Token(token_c, token)
       end
     end
 
     if @lex_state == :EXPR_FNAME
       @lex_state = :EXPR_END
-      if peek(0) == '='
+      if peek(0) == '=' and peek(1) != '>'
         token.concat getc
       end
     elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
@@ -1081,19 +1089,20 @@ def identify_identifier
 
     if token[0, 1] =~ /[A-Z]/
       if token[-1] =~ /[!?]/
-        return Token(TkIDENTIFIER, token)
+        token_c = TkIDENTIFIER
       else
-        return Token(TkCONSTANT, token)
+        token_c = TkCONSTANT
       end
     elsif token[token.size - 1, 1] =~ /[!?]/
-      return Token(TkFID, token)
+      token_c = TkFID
     else
-      if peek(0) == ':' and !peek_match?(/^::/)
-        token.concat getc
-        return Token(TkSYMBOL, token)
-      else
-        return Token(TkIDENTIFIER, token)
-      end
+      token_c = TkIDENTIFIER
+    end
+    if peek(0) == ':' and !peek_match?(/^::/)
+      token.concat getc
+      return Token(TkSYMBOL, token)
+    else
+      return Token(token_c, token)
     end
   end
 
@@ -1132,7 +1141,7 @@ def identify_here_document(op)
       indent: indent,
       started: false
     }
-    @lex_state = :EXPR_BEG
+    @lex_state = :EXPR_END
     Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
   end
 
@@ -1331,13 +1340,13 @@ def identify_string(ltype, quoted = ltype, type = nil)
             ungetc
           end
         elsif ch == '\\'
-          if %w[' /].include? @ltype then
+          case @ltype
+          when "'" then
             case ch = getc
-            when "\n", "'"
-            when @ltype
+            when "'", '\\' then
               str << ch
             else
-              ungetc
+              str << ch
             end
           else
             str << read_escape
diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb
@@ -44,10 +44,18 @@ def self.to_html token_stream
               when RDoc::RubyToken::TkVal      then 'ruby-value'
               end
 
-      text = CGI.escapeHTML t.text
+      comment_with_nl = false
+      case t
+      when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
+        comment_with_nl = true if t.text =~ /\n$/
+        text = t.text.rstrip
+      else
+        text = t.text
+      end
+      text = CGI.escapeHTML text
 
       if style then
-        "<span class=\"#{style}\">#{text}</span>"
+        "<span class=\"#{style}\">#{text}</span>#{"\n" if comment_with_nl}"
       else
         text
       end
diff --git a/test/test_rdoc_markup_to_html.rb b/test/test_rdoc_markup_to_html.rb
@@ -485,6 +485,66 @@ def test_accept_verbatim_pipe
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_escape_in_string
+    code = <<-'RUBY'
+def foo
+  [
+    '\\',
+    '\'',
+    "'",
+    "\'\"\`",
+    "\#",
+    "\#{}",
+    "#",
+    "#{}",
+    /'"/,
+    /\'\"/,
+    /\//,
+    /\\/,
+    /\#/,
+    /\#{}/,
+    /#/,
+    /#{}/
+  ]
+end
+def bar
+end
+    RUBY
+    verb = @RM::Verbatim.new(*code.split(/(?<=\n)/))
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-'EXPECTED'
+
+<pre class="ruby"><span class="ruby-keyword">def</span> <span class="ruby-identifier">foo</span>
+  [
+    <span class="ruby-string">&#39;\\&#39;</span>,
+    <span class="ruby-string">&#39;\&#39;&#39;</span>,
+    <span class="ruby-string">&quot;&#39;&quot;</span>,
+    <span class="ruby-string">&quot;\&#39;\&quot;\`&quot;</span>,
+    <span class="ruby-string">&quot;\#&quot;</span>,
+    <span class="ruby-string">&quot;\#{}&quot;</span>,
+    <span class="ruby-string">&quot;#&quot;</span>,
+    <span class="ruby-node">&quot;#{}&quot;</span>,
+    <span class="ruby-regexp">/&#39;&quot;/</span>,
+    <span class="ruby-regexp">/\&#39;\&quot;/</span>,
+    <span class="ruby-regexp">/\//</span>,
+    <span class="ruby-regexp">/\\/</span>,
+    <span class="ruby-regexp">/\#/</span>,
+    <span class="ruby-regexp">/\#{}/</span>,
+    <span class="ruby-regexp">/#/</span>,
+    <span class="ruby-regexp">/#{}/</span>
+  ]
+<span class="ruby-keyword">end</span>
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+<span class="ruby-keyword">end</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_accept_verbatim_ruby
     verb = @RM::Verbatim.new("1 + 1\n")
     verb.format = :ruby
diff --git a/test/test_rdoc_parser_ruby.rb b/test/test_rdoc_parser_ruby.rb
diff --git a/test/test_rdoc_ruby_lex.rb b/test/test_rdoc_ruby_lex.rb
