Merge pull request #470 from aycabta/handle-redefinable-operators

Handle redefinable operators as identifier

Author: hsbt

lib/rdoc/ruby_lex.rb

@@ -481,25 +481,44 @@ def lex_init()
       Token(TkNL)
     end
 
+    @OP.def_rules("=") do
+      |op, io|
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+      else
+        @lex_state = :EXPR_BEG
+      end
+      Token(op)
+    end
+
     @OP.def_rules("*", "**",
-                  "=", "==", "===",
+                  "==", "===",
                   "=~", "<=>",
                   "<", "<=",
                   ">", ">=", ">>", "=>") do
       |op, io|
       case @lex_state
       when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
         @lex_state = :EXPR_ARG
       else
+        tk = Token(op)
         @lex_state = :EXPR_BEG
       end
-      Token(op)
+      tk
     end
 
     @OP.def_rules("!", "!=", "!~") do
       |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      else
+        @lex_state = :EXPR_BEG
+        Token(op)
+      end
     end
 
     @OP.def_rules("<<") do
@@ -513,11 +532,12 @@ def lex_init()
         end
       end
       unless tk
-        tk = Token(op)
         case @lex_state
         when :EXPR_FNAME, :EXPR_DOT
+          tk = Token(TkId, op)
           @lex_state = :EXPR_ARG
         else
+          tk = Token(op)
           @lex_state = :EXPR_BEG
         end
       end
@@ -531,9 +551,9 @@ def lex_init()
 
     @OP.def_rules("`") do
       |op, io|
-      if @lex_state == :EXPR_FNAME
-        @lex_state = :EXPR_END
-        Token(op)
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
       else
         identify_string(op)
       end
@@ -557,12 +577,25 @@ def lex_init()
       end
     end
 
-    @OP.def_rules("&", "&&", "|", "||") do
+    @OP.def_rules("&&", "||") do
       |op, io|
       @lex_state = :EXPR_BEG
       Token(op)
     end
 
+    @OP.def_rules("&", "|") do
+      |op, io|
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
+    end
+
     @OP.def_rules("+=", "-=", "*=", "**=",
                   "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
       |op, io|
@@ -574,19 +607,22 @@ def lex_init()
     @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token(op)
+      Token(TkId, op)
     end
 
     @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token(op)
+      Token(TkId, op)
     end
 
     @OP.def_rules("+", "-") do
       |op, io|
       catch(:RET) do
-        if @lex_state == :EXPR_ARG
+        if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+          tk = Token(TkId, op)
+          @lex_state = :EXPR_ARG
+        elsif @lex_state == :EXPR_ARG
           if @space_seen and peek(0) =~ /[0-9]/
             throw :RET, identify_number(op)
           else
@@ -597,7 +633,8 @@ def lex_init()
         else
           @lex_state = :EXPR_BEG
         end
-        Token(op)
+        tk = Token(op) unless tk
+        tk
       end
     end
 
@@ -657,7 +694,10 @@ def lex_int2
 
     @OP.def_rule("/") do
       |op, io|
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
         identify_string(op)
       elsif peek(0) == '='
         getc
@@ -673,8 +713,15 @@ def lex_int2
 
     @OP.def_rules("^") do
       |op, io|
-      @lex_state = :EXPR_BEG
-      Token("^")
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        tk = Token(TkId, op)
+        @lex_state = :EXPR_ARG
+      else
+        tk = Token(op)
+        @lex_state = :EXPR_BEG
+      end
+      tk
     end
 
     #       @OP.def_rules("^=") do
@@ -701,8 +748,14 @@ def lex_int2
 
     @OP.def_rule("~") do
       |op, io|
-      @lex_state = :EXPR_BEG
-      Token("~")
+      case @lex_state
+      when :EXPR_FNAME, :EXPR_DOT
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      else
+        @lex_state = :EXPR_BEG
+        Token(op)
+      end
     end
 
     @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
@@ -728,13 +781,13 @@ def lex_int2
     @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token("[]")
+      Token(TkId, op)
     end
 
     @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
       |op, io|
       @lex_state = :EXPR_ARG
-      Token("[]=")
+      Token(TkId, op)
     end
 
     @OP.def_rule("[") do
@@ -796,7 +849,10 @@ def lex_int2
 
     @OP.def_rule('%') do
       |op, io|
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
+      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
+        @lex_state = :EXPR_ARG
+        Token(TkId, op)
+      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
         identify_quotation
       elsif peek(0) == '='
         getc

test/test_rdoc_markup_to_html.rb

@@ -485,6 +485,36 @@ def test_accept_verbatim_ruby
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_redefinable_operators
+    functions = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` !  != !~].map { |redefinable_op|
+      ["def #{redefinable_op}\n", "end\n"]
+    }.flatten
+
+    verb = @RM::Verbatim.new(*functions)
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-EXPECTED
+
+<pre class="ruby">
+    EXPECTED
+    expected = expected.rstrip
+
+    %w[| ^ &amp; &lt;=&gt; == === =~ &gt; &gt;= &lt; &lt;= &lt;&lt; &gt;&gt; + - * / % ** ~ +@ -@ [] []= ` !  != !~].each do |html_escaped_op|
+      expected += <<-EXPECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">#{html_escaped_op}</span>
+<span class="ruby-keyword">end</span>
+      EXPECTED
+    end
+
+    expected += <<-EXPECTED
+</pre>
+EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_convert_string
     assert_equal '&lt;&gt;', @to.convert_string('<>')
   end

test/test_rdoc_parser_ruby.rb

@@ -1650,6 +1650,25 @@ def test_parse_method
     assert_equal stream, foo.token_stream
   end
 
+  def test_parse_redefinable_methods
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    comment = RDoc::Comment.new "", @top_level
+
+    redefinable_ops = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` !  != !~]
+    redefinable_ops.each do |redefinable_op|
+      util_parser "def #{redefinable_op}\nend\n"
+      tk = @parser.get_tk
+      @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
+    end
+
+    klass.method_list.each do |method|
+      assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
+      assert_includes redefinable_ops, method.token_stream[5].text
+    end
+  end
+
   def test_parse_method_bracket
     util_parser <<-RUBY
 class C