Merge pull request #482 from aycabta/fix-exclamation-after-constant

Fix "!" after constant handling, it's identifier

Author: hsbt

lib/rdoc/ruby_lex.rb

@@ -1056,7 +1056,11 @@ def identify_identifier
     end
 
     if token[0, 1] =~ /[A-Z]/
-      return Token(TkCONSTANT, token)
+      if token[-1] =~ /[!?]/
+        return Token(TkIDENTIFIER, token)
+      else
+        return Token(TkCONSTANT, token)
+      end
     elsif token[token.size - 1, 1] =~ /[!?]/
       return Token(TkFID, token)
     else

test/test_rdoc_ruby_lex.rb

@@ -553,5 +553,21 @@ def test_class_tokenize_square_bracket_as_method
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_constant_with_exclamation
+    tokens = RDoc::RubyLex.tokenize "Hello there, Dave!", nil
+
+    expected = [
+      @TK::TkCONSTANT  .new( 0, 1,  0, "Hello"),
+      @TK::TkSPACE     .new( 5, 1,  5, " "),
+      @TK::TkIDENTIFIER.new( 6, 1,  6, "there"),
+      @TK::TkCOMMA     .new(11, 1, 11, ","),
+      @TK::TkSPACE     .new(12, 1, 12, " "),
+      @TK::TkIDENTIFIER.new(13, 1, 13, "Dave!"),
+      @TK::TkNL        .new(18, 1, 18, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
 end