Merge pull request #489 from aycabta/fix-not-equal-handling

Separate "foo!=bar" to "foo", "!=" and "bar"

Author: hsbt

lib/rdoc/ruby_lex.rb

@@ -985,7 +985,7 @@ def identify_identifier
 
     ungetc
 
-    if (ch == "!" || ch == "?") && token[0,1] =~ /\w/ && peek(0) != "="
+    if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
       token.concat getc
     end
 

test/test_rdoc_ruby_lex.rb

@@ -773,5 +773,22 @@ def test_class_tokenize_constant_with_exclamation
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_identifer_not_equal
+    tokens = RDoc::RubyLex.tokenize "foo!=bar\nfoo?=bar", nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1, 0, "foo"),
+      @TK::TkNEQ       .new( 3, 1, 3, "!="),
+      @TK::TkIDENTIFIER.new( 5, 1, 5, "bar"),
+      @TK::TkNL        .new( 8, 1, 8, "\n"),
+      @TK::TkFID       .new( 9, 2, 0, "foo?"),
+      @TK::TkASSIGN    .new(13, 2, 4, "="),
+      @TK::TkIDENTIFIER.new(14, 2, 5, "bar"),
+      @TK::TkNL        .new(17, 2, 9, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
 end