Tokenize out-of-place reserved words as identifiers

Author: drbrain

History.rdoc

@@ -7,6 +7,7 @@
     Noria.
   * Fixed tokenization of % when it is not followed by a $-string type
   * Fixed display of __END__ in documentation examples in HTML output
+  * Fixed tokenization of reserved words used as new-style hash keys
 
 
 === 3.12.1 / 2013-02-05

lib/rdoc/ruby_lex.rb

@@ -936,6 +936,8 @@ def identify_identifier
                   @indent += 1
                   @indent_stack.push token_c
                 end
+              else
+                token_c = TkIDENTIFIER
               end
 
             elsif DEINDENT_CLAUSE.include?(token)

test/test_rdoc_ruby_lex.rb

@@ -77,6 +77,23 @@ def x
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_hash_symbol
+    tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
+
+    expected = [
+      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
+      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
+      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
+      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkRBRACE    .new(14, 1, 14, '}'),
+      @TK::TkNL        .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_heredoc_indent
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
 string = <<-STRING