Merge pull request #467 from aycabta/handle-json-style-hask-key-as-symbol

Handle JSON-style hash key as symbol

Author: hsbt

lib/rdoc/ruby_lex.rb

@@ -963,7 +963,12 @@ def identify_identifier
                   @indent_stack.push token_c
                 end
               else
-                token_c = TkIDENTIFIER
+                if peek(0) == ':' and !peek_match?(/^::/)
+                  token.concat getc
+                  token_c = TkSYMBOL
+                else
+                  token_c = TkIDENTIFIER
+                end
               end
 
             elsif DEINDENT_CLAUSE.include?(token)
@@ -996,7 +1001,12 @@ def identify_identifier
     elsif token[token.size - 1, 1] =~ /[!?]/
       return Token(TkFID, token)
     else
-      return Token(TkIDENTIFIER, token)
+      if peek(0) == ':' and !peek_match?(/^::/)
+        token.concat getc
+        return Token(TkSYMBOL, token)
+      else
+        return Token(TkIDENTIFIER, token)
+      end
     end
   end
 

test/test_rdoc_ruby_lex.rb

@@ -103,12 +103,27 @@ def test_class_tokenize_hash_symbol
     tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
 
     expected = [
-      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
-      @TK::TkSYMBOL    .new( 7, 1,  7, ':"foo"'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkRBRACE    .new(14, 1, 14, '}'),
+      @TK::TkLBRACE.new( 0, 1,  0, '{'),
+      @TK::TkSPACE .new( 1, 1,  1, ' '),
+      @TK::TkSYMBOL.new( 2, 1,  2, 'class:'),
+      @TK::TkSTRING.new( 8, 1,  8, '"foo"'),
+      @TK::TkSPACE .new(13, 1, 13, ' '),
+      @TK::TkRBRACE.new(14, 1, 14, '}'),
+      @TK::TkNL    .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_double_colon_is_not_hash_symbol
+    tokens = RDoc::RubyLex.tokenize 'self.class::Row', nil
+
+    expected = [
+      @TK::TkSELF      .new( 0, 1,  0, "self"),
+      @TK::TkDOT       .new( 4, 1,  4, "."),
+      @TK::TkIDENTIFIER.new( 5, 1,  5, "class"),
+      @TK::TkCOLON2    .new(10, 1, 10, "::"),
+      @TK::TkCONSTANT  .new(12, 1, 12, "Row"),
       @TK::TkNL        .new(15, 1, 15, "\n"),
     ]
 
@@ -444,8 +459,7 @@ def test_class_tokenize_symbol
     expected = [
       @TK::TkIDENTIFIER.new( 0, 1,  0, 'scope'),
       @TK::TkSPACE     .new( 5, 1,  5, ' '),
-      @TK::TkIDENTIFIER.new( 6, 1,  6, 'module'),
-      @TK::TkCOLON     .new(12, 1, 12, ':'),
+      @TK::TkSYMBOL    .new( 6, 1,  6, 'module:'),
       @TK::TkSPACE     .new(13, 1, 13, ' '),
       @TK::TkSYMBOL    .new(14, 1, 14, ':v1'),
       @TK::TkNL        .new(17, 1, 17, "\n"),