Merge pull request #462 from aycabta/move-collecting-up-symbol-to-ruby-lex

Move collecting up tokens for a symbol to RubyLex

Author: Zachary Scott

lib/rdoc/parser/ruby_tools.rb

@@ -33,26 +33,6 @@ def get_tk
 
     tk = nil if TkEND_OF_SCRIPT === tk
 
-    if TkSYMBEG === tk then
-      set_token_position tk.line_no, tk.char_no
-
-      case tk1 = get_tk
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL).set_text(":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL).set_text(":" + tk1.text)
-        end
-
-        # remove the identifier we just read to replace it with a symbol
-        @token_listeners.each do |obj|
-          obj.pop_token
-        end if @token_listeners
-      else
-        tk = tk1
-      end
-    end
-
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
       obj.add_token(tk)

lib/rdoc/ruby_lex.rb

@@ -361,6 +361,22 @@ def token
     if @readed_auto_clean_up
       get_readed
     end
+
+    if TkSYMBEG === tk then
+      tk1 = token
+      set_token_position tk.line_no, tk.char_no
+
+      case tk1
+      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
+        if tk1.respond_to?(:name) then
+          tk = Token(TkSYMBOL, ":" + tk1.name)
+        else
+          tk = Token(TkSYMBOL, ":" + tk1.text)
+        end
+      else
+        tk = tk1
+      end
+    end
     #      Tracer.off
     tk
   end

test/test_rdoc_ruby_lex.rb

@@ -81,8 +81,7 @@ def test_class_tokenize_hash_symbol
       @TK::TkLBRACE    .new( 0, 1,  0, '{'),
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
       @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
-      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
-      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
+      @TK::TkSYMBOL    .new( 7, 1,  7, ':"foo"'),
       @TK::TkSPACE     .new(13, 1, 13, ' '),
       @TK::TkRBRACE    .new(14, 1, 14, '}'),
       @TK::TkNL        .new(15, 1, 15, "\n"),
@@ -393,8 +392,7 @@ def test_class_tokenize_symbol
       @TK::TkIDENTIFIER.new( 6, 1,  6, 'module'),
       @TK::TkCOLON     .new(12, 1, 12, ':'),
       @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkSYMBEG    .new(14, 1, 14, ':'),
-      @TK::TkIDENTIFIER.new(15, 1, 15, 'v1'),
+      @TK::TkSYMBOL    .new(14, 1, 14, ':v1'),
       @TK::TkNL        .new(17, 1, 17, "\n"),
     ]