Merge pull request #493 from aycabta/fix-symbol-handling-in-args-of-yield

Fix symbol handle in args of yield/super/defined?

Author: hsbt

lib/rdoc/ruby_token.rb

@@ -303,8 +303,8 @@ def Token(token, value = nil)
     [:TkIN,         TkKW,  "in",     :EXPR_BEG],
     [:TkDO,         TkKW,  "do",     :EXPR_BEG],
     [:TkRETURN,     TkKW,  "return", :EXPR_MID],
-    [:TkYIELD,      TkKW,  "yield",  :EXPR_END],
-    [:TkSUPER,      TkKW,  "super",  :EXPR_END],
+    [:TkYIELD,      TkKW,  "yield",  :EXPR_ARG],
+    [:TkSUPER,      TkKW,  "super",  :EXPR_ARG],
     [:TkSELF,       TkKW,  "self",   :EXPR_END],
     [:TkNIL,        TkKW,  "nil",    :EXPR_END],
     [:TkTRUE,       TkKW,  "true",   :EXPR_END],
@@ -317,7 +317,7 @@ def Token(token, value = nil)
     [:TkWHILE_MOD,  TkKW],
     [:TkUNTIL_MOD,  TkKW],
     [:TkALIAS,      TkKW,  "alias",    :EXPR_FNAME],
-    [:TkDEFINED,    TkKW,  "defined?", :EXPR_END],
+    [:TkDEFINED,    TkKW,  "defined?", :EXPR_ARG],
     [:TklBEGIN,     TkKW,  "BEGIN",    :EXPR_END],
     [:TklEND,       TkKW,  "END",      :EXPR_END],
     [:Tk__LINE__,   TkKW,  "__LINE__", :EXPR_END],

test/test_rdoc_ruby_lex.rb

@@ -101,6 +101,31 @@ def x
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_reserved_keyword_with_args
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+yield :foo
+super :bar
+defined? :baz
+    RUBY
+
+    expected = [
+      @TK::TkYIELD  .new( 0, 1,  0, "yield"),
+      @TK::TkSPACE  .new( 5, 1,  5, " "),
+      @TK::TkSYMBOL .new( 6, 1,  6, ":foo"),
+      @TK::TkNL     .new(10, 1, 10,  "\n"),
+      @TK::TkSUPER  .new(11, 2,  0, "super"),
+      @TK::TkSPACE  .new(16, 2,  5, " "),
+      @TK::TkSYMBOL .new(17, 2,  6, ":bar"),
+      @TK::TkNL     .new(21, 2, 11,  "\n"),
+      @TK::TkDEFINED.new(22, 3,  0, "defined?"),
+      @TK::TkSPACE  .new(30, 3,  8, " "),
+      @TK::TkSYMBOL .new(31, 3,  9, ":baz"),
+      @TK::TkNL     .new(35, 3, 22,  "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_hash_symbol
     tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil