@@ -101,6 +101,31 @@ def x
101101 assert_equal expected , tokens
102102 end
103103
104+ def test_class_tokenize_reserved_keyword_with_args
105+ tokens = RDoc ::RubyLex . tokenize <<-'RUBY' , nil
106+ yield :foo
107+ super :bar
108+ defined? :baz
109+ RUBY
110+
111+ expected = [
112+ @TK ::TkYIELD . new ( 0 , 1 , 0 , "yield" ) ,
113+ @TK ::TkSPACE . new ( 5 , 1 , 5 , " " ) ,
114+ @TK ::TkSYMBOL . new ( 6 , 1 , 6 , ":foo" ) ,
115+ @TK ::TkNL . new ( 10 , 1 , 10 , "\n " ) ,
116+ @TK ::TkSUPER . new ( 11 , 2 , 0 , "super" ) ,
117+ @TK ::TkSPACE . new ( 16 , 2 , 5 , " " ) ,
118+ @TK ::TkSYMBOL . new ( 17 , 2 , 6 , ":bar" ) ,
119+ @TK ::TkNL . new ( 21 , 2 , 11 , "\n " ) ,
120+ @TK ::TkDEFINED . new ( 22 , 3 , 0 , "defined?" ) ,
121+ @TK ::TkSPACE . new ( 30 , 3 , 8 , " " ) ,
122+ @TK ::TkSYMBOL . new ( 31 , 3 , 9 , ":baz" ) ,
123+ @TK ::TkNL . new ( 35 , 3 , 22 , "\n " )
124+ ]
125+
126+ assert_equal expected , tokens
127+ end
128+
104129 def test_class_tokenize_hash_symbol
105130 tokens = RDoc ::RubyLex . tokenize '{ class:"foo" }' , nil
106131
@@ -132,6 +157,19 @@ def test_class_tokenize_double_colon_is_not_hash_symbol
132157 assert_equal expected , tokens
133158 end
134159
160+ def test_class_tokenize_safe_nav_operator
161+ tokens = RDoc ::RubyLex . tokenize 'receiver&.meth' , nil
162+
163+ expected = [
164+ @TK ::TkIDENTIFIER . new ( 0 , 1 , 0 , "receiver" ) ,
165+ @TK ::TkSAFENAV . new ( 8 , 1 , 8 , "&." ) ,
166+ @TK ::TkIDENTIFIER . new ( 10 , 1 , 10 , "meth" ) ,
167+ @TK ::TkNL . new ( 14 , 1 , 14 , "\n " ) ,
168+ ]
169+
170+ assert_equal expected , tokens
171+ end
172+
135173 def test_class_tokenize_hash_rocket
136174 tokens = RDoc ::RubyLex . tokenize '{ :class => "foo" }' , nil
137175
@@ -439,6 +477,101 @@ def test_class_tokenize_string
439477 assert_equal expected , tokens
440478 end
441479
480+ def test_class_tokenize_string_with_escape
481+ tokens = RDoc ::RubyLex . tokenize <<'RUBY' , nil
482+ [
483+ '\\',
484+ '\'',
485+ "'",
486+ "\'\"\`",
487+ "\#",
488+ "\#{}",
489+ "#",
490+ "#{}",
491+ /'"/,
492+ /\'\"/,
493+ /\//,
494+ /\\/,
495+ /\#/,
496+ /\#{}/,
497+ /#/,
498+ /#{}/
499+ ]
500+ RUBY
501+
502+ expected = [
503+ @TK ::TkLBRACK . new ( 0 , 1 , 0 , "[" ) ,
504+ @TK ::TkNL . new ( 1 , 1 , 1 , "\n " ) ,
505+ @TK ::TkSPACE . new ( 2 , 2 , 0 , " " ) ,
506+ @TK ::TkSTRING . new ( 4 , 2 , 2 , "'\\ \\ '" ) ,
507+ @TK ::TkCOMMA . new ( 8 , 2 , 6 , "," ) ,
508+ @TK ::TkNL . new ( 9 , 2 , 2 , "\n " ) ,
509+ @TK ::TkSPACE . new ( 10 , 3 , 0 , " " ) ,
510+ @TK ::TkSTRING . new ( 12 , 3 , 2 , "'\\ ''" ) ,
511+ @TK ::TkCOMMA . new ( 16 , 3 , 6 , "," ) ,
512+ @TK ::TkNL . new ( 17 , 3 , 10 , "\n " ) ,
513+ @TK ::TkSPACE . new ( 18 , 4 , 0 , " " ) ,
514+ @TK ::TkSTRING . new ( 20 , 4 , 2 , "\" '\" " ) ,
515+ @TK ::TkCOMMA . new ( 23 , 4 , 5 , "," ) ,
516+ @TK ::TkNL . new ( 24 , 4 , 18 , "\n " ) ,
517+ @TK ::TkSPACE . new ( 25 , 5 , 0 , " " ) ,
518+ @TK ::TkSTRING . new ( 27 , 5 , 2 , "\" \\ '\\ \" \\ `\" " ) ,
519+ @TK ::TkCOMMA . new ( 35 , 5 , 10 , "," ) ,
520+ @TK ::TkNL . new ( 36 , 5 , 25 , "\n " ) ,
521+ @TK ::TkSPACE . new ( 37 , 6 , 0 , " " ) ,
522+ @TK ::TkSTRING . new ( 39 , 6 , 2 , "\" \\ #\" " ) ,
523+ @TK ::TkCOMMA . new ( 43 , 6 , 6 , "," ) ,
524+ @TK ::TkNL . new ( 44 , 6 , 37 , "\n " ) ,
525+ @TK ::TkSPACE . new ( 45 , 7 , 0 , " " ) ,
526+ @TK ::TkSTRING . new ( 47 , 7 , 2 , "\" \\ \# {}\" " ) ,
527+ @TK ::TkCOMMA . new ( 53 , 7 , 8 , "," ) ,
528+ @TK ::TkNL . new ( 54 , 7 , 45 , "\n " ) ,
529+ @TK ::TkSPACE . new ( 55 , 8 , 0 , " " ) ,
530+ @TK ::TkSTRING . new ( 57 , 8 , 2 , "\" #\" " ) ,
531+ @TK ::TkCOMMA . new ( 60 , 8 , 5 , "," ) ,
532+ @TK ::TkNL . new ( 61 , 8 , 55 , "\n " ) ,
533+ @TK ::TkSPACE . new ( 62 , 9 , 0 , " " ) ,
534+ @TK ::TkDSTRING . new ( 64 , 9 , 2 , "\" \# {}\" " ) ,
535+ @TK ::TkCOMMA . new ( 69 , 9 , 7 , "," ) ,
536+ @TK ::TkNL . new ( 70 , 9 , 62 , "\n " ) ,
537+ @TK ::TkSPACE . new ( 71 , 10 , 0 , " " ) ,
538+ @TK ::TkREGEXP . new ( 73 , 10 , 2 , "/'\" /" ) ,
539+ @TK ::TkCOMMA . new ( 77 , 10 , 6 , "," ) ,
540+ @TK ::TkNL . new ( 78 , 10 , 71 , "\n " ) ,
541+ @TK ::TkSPACE . new ( 79 , 11 , 0 , " " ) ,
542+ @TK ::TkREGEXP . new ( 81 , 11 , 2 , "/\\ '\\ \" /" ) ,
543+ @TK ::TkCOMMA . new ( 87 , 11 , 8 , "," ) ,
544+ @TK ::TkNL . new ( 88 , 11 , 79 , "\n " ) ,
545+ @TK ::TkSPACE . new ( 89 , 12 , 0 , " " ) ,
546+ @TK ::TkREGEXP . new ( 91 , 12 , 2 , "/\\ //" ) ,
547+ @TK ::TkCOMMA . new ( 95 , 12 , 6 , "," ) ,
548+ @TK ::TkNL . new ( 96 , 12 , 89 , "\n " ) ,
549+ @TK ::TkSPACE . new ( 97 , 13 , 0 , " " ) ,
550+ @TK ::TkREGEXP . new ( 99 , 13 , 2 , "/\\ \\ /" ) ,
551+ @TK ::TkCOMMA . new ( 103 , 13 , 6 , "," ) ,
552+ @TK ::TkNL . new ( 104 , 13 , 97 , "\n " ) ,
553+ @TK ::TkSPACE . new ( 105 , 14 , 0 , " " ) ,
554+ @TK ::TkREGEXP . new ( 107 , 14 , 2 , "/\\ #/" ) ,
555+ @TK ::TkCOMMA . new ( 111 , 14 , 6 , "," ) ,
556+ @TK ::TkNL . new ( 112 , 14 , 105 , "\n " ) ,
557+ @TK ::TkSPACE . new ( 113 , 15 , 0 , " " ) ,
558+ @TK ::TkREGEXP . new ( 115 , 15 , 2 , "/\\ \# {}/" ) ,
559+ @TK ::TkCOMMA . new ( 121 , 15 , 8 , "," ) ,
560+ @TK ::TkNL . new ( 122 , 15 , 113 , "\n " ) ,
561+ @TK ::TkSPACE . new ( 123 , 16 , 0 , " " ) ,
562+ @TK ::TkREGEXP . new ( 125 , 16 , 2 , "/#/" ) ,
563+ @TK ::TkCOMMA . new ( 128 , 16 , 5 , "," ) ,
564+ @TK ::TkNL . new ( 129 , 16 , 123 , "\n " ) ,
565+ @TK ::TkSPACE . new ( 130 , 17 , 0 , " " ) ,
566+ @TK ::TkDREGEXP . new ( 132 , 17 , 2 , "/\# {}/" ) ,
567+ @TK ::TkNL . new ( 137 , 17 , 7 , "\n " ) ,
568+ @TK ::TkRBRACK . new ( 138 , 18 , 0 , "]" ) ,
569+ @TK ::TkNL . new ( 139 , 18 , 138 , "\n " )
570+ ]
571+
572+ assert_equal expected , tokens
573+ end
574+
442575 def test_class_tokenize_string_escape
443576 tokens = RDoc ::RubyLex . tokenize '"\\n"' , nil
444577 assert_equal @TK ::TkSTRING . new ( 0 , 1 , 0 , "\" \\ n\" " ) , tokens . first
0 commit comments