Remove tokenizer restrictions on header lengths for verbatim sections. Issue #49

Author: drbrain

History.txt

@@ -15,6 +15,8 @@
   * `ri` now has space between class comments from multiple files.
   * :stopdoc: no longer creates Object references.  Issue #55 by Simon Chiang
   * :nodoc: works on class aliases now.  Issue #51 by Steven G. Harms
+  * Remove tokenizer restriction on header lengths for verbatim sections.
+    Issue #49 by trans
 
 === 3.8 / 2011-06-29
 

lib/rdoc/markup/parser.rb

@@ -405,13 +405,19 @@ def tokenize input
                    @line += 1
                    token
                  # === text => :HEADER then :TEXT
-                 when s.scan(/(=+)\s*/) then
+                 when s.scan(/(=+)(\s*)/) then
                    level = s[1].length
-                   level = 6 if level > 6
-                   @tokens << [:HEADER, level, *token_pos(pos)]
-                   pos = s.pos
-                   s.scan(/.*/)
-                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                   header = [:HEADER, level, *token_pos(pos)]
+
+                   if s[2] =~ /^\r?\n/ then
+                     s.pos -= s[2].length
+                     header
+                   else
+                     pos = s.pos
+                     s.scan(/.*/)
+                     @tokens << header
+                     [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                   end
                  # --- (at least 3) and nothing else on the line => :RULE
                  when s.scan(/(-{3,}) *$/) then
                    [:RULE, s[1].length - 2, *token_pos(pos)]

test/test_rdoc_markup_parser.rb

@@ -1366,6 +1366,44 @@ def test_tokenize_verbatim_heading
     assert_equal expected, @RMP.tokenize(str)
   end
 
+  def test_tokenize_verbatim_rule
+    str = <<-STR
+  Verbatim section here that is double-underlined
+  ===============================================
+    STR
+
+    expected = [
+      [:TEXT,    'Verbatim section here that is double-underlined',  2, 0],
+      [:NEWLINE, "\n",                                              49, 0],
+      [:HEADER,  47,                                                 2, 1],
+      [:NEWLINE, "\n",                                              49, 1],
+    ]
+
+    assert_equal expected, @RMP.tokenize(str)
+  end
+
+  def test_tokenize_verbatim_rule_fancy
+    str = <<-STR
+  A
+    b
+  ===============================================
+    c
+    STR
+
+    expected = [
+      [:TEXT,    'A',   2, 0],
+      [:NEWLINE, "\n",  3, 0],
+      [:TEXT,    'b',   4, 1],
+      [:NEWLINE, "\n",  5, 1],
+      [:HEADER,  47,    2, 2],
+      [:NEWLINE, "\n", 49, 2],
+      [:TEXT,    'c',   4, 3],
+      [:NEWLINE, "\n",  5, 3],
+    ]
+
+    assert_equal expected, @RMP.tokenize(str)
+  end
+
   # HACK move to Verbatim test case
   def test_verbatim_normalize
     v = @RM::Verbatim.new "foo\n", "\n", "\n", "bar\n"