Add experimental feature to insert a blank line if a line of a paragraph ends in two spaces

Author: drbrain

History.rdoc

@@ -73,6 +73,8 @@
   * "Top" link in section headers is no longer inside the heading element.
   * RDoc avoids printing some warnings unless run with `rdoc --verbose`.  For
     Rails issue #1646.
+  * Finishing a paragraph with two or more spaces will result in a line break.
+    This feature is experimental and may be modified or removed.
 
 * Bug fixes
   * Markup defined by RDoc::Markup#add_special inside a <tt><tt></tt> is no

lib/rdoc/markup/parser.rb

@@ -203,6 +203,11 @@ def build_paragraph margin
 
       if type == :TEXT && column == margin then
         paragraph << data
+
+        if peek_token[0] == :BREAK then
+          break
+        end
+
         skip :NEWLINE
       else
         unget
@@ -268,7 +273,7 @@ def build_verbatim margin
         peek_column ||= column + width
         indent = peek_column - column - width
         line << ' ' * indent
-      when :TEXT then
+      when :BREAK, :TEXT then
         line << data
       else # *LIST_TOKENS
         list_marker = case type
@@ -322,7 +327,12 @@ def parse parent, indent = 0
     until @tokens.empty? do
       type, data, column, = get
 
-      if type == :NEWLINE then
+      case type
+      when :BREAK then
+        parent << RDoc::Markup::BlankLine.new
+        skip :NEWLINE, false
+        next
+      when :NEWLINE then
         # trailing newlines are skipped below, so this is a blank line
         parent << RDoc::Markup::BlankLine.new
         skip :NEWLINE, false
@@ -458,8 +468,15 @@ def tokenize input
                  when s.scan(/(.*?)::( +|\r?$)/) then
                    [:NOTE, s[1], *token_pos(pos)]
                  # anything else: :TEXT
-                 else s.scan(/.*/)
-                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                 else s.scan(/(.*?)(  )?\r?$/)
+                   token = [:TEXT, s[1], *token_pos(pos)]
+
+                   if s[2] then
+                     @tokens << token
+                     [:BREAK, s[2], *token_pos(pos + s[1].length)]
+                   else
+                     token
+                   end
                  end
     end
 

test/test_rdoc_markup_parser.rb

@@ -441,6 +441,17 @@ def test_parse_lalpha_utf_8
     assert_equal expected, @RMP.parse(str).parts
   end
 
+  def test_parse_line_break
+    str = "now is\nthe time  \nfor all"
+
+    expected = [
+      @RM::Paragraph.new('now is', 'the time'),
+      @RM::BlankLine.new,
+      @RM::Paragraph.new('for all')]
+
+    assert_equal expected, @RMP.parse(str).parts
+  end
+
   def test_parse_list_list_1
     str = <<-STR
 10. para 1
@@ -1233,6 +1244,53 @@ def test_tokenize_lalpha
     assert_equal expected, @RMP.tokenize(str)
   end
 
+  def test_tokenize_line_break
+    str = "now is\nthe time  \nfor all\n"
+
+    expected = [
+      [:TEXT,    'now is',    0, 0],
+      [:NEWLINE, "\n",        6, 0],
+      [:TEXT,    'the time',  0, 1],
+      [:BREAK,   "  ",        8, 1],
+      [:NEWLINE, "\n",       10, 1],
+      [:TEXT,    'for all',   0, 2],
+      [:NEWLINE, "\n",        7, 2],
+    ]
+
+    assert_equal expected, @RMP.tokenize(str)
+  end
+
+  def test_tokenize_line_break_long
+    str = "now is\nthe time   \nfor all\n"
+
+    expected = [
+      [:TEXT,    'now is',     0, 0],
+      [:NEWLINE, "\n",         6, 0],
+      [:TEXT,    'the time ',  0, 1],
+      [:BREAK,   '  ',         9, 1],
+      [:NEWLINE, "\n",        11, 1],
+      [:TEXT,    'for all',    0, 2],
+      [:NEWLINE, "\n",         7, 2],
+    ]
+
+    assert_equal expected, @RMP.tokenize(str)
+  end
+
+  def test_tokenize_line_break_no_short
+    str = "now is\nthe time \nfor all\n"
+
+    expected = [
+      [:TEXT,    'now is',    0, 0],
+      [:NEWLINE, "\n",        6, 0],
+      [:TEXT,    'the time ', 0, 1],
+      [:NEWLINE, "\n",        9, 1],
+      [:TEXT,    'for all',   0, 2],
+      [:NEWLINE, "\n",        7, 2],
+    ]
+
+    assert_equal expected, @RMP.tokenize(str)
+  end
+
   def test_tokenize_note
     str = <<-STR
 cat:: l1