Use a real parser for RDoc::Markup::ToHtml#parseable? (Fixes #320)

This uses BEGIN {return true} when evaluating the text, which will do
a syntax check, but not actually execute the code.

Using a real ruby parser is a big improvement for two reasons.
First, it highlights valid ruby syntax that was left unhighlighted
by the previous parser.  Second and more important, it doesn't
highlight invalid ruby syntax, which makes it easier to catch
errors in example code in the documentation.  This even fixes some
errors in the tests, where code like "class C end" was considered
valid syntax.

There is some fallout in the tests where code that wasn't highlighted
before is now highlighted, this also fixes those cases so the tests
pass.

Author: jeremyevans

lib/rdoc/markup/to_html.rb

@@ -379,11 +379,12 @@ def list_end_for(list_type)
   end
 
   ##
-  # Returns true if Ripper is available it can create a sexp from +text+
+  # Returns true if text is valid ruby syntax
 
   def parseable? text
-    text =~ /\b(def|class|module|require) |=>|\{\s?\||do \|/ and
-      text !~ /<%|%>/
+    eval("BEGIN {return true}\n#{text}")
+  rescue SyntaxError
+    false
   end
 
   ##

test/test_rdoc_markup_to_html.rb

@@ -292,7 +292,7 @@ def accept_rule
   end
 
   def accept_verbatim
-    assert_equal "\n<pre>hi\n  world</pre>\n", @to.res.join
+    assert_equal "\n<pre class=\"ruby\"><span class=\"ruby-identifier\">hi</span>\n  <span class=\"ruby-identifier\">world</span>\n</pre>\n", @to.res.join
   end
 
   def end_accepting
@@ -444,8 +444,7 @@ def test_accept_verbatim_parseable_error
 
     expected = <<-EXPECTED
 
-<pre>#{inner}
-</pre>
+<pre>#{inner}</pre>
     EXPECTED
 
     assert_equal expected, @to.res.join
@@ -604,8 +603,9 @@ def test_list_verbatim_2
 <ul><li>
 <p>one</p>
 
-<pre>verb1
-verb2</pre>
+<pre class=\"ruby\"><span class=\"ruby-identifier\">verb1</span>
+<span class=\"ruby-identifier\">verb2</span>
+</pre>
 </li><li>
 <p>two</p>
 </li></ul>
@@ -615,16 +615,36 @@ def test_list_verbatim_2
   end
 
   def test_parseable_eh
-    assert @to.parseable?('def x() end'),      'def'
-    assert @to.parseable?('class C end'),      'class'
-    assert @to.parseable?('module M end'),     'module'
-    assert @to.parseable?('a # => blah'),      '=>'
-    assert @to.parseable?('x { |y| ... }'),    '{ |x|'
-    assert @to.parseable?('x do |y| ... end'), 'do |x|'
-    refute @to.parseable?('* 1'),              '* 1'
-    refute @to.parseable?('# only a comment'), '# only a comment'
-    refute @to.parseable?('<% require "foo" %>'),    'ERB'
-    refute @to.parseable?('class="foo"'),      'HTML class'
+    valid_syntax = [
+      'def x() end',
+      'def x; end',
+      'class C; end',
+      "module M end",
+      'a # => blah',
+      'x { |y| nil }',
+      'x do |y| nil end',
+      '# only a comment',
+      'require "foo"',
+      'cls="foo"'
+    ]
+    invalid_syntax = [
+      'def x end',
+      'class C end',
+      'class C < end',
+      'module M < C end',
+      'a=># blah',
+      'x { |y| ... }',
+      'x do |y| ... end',
+      '// only a comment',
+      '<% require "foo" %>',
+      'class="foo"'
+    ]
+    valid_syntax.each do |t|
+      assert @to.parseable?(t), "valid syntax considered invalid: #{t}"
+    end
+    invalid_syntax.each do |t|
+      refute @to.parseable?(t), "invalid syntax considered valid: #{t}"
+    end
   end
 
   def test_to_html

test/test_rdoc_markup_to_html_snippet.rb

@@ -309,7 +309,7 @@ def accept_rule
   end
 
   def accept_verbatim
-    assert_equal "\n<pre>hi\n  world</pre>\n", @to.res.join
+    assert_equal "\n<pre class=\"ruby\"><span class=\"ruby-identifier\">hi</span>\n  <span class=\"ruby-identifier\">world</span>\n</pre>\n", @to.res.join
     assert_equal 10, @to.characters
   end
 
@@ -427,8 +427,7 @@ def test_accept_verbatim_ruby_error
 
     expected = <<-EXPECTED
 
-<pre>#{inner}
-</pre>
+<pre>#{inner}</pre>
     EXPECTED
 
     assert_equal expected, @to.res.join
@@ -588,8 +587,9 @@ def test_convert_limit_verbatim_multiline
     expected = <<-EXPECTED
 <p>Look for directives in a normal comment block:
 
-<pre># :stopdoc:
-#{inner}</pre>
+<pre class=\"ruby\"><span class=\"ruby-comment\"># :stopdoc:</span>
+<span class=\"ruby-comment\">#{inner}</span>
+</pre>
     EXPECTED
 
     actual = @to.convert rdoc
@@ -665,8 +665,9 @@ def test_list_verbatim_2
     expected = <<-EXPECTED
 <p>one
 
-<pre>verb1
-verb2</pre>
+<pre class=\"ruby\"><span class=\"ruby-identifier\">verb1</span>
+<span class=\"ruby-identifier\">verb2</span>
+</pre>
 <p>two
 
     EXPECTED