Add support for the preprocess_html flag.

This adds support for replacing consecutive spaces/nbsp to be converted to plain spaces for use with the Rouge lexer.

Author: lowjoel

lib/html/pipeline/rouge_filter.rb

@@ -13,8 +13,7 @@ def call
           default   = must_str(context[:highlight])
           next unless lang = node["lang"] || default
           next unless lexer = lexer_for(lang)
-          node.css("br").each { |br| br.replace("\n") } if replace_br
-          text = node.inner_text
+          text = preprocess_html(node)
           html = highlight_with(lexer, text)
           next if html.nil?
 
@@ -28,6 +27,15 @@ def call
         doc
       end
 
+      def preprocess_html(node)
+        node.css("br").each { |br| br.replace("\n") } if replace_br?
+        result = node.inner_text
+        return result unless preprocess_html?
+
+        result.gsub!("\u00A0", ' ')
+        result
+      end
+
       def highlight_with(lexer, text)
         formatter.format(lexer.lex(text))
       end
@@ -40,8 +48,12 @@ def line_numbers
         context[:line_numbers] || false
       end
 
-      def replace_br
-        context[:replace_br] || false
+      def replace_br?
+        context[:replace_br] || preprocess_html?
+      end
+
+      def preprocess_html?
+        context[:preprocess_html] || false
       end
 
       def formatter(css_class: default_css_class)

test/rouge_filter_test.rb

@@ -97,4 +97,13 @@ def test_replacing_br
     assert_equal "<pre class=\"highlight highlight-ruby\"><code>"\
                  "<span class=\"n\">hello</span>\n<span class=\"n\">world</span></code></pre>\n", doc.to_html
   end
+
+  def test_preprocess_html
+    filter = RougeFilter.new \
+      "<pre lang='ruby'> &nbsp;hello<br>world</pre>", preprocess_html: true
+
+    doc = filter.call
+    assert_equal "<pre class=\"highlight highlight-ruby\"><code>  "\
+                 "<span class=\"n\">hello</span>\n<span class=\"n\">world</span></code></pre>\n", doc.to_html
+  end
 end