Remove ASCII control characters before reading the input

libxml2 removes them during serialization which means that
an input like <a href="java\x1bscript:alert()">Link</a>
contains the control character during cleaning but
not after it's serialized back to a string.

This basically overcomes checks in lxml_html_clean
and the output might then be malicious even the input is not.

Author: frenzymadness

lxml_html_clean/clean.py

@@ -12,7 +12,7 @@
 
 from lxml import etree
 from lxml.html import defs
-from lxml.html import fromstring, XHTML_NAMESPACE
+from lxml.html import fromstring as lxml_fromstring, XHTML_NAMESPACE
 from lxml.html import xhtml_to_html, _transform_result
 
 
@@ -83,6 +83,21 @@ def _has_javascript_scheme(s):
      "descendant-or-self::x:a[normalize-space(@href) and substring(normalize-space(@href),1,1) != '#']"),
     namespaces={'x':XHTML_NAMESPACE})
 
+# Regex to remove all ASCII control characters (00-1F,7F) except:
+# - 09 - Horizontal tab
+# - 0A - Line Feed
+# - 0B - Vertical tab
+# - 0D - Carriage Return
+_ascii_control_characters = re.compile(r"[\x00-\x08\x0C\x0E-\x1F\x7F]")
+
+
+def fromstring(string):
+    """
+    Enhanced fromstring function that removes ASCII control chars
+    before passing the input to the original lxml.html.fromstring.
+    """
+    return lxml_fromstring(_ascii_control_characters.sub("", string))
+
 
 class Cleaner:
     """

tests/test_clean.py

@@ -327,3 +327,9 @@ def test_host_whitelist_sneaky_userinfo(self):
         expected = '<div></div>'
         cleaner = Cleaner(frames=False, host_whitelist=["example.com"])
         self.assertEqual(expected, cleaner.clean_html(html))
+
+    def test_ascii_control_chars_removed(self):
+        html = """<a href="java\x1bscript:alert()">Link</a>"""
+        expected = """<a href="">Link</a>"""
+        cleaner = Cleaner()
+        self.assertEqual(expected, cleaner.clean_html(html))