Merge pull request #124 from akshayphilar/remove-null-bytes

lopuhin · web-flow · commit b376686563db · 2018-10-25T18:54:35.000+03:00
[MRG+1] remove null bytes from html body
diff --git a/parsel/selector.py b/parsel/selector.py
@@ -38,7 +38,7 @@ def _st(st):
 def create_root_node(text, parser_cls, base_url=None):
     """Create root node for text using given parser class.
     """
-    body = text.strip().encode('utf8') or b'<html/>'
+    body = text.strip().replace('\x00', '').encode('utf8') or b'<html/>'
     parser = parser_cls(recover=True, encoding='utf8')
     root = etree.fromstring(body, parser=parser, base_url=base_url)
     if root is None:
diff --git a/tests/test_selector.py b/tests/test_selector.py
@@ -694,7 +694,6 @@ def test_configure_base_url(self):
         sel = self.sscls(text=u'nothing', base_url='http://example.com')
         self.assertEqual(u'http://example.com', sel.root.base)
 
-
     def test_extending_selector(self):
         class MySelectorList(Selector.selectorlist_cls):
             pass
@@ -708,6 +707,11 @@ class MySelector(Selector):
         self.assertIsInstance(sel.css('div'), MySelectorList)
         self.assertIsInstance(sel.css('div')[0], MySelector)
 
+    def test_replacement_null_char_from_body(self):
+        text = u'<html>\x00<body><p>Grainy</p></body></html>'
+        self.assertEqual(u'<html><body><p>Grainy</p></body></html>',
+                          self.sscls(text).extract())
+
 class ExsltTestCase(unittest.TestCase):
 
     sscls = Selector
