Merge pull request #77 from starrify/remove-comments-for-base-url

[MRG+1] Added: Removing comments before extracting base URLs. Not a solution to #70, but does help in some cases.

Author: wRAR

tests/test_html.py

@@ -372,6 +372,30 @@ def test_get_base_url(self):
             get_base_url(text, baseurl.encode("ascii")), "http://example.org/something"
         )
 
+    def test_base_url_in_comment(self):
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> -->"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/>"""), ""
+        )
+        self.assertEqual(
+            get_base_url("""<!-- <base href="http://example.com/"/> --"""), ""
+        )
+        self.assertEqual(
+            get_base_url(
+                """<!-- <!--  <base href="http://example.com/"/> -- -->  <base href="http://example_2.com/"/> """
+            ),
+            "http://example_2.com/",
+        )
+
+        self.assertEqual(
+            get_base_url(
+                """<!-- <base href="http://example.com/"/> --> <!-- <base href="http://example_2.com/"/> --> <base href="http://example_3.com/"/>"""
+            ),
+            "http://example_3.com/",
+        )
+
     def test_relative_url_with_absolute_path(self):
         baseurl = "https://example.org"
         text = """\

w3lib/html.py

@@ -311,7 +311,7 @@ def get_base_url(
 
     """
 
-    utext = to_unicode(text, encoding)
+    utext: str = remove_comments(text, encoding=encoding)
     m = _baseurl_re.search(utext)
     if m:
         return urljoin(