Merge pull request #179 from AngelikiBoura/master

Fix issue #162 add different regex pattern to search for meta tags

Author: kmike

pylintrc

@@ -7,6 +7,7 @@ disable=bad-continuation,
         consider-using-in,
         expression-not-assigned,
         fixme,
+        implicit-str-concat,
         import-error,
         import-outside-toplevel,
         inconsistent-return-statements,
@@ -31,8 +32,10 @@ disable=bad-continuation,
         trailing-newlines,
         trailing-whitespace,
         unidiomatic-typecheck,
+        unnecessary-lambda-assignment,
         unreachable,
         unused-argument,
         unused-variable,
+        useless-option-value,
         wrong-import-order,
         wrong-import-position

tests/test_html.py

@@ -73,54 +73,18 @@ def test_browser_hack(self):
 
     def test_missing_semicolon(self):
         for entity, result in (
-            (
-                "&lt&lt!",
-                "<<!",
-            ),
-            (
-                "&LT!",
-                "<!",
-            ),
-            (
-                "&#X41 ",
-                "A ",
-            ),
-            (
-                "&#x41!",
-                "A!",
-            ),
-            (
-                "&#x41h",
-                "Ah",
-            ),
-            (
-                "&#65!",
-                "A!",
-            ),
-            (
-                "&#65x",
-                "Ax",
-            ),
-            (
-                "&sup3!",
-                "\u00B3!",
-            ),
-            (
-                "&Aacute!",
-                "\u00C1!",
-            ),
-            (
-                "&#9731!",
-                "\u2603!",
-            ),
-            (
-                "&#153",
-                "\u2122",
-            ),
-            (
-                "&#x99",
-                "\u2122",
-            ),
+            ("&lt&lt!", "<<!"),
+            ("&LT!", "<!"),
+            ("&#X41 ", "A "),
+            ("&#x41!", "A!"),
+            ("&#x41h", "Ah"),
+            ("&#65!", "A!"),
+            ("&#65x", "Ax"),
+            ("&sup3!", "\u00B3!"),
+            ("&Aacute!", "\u00C1!"),
+            ("&#9731!", "\u2603!"),
+            ("&#153", "\u2122"),
+            ("&#x99", "\u2122"),
         ):
             self.assertEqual(replace_entities(entity, encoding="cp1252"), result)
             self.assertEqual(
@@ -203,16 +167,7 @@ def test_returns_unicode(self):
     def test_remove_tags_without_tags(self):
         # text without tags
         self.assertEqual(remove_tags("no tags"), "no tags")
-        self.assertEqual(
-            remove_tags(
-                "no tags",
-                which_ones=(
-                    "p",
-                    "b",
-                ),
-            ),
-            "no tags",
-        )
+        self.assertEqual(remove_tags("no tags", which_ones=("p", "b")), "no tags")
 
     def test_remove_tags(self):
         # text with tags
@@ -294,14 +249,7 @@ def test_without_tags(self):
         # text without tags
         self.assertEqual(remove_tags_with_content("no tags"), "no tags")
         self.assertEqual(
-            remove_tags_with_content(
-                "no tags",
-                which_ones=(
-                    "p",
-                    "b",
-                ),
-            ),
-            "no tags",
+            remove_tags_with_content("no tags", which_ones=("p", "b")), "no tags"
         )
 
     def test_with_tags(self):
@@ -340,28 +288,10 @@ def test_returns_unicode(self):
         assert isinstance(replace_escape_chars(b"no ec"), str)
         assert isinstance(replace_escape_chars(b"no ec", replace_by="str"), str)
         assert isinstance(replace_escape_chars(b"no ec", replace_by="str"), str)
-        assert isinstance(
-            replace_escape_chars(
-                b"no ec",
-                which_ones=(
-                    "\n",
-                    "\t",
-                ),
-            ),
-            str,
-        )
+        assert isinstance(replace_escape_chars(b"no ec", which_ones=("\n", "\t")), str)
         assert isinstance(replace_escape_chars("no ec"), str)
         assert isinstance(replace_escape_chars("no ec", replace_by="str"), str)
-        assert isinstance(
-            replace_escape_chars(
-                "no ec",
-                which_ones=(
-                    "\n",
-                    "\t",
-                ),
-            ),
-            str,
-        )
+        assert isinstance(replace_escape_chars("no ec", which_ones=("\n", "\t")), str)
 
     def test_without_escape_chars(self):
         # text without escape chars
@@ -669,3 +599,14 @@ def test_inside_script(self):
             get_meta_refresh(body, baseurl, ignore_tags=()),
             (0.0, "http://example.org/foobar_required"),
         )
+
+    def test_redirections_in_different_ordering__in_meta_tag(self):
+        baseurl = "http://localhost:8000"
+        url1 = '<html><head><meta http-equiv="refresh" content="0;url=dummy.html"></head></html>'
+        url2 = '<html><head><meta content="0;url=dummy.html" http-equiv="refresh"></head></html>'
+        self.assertEqual(
+            get_meta_refresh(url1, baseurl), (0.0, "http://localhost:8000/dummy.html")
+        )
+        self.assertEqual(
+            get_meta_refresh(url2, baseurl), (0.0, "http://localhost:8000/dummy.html")
+        )

tox.ini

@@ -42,13 +42,13 @@ commands =
 [testenv:pylint]
 deps =
     {[testenv]deps}
-    pylint
+    pylint==2.14.2
 commands =
     pylint conftest.py docs setup.py tests w3lib
 
 [testenv:black]
 deps =
-    black
+    black==22.3.0
 commands =
     black --check {posargs:conftest.py setup.py tests w3lib}
 

w3lib/html.py

@@ -21,6 +21,11 @@
     r'<meta\s[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)',
     re.DOTALL | re.IGNORECASE,
 )
+_meta_refresh_re2 = re.compile(
+    r'<meta\s[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)[^>]*?\shttp-equiv\s*=[^>]*refresh',
+    re.DOTALL | re.IGNORECASE,
+)
+
 _cdata_re = re.compile(
     r"((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))", re.DOTALL
 )
@@ -228,7 +233,7 @@ def remove_tags_with_content(
 
     utext = to_unicode(text, encoding)
     if which_ones:
-        tags = "|".join([fr"<{tag}\b.*?</{tag}>|<{tag}\s*/>" for tag in which_ones])
+        tags = "|".join([rf"<{tag}\b.*?</{tag}>|<{tag}\s*/>" for tag in which_ones])
         retags = re.compile(tags, re.DOTALL | re.IGNORECASE)
         utext = retags.sub("", utext)
     return utext
@@ -338,7 +343,7 @@ def get_meta_refresh(
         raise
     utext = remove_tags_with_content(utext, ignore_tags)
     utext = remove_comments(replace_entities(utext))
-    m = _meta_refresh_re.search(utext)
+    m = _meta_refresh_re.search(utext) or _meta_refresh_re2.search(utext)
     if m:
         interval = float(m.group("int"))
         url = safe_url_string(m.group("url").strip(" \"'"), encoding)

w3lib/url.py

@@ -446,7 +446,7 @@ def parse_data_uri(uri: StrOrBytes) -> ParseDataURIResult:
         if m:
             attribute, value, value_quoted = m.groups()
             if value_quoted:
-                value = re.sub(br"\\(.)", rb"\1", value_quoted)
+                value = re.sub(rb"\\(.)", rb"\1", value_quoted)
             media_type_params[attribute.decode()] = value.decode()
             uri = uri[m.end() :]
         else: