Skip to content

Fix issue #162 add different regex pattern to search for meta tags #179

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Jun 16, 2022
Merged

Fix issue #162 add different regex pattern to search for meta tags #179

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
bb741e6
Add different regex pattern to search for meta tags
AngelikiBoura Jun 13, 2022
17e4780
Remove unneeded lines of code
AngelikiBoura Jun 14, 2022
a91b174
Merge pull request #1 from AngelikiBoura/still-bitting
AngelikiBoura Jun 14, 2022
80c1ea7
Fix new regex pattern's flexibility
AngelikiBoura Jun 15, 2022
83a1ecf
Fix format
AngelikiBoura Jun 15, 2022
e2339fb
Apply black
Gallaecio Jun 15, 2022
9757664
Freeze black
Gallaecio Jun 15, 2022
a0a0b0d
Freeze pylint
Gallaecio Jun 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 27 additions & 86 deletions tests/test_html.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,54 +73,18 @@ def test_browser_hack(self):

def test_missing_semicolon(self):
for entity, result in (
(
"&lt&lt!",
"<<!",
),
(
"&LT!",
"<!",
),
(
"&#X41 ",
"A ",
),
(
"&#x41!",
"A!",
),
(
"&#x41h",
"Ah",
),
(
"&#65!",
"A!",
),
(
"&#65x",
"Ax",
),
(
"&sup3!",
"\u00B3!",
),
(
"&Aacute!",
"\u00C1!",
),
(
"&#9731!",
"\u2603!",
),
(
"&#153",
"\u2122",
),
(
"&#x99",
"\u2122",
),
("&lt&lt!", "<<!"),
("&LT!", "<!"),
("&#X41 ", "A "),
("&#x41!", "A!"),
("&#x41h", "Ah"),
("&#65!", "A!"),
("&#65x", "Ax"),
("&sup3!", "\u00B3!"),
("&Aacute!", "\u00C1!"),
("&#9731!", "\u2603!"),
("&#153", "\u2122"),
("&#x99", "\u2122"),
):
self.assertEqual(replace_entities(entity, encoding="cp1252"), result)
self.assertEqual(
Expand Down Expand Up @@ -203,16 +167,7 @@ def test_returns_unicode(self):
def test_remove_tags_without_tags(self):
# text without tags
self.assertEqual(remove_tags("no tags"), "no tags")
self.assertEqual(
remove_tags(
"no tags",
which_ones=(
"p",
"b",
),
),
"no tags",
)
self.assertEqual(remove_tags("no tags", which_ones=("p", "b")), "no tags")

def test_remove_tags(self):
# text with tags
Expand Down Expand Up @@ -294,14 +249,7 @@ def test_without_tags(self):
# text without tags
self.assertEqual(remove_tags_with_content("no tags"), "no tags")
self.assertEqual(
remove_tags_with_content(
"no tags",
which_ones=(
"p",
"b",
),
),
"no tags",
remove_tags_with_content("no tags", which_ones=("p", "b")), "no tags"
)

def test_with_tags(self):
Expand Down Expand Up @@ -340,28 +288,10 @@ def test_returns_unicode(self):
assert isinstance(replace_escape_chars(b"no ec"), str)
assert isinstance(replace_escape_chars(b"no ec", replace_by="str"), str)
assert isinstance(replace_escape_chars(b"no ec", replace_by="str"), str)
assert isinstance(
replace_escape_chars(
b"no ec",
which_ones=(
"\n",
"\t",
),
),
str,
)
assert isinstance(replace_escape_chars(b"no ec", which_ones=("\n", "\t")), str)
assert isinstance(replace_escape_chars("no ec"), str)
assert isinstance(replace_escape_chars("no ec", replace_by="str"), str)
assert isinstance(
replace_escape_chars(
"no ec",
which_ones=(
"\n",
"\t",
),
),
str,
)
assert isinstance(replace_escape_chars("no ec", which_ones=("\n", "\t")), str)

def test_without_escape_chars(self):
# text without escape chars
Expand Down Expand Up @@ -669,3 +599,14 @@ def test_inside_script(self):
get_meta_refresh(body, baseurl, ignore_tags=()),
(0.0, "http://example.org/foobar_required"),
)

def test_redirections_in_different_ordering__in_meta_tag(self):
baseurl = "http://localhost:8000"
url1 = '<html><head><meta http-equiv="refresh" content="0;url=dummy.html"></head></html>'
url2 = '<html><head><meta content="0;url=dummy.html" http-equiv="refresh"></head></html>'
self.assertEqual(
get_meta_refresh(url1, baseurl), (0.0, "http://localhost:8000/dummy.html")
)
self.assertEqual(
get_meta_refresh(url2, baseurl), (0.0, "http://localhost:8000/dummy.html")
)
7 changes: 6 additions & 1 deletion w3lib/html.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,11 @@
r'<meta\s[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)',
re.DOTALL | re.IGNORECASE,
)
_meta_refresh_re2 = re.compile(
r'<meta\s[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=\s*(?P<url>.*?)(?P=quote)[^>]*?\shttp-equiv\s*=[^>]*refresh',
re.DOTALL | re.IGNORECASE,
)

_cdata_re = re.compile(
r"((?P<cdata_s><!\[CDATA\[)(?P<cdata_d>.*?)(?P<cdata_e>\]\]>))", re.DOTALL
)
Expand Down Expand Up @@ -338,7 +343,7 @@ def get_meta_refresh(
raise
utext = remove_tags_with_content(utext, ignore_tags)
utext = remove_comments(replace_entities(utext))
m = _meta_refresh_re.search(utext)
m = _meta_refresh_re.search(utext) or _meta_refresh_re2.search(utext)
if m:
interval = float(m.group("int"))
url = safe_url_string(m.group("url").strip(" \"'"), encoding)
Expand Down