Merge pull request #136 from scrapy/strip-spaces-in-canonicalize-url

Strip spaces in canonicalize_url

Author: wRAR

tests/test_url.py

@@ -1085,6 +1085,17 @@ def test_preserve_nonfragment_hash(self):
             "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag",
         )
 
+    def test_strip_spaces(self):
+        self.assertEqual(
+            canonicalize_url(" https://example.com"), "https://example.com/"
+        )
+        self.assertEqual(
+            canonicalize_url("https://example.com "), "https://example.com/"
+        )
+        self.assertEqual(
+            canonicalize_url(" https://example.com "), "https://example.com/"
+        )
+
 
 class DataURITests(unittest.TestCase):
     def test_default_mediatype_charset(self):

w3lib/url.py

@@ -538,6 +538,8 @@ def canonicalize_url(
     # UTF-8 can handle all Unicode characters,
     # so we should be covered regarding URL normalization,
     # if not for proper URL expected by remote website.
+    if isinstance(url, str):
+        url = url.strip()
     try:
         scheme, netloc, path, params, query, fragment = _safe_ParseResult(
             parse_url(url), encoding=encoding or "utf8"