Merge pull request #63 from redapple/safe-url-idna-error

Handle IDNA encoding failures when making URLs safe

Author: kmike

tests/test_url.py

@@ -165,6 +165,20 @@ def test_safe_url_idna(self):
             safeurl = safe_url_string(safe_result)
             self.assertEqual(safeurl, safe_result)
 
+    def test_safe_url_idna_encoding_failure(self):
+        # missing DNS label
+        self.assertEqual(
+            safe_url_string(u"http://.example.com/résumé?q=résumé"),
+            "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+
+        # DNS label too long
+        self.assertEqual(
+            safe_url_string(
+                u"http://www.{label}.com/résumé?q=résumé".format(
+                    label=u"example"*11)),
+            "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
+                    label=u"example"*11))
+
     def test_safe_download_url(self):
         self.assertEqual(safe_download_url('http://www.example.org'),
                          'http://www.example.org/')

w3lib/url.py

@@ -90,11 +90,18 @@ def safe_url_string(url, encoding='utf8', path_encoding='utf8'):
     parts = urlsplit(to_unicode(url, encoding=encoding,
                                 errors='percentencode'))
 
+    # IDNA encoding can fail for too long labels (>63 characters)
+    # or missing labels (e.g. http://.example.com)
+    try:
+        netloc = parts.netloc.encode('idna')
+    except UnicodeError:
+        netloc = parts.netloc
+
     # quote() in Python2 return type follows input type;
     # quote() in Python3 always returns Unicode (native str)
     return urlunsplit((
         to_native_str(parts.scheme),
-        to_native_str(parts.netloc.encode('idna')),
+        to_native_str(netloc),
 
         # default encoding for path component SHOULD be UTF-8
         quote(to_bytes(parts.path, path_encoding), _safe_chars),