Encode hostname directly instead of netloc (#174)

* Encode hostname directly instead netloc

* simplify logic

* Attend PR comments

* we don't need else

* Add more tests, encode username and password properly

* run black

* empty password test

* safe_url_string: encode | and % in userinfo

Co-authored-by: Adrián Chaves <[email protected]>

Author: Laerte

tests/test_url.py

@@ -303,6 +303,47 @@ def test_safe_url_string_preserve_nonfragment_hash(self):
             "http://www.example.com/path/to/%23/foo/bar?url=http%3A%2F%2Fwww.example.com%2F%2Fpath%2Fto%2F%23%2Fbar%2Ffoo#frag",
         )
 
+    def test_safe_url_string_encode_idna_domain_with_port(self):
+        self.assertEqual(
+            safe_url_string("http://新华网.中国:80"), "http://xn--xkrr14bows.xn--fiqs8s:80"
+        )
+
+    def test_safe_url_string_encode_idna_domain_with_username_password_and_port_number(
+        self,
+    ):
+        self.assertEqual(
+            safe_url_string("ftp://admin:admin@新华网.中国:21"),
+            "ftp://admin:[email protected]:21",
+        )
+        self.assertEqual(
+            safe_url_string("http://Åsa:abc123@➡.ws:81/admin"),
+            "http://%C3%85sa:[email protected]:81/admin",
+        )
+        self.assertEqual(
+            safe_url_string("http://japão:não@️i❤️.ws:8000/"),
+            "http://jap%C3%A3o:n%C3%[email protected]:8000/",
+        )
+
+    def test_safe_url_string_encode_idna_domain_with_username_and_empty_password_and_port_number(
+        self,
+    ):
+        self.assertEqual(
+            safe_url_string("ftp://admin:@新华网.中国:21"),
+            "ftp://admin:@xn--xkrr14bows.xn--fiqs8s:21",
+        )
+        self.assertEqual(
+            safe_url_string("ftp://admin@新华网.中国:21"),
+            "ftp://[email protected]:21",
+        )
+
+    def test_safe_url_string_userinfo_unsafe_chars(
+        self,
+    ):
+        self.assertEqual(
+            safe_url_string("ftp://admin:|%@example.com"),
+            "ftp://admin:%7C%[email protected]",
+        )
+
     def test_safe_download_url(self):
         self.assertEqual(
             safe_download_url("http://www.example.org"), "http://www.example.org/"

w3lib/url.py

@@ -55,6 +55,7 @@ def _quote_byte(error: UnicodeError) -> Tuple[str, int]:
 
 _safe_chars = RFC3986_RESERVED + RFC3986_UNRESERVED + EXTRA_SAFE_CHARS + b"%"
 _path_safe_chars = _safe_chars.replace(b"#", b"")
+RFC3986_USERINFO_SAFE_CHARS = RFC3986_UNRESERVED + RFC3986_SUB_DELIMS + b":"
 
 _ascii_tab_newline_re = re.compile(
     r"[\t\n\r]"
@@ -95,14 +96,34 @@ def safe_url_string(
     decoded = to_unicode(url, encoding=encoding, errors="percentencode")
     parts = urlsplit(_ascii_tab_newline_re.sub("", decoded))
 
-    # IDNA encoding can fail for too long labels (>63 characters)
-    # or missing labels (e.g. http://.example.com)
-    try:
-        netloc_bytes = parts.netloc.encode("idna")
-    except UnicodeError:
-        netloc = parts.netloc
-    else:
-        netloc = netloc_bytes.decode()
+    username, password, hostname, port = (
+        parts.username,
+        parts.password,
+        parts.hostname,
+        parts.port,
+    )
+    netloc_bytes = b""
+    if username is not None or password is not None:
+        if username is not None:
+            safe_username = quote(username, RFC3986_USERINFO_SAFE_CHARS)
+            netloc_bytes += safe_username.encode(encoding)
+        if password is not None:
+            netloc_bytes += b":"
+            safe_password = quote(password, RFC3986_USERINFO_SAFE_CHARS)
+            netloc_bytes += safe_password.encode(encoding)
+        netloc_bytes += b"@"
+    if hostname is not None:
+        try:
+            netloc_bytes += hostname.encode("idna")
+        except UnicodeError:
+            # IDNA encoding can fail for too long labels (>63 characters) or
+            # missing labels (e.g. http://.example.com)
+            netloc_bytes += hostname.encode(encoding)
+    if port is not None:
+        netloc_bytes += b":"
+        netloc_bytes += str(port).encode(encoding)
+
+    netloc = netloc_bytes.decode()
 
     # default encoding for path component SHOULD be UTF-8
     if quote_path:
@@ -113,7 +134,7 @@ def safe_url_string(
     return urlunsplit(
         (
             parts.scheme,
-            netloc.rstrip(":"),
+            netloc,
             path,
             quote(parts.query.encode(encoding), _safe_chars),
             quote(parts.fragment.encode(encoding), _safe_chars),