safe_url_string: make path quoting optional (#119)

Author: maramsumanth

tests/test_url.py

@@ -76,6 +76,17 @@ def test_safe_url_string_remove_ascii_tab_and_newlines(self):
     def test_safe_url_string_unsafe_chars(self):
         safeurl = safe_url_string(r"http://localhost:8001/unwise{,},|,\,^,[,],`?|=[]&[]=|")
         self.assertEqual(safeurl, r"http://localhost:8001/unwise%7B,%7D,|,%5C,%5E,[,],%60?|=[]&[]=|")
+        
+    def test_safe_url_string_quote_path(self):
+        safeurl = safe_url_string(u'http://google.com/"hello"', quote_path=True)
+        self.assertEqual(safeurl, u'http://google.com/%22hello%22')
+        
+        safeurl = safe_url_string(u'http://google.com/"hello"', quote_path=False)
+        self.assertEqual(safeurl, u'http://google.com/"hello"')
+        
+        safeurl = safe_url_string(u'http://google.com/"hello"')
+        self.assertEqual(safeurl, u'http://google.com/%22hello%22')
+        
 
     def test_safe_url_string_with_query(self):
         safeurl = safe_url_string(u"http://www.example.com/£?unit=µ")

w3lib/url.py

@@ -36,15 +36,18 @@ def _quote_byte(error):
 
 _ascii_tab_newline_re = re.compile(r'[\t\n\r]')  # see https://infra.spec.whatwg.org/#ascii-tab-or-newline
 
-def safe_url_string(url, encoding='utf8', path_encoding='utf8'):
+def safe_url_string(url, encoding='utf8', path_encoding='utf8', quote_path=True):
     """Convert the given URL into a legal URL by escaping unsafe characters
     according to RFC-3986. Also, ASCII tabs and newlines are removed
     as per https://url.spec.whatwg.org/#url-parsing.
 
     If a bytes URL is given, it is first converted to `str` using the given
-    encoding (which defaults to 'utf-8'). 'utf-8' encoding is used for
-    URL path component (unless overriden by path_encoding), and given
-    encoding is used for query string or form data.
+    encoding (which defaults to 'utf-8'). If quote_path is True (default), 
+    path_encoding ('utf-8' by default) is used to encode URL path component
+    which is then quoted. Otherwise, if quote_path is False, path component
+    is not encoded or quoted. Given encoding is used for query string 
+    or form data.
+
     When passing an encoding, you should use the encoding of the
     original page (the page from which the URL was extracted from).
 
@@ -69,15 +72,18 @@ def safe_url_string(url, encoding='utf8', path_encoding='utf8'):
     except UnicodeError:
         netloc = parts.netloc
 
+    # default encoding for path component SHOULD be UTF-8
+    if quote_path:
+        path = quote(to_bytes(parts.path, path_encoding), _safe_chars)
+    else:
+        path = to_native_str(parts.path)
+    
     # quote() in Python2 return type follows input type;
     # quote() in Python3 always returns Unicode (native str)
     return urlunsplit((
         to_native_str(parts.scheme),
         to_native_str(netloc).rstrip(':'),
-
-        # default encoding for path component SHOULD be UTF-8
-        quote(to_bytes(parts.path, path_encoding), _safe_chars),
-
+        path,
         # encoding of query and fragment follows page encoding
         # or form-charset (if known and passed)
         quote(to_bytes(parts.query, encoding), _safe_chars),