Percent-encode % chars

Author: sethmlarson

src/rfc3986/abnf_regexp.py

@@ -20,16 +20,16 @@
 SUB_DELIMS = SUB_DELIMITERS = "!$&'()*+,;="
 SUB_DELIMITERS_SET = set(SUB_DELIMITERS)
 # Escape the '*' for use in regular expressions
-SUB_DELIMITERS_RE = "!$&'()\*+,;="
+SUB_DELIMITERS_RE = r"!$&'()\*+,;="
 RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET)
 ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
 DIGIT = '0123456789'
 # https://tools.ietf.org/html/rfc3986#section-2.3
 UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + '._!-'
 UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS)
-NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET).union('%')
+NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET)
 # We need to escape the '-' in this case:
-UNRESERVED_RE = 'A-Za-z0-9._~\-'
+UNRESERVED_RE = r'A-Za-z0-9._~\-'
 
 # Percent encoded character values
 PERCENT_ENCODED = PCT_ENCODED = '%[A-Fa-f0-9]{2}'
@@ -59,9 +59,9 @@
 # modified to ignore other matches that are not important to the parsing of
 # the reference so we can also simply use SRE_Match#groups.
 URL_PARSING_RE = (
-    '(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?'
-    '(?P<path>{path})(?:\?(?P<query>{query}))?'
-    '(?:#(?P<fragment>{fragment}))?'
+    r'(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?'
+    r'(?P<path>{path})(?:\?(?P<query>{query}))?'
+    r'(?:#(?P<fragment>{fragment}))?'
 ).format(**COMPONENT_PATTERN_DICT)
 
 
@@ -120,7 +120,7 @@
 ZONE_ID = '(?:[' + UNRESERVED_RE + ']|' + PCT_ENCODED + ')+'
 IPv6_ADDRZ_RE = IPv6_RE + '%25' + ZONE_ID
 
-IP_LITERAL_RE = '\[({0}|(?:{1})|{2})\]'.format(
+IP_LITERAL_RE = r'\[({0}|(?:{1})|{2})\]'.format(
     IPv6_RE,
     IPv6_ADDRZ_RE,
     IPv_FUTURE_RE,

src/rfc3986/misc.py

@@ -75,13 +75,13 @@
 # Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1
 SCHEME_MATCHER = re.compile('^{0}$'.format(abnf_regexp.SCHEME_RE))
 
-RELATIVE_REF_MATCHER = re.compile('^%s(\?%s)?(#%s)?$' % (
+RELATIVE_REF_MATCHER = re.compile(r'^%s(\?%s)?(#%s)?$' % (
     abnf_regexp.RELATIVE_PART_RE, abnf_regexp.QUERY_RE,
     abnf_regexp.FRAGMENT_RE,
 ))
 
 # See http://tools.ietf.org/html/rfc3986#section-4.3
-ABSOLUTE_URI_MATCHER = re.compile('^%s:%s(\?%s)?$' % (
+ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % (
     abnf_regexp.COMPONENT_PATTERN_DICT['scheme'],
     abnf_regexp.HIER_PART_RE,
     abnf_regexp.QUERY_RE[1:-1],

src/rfc3986/normalizers.py

@@ -129,15 +129,23 @@ def encode_component(uri_component, encoding):
     if uri_component is None:
         return uri_component
 
+    # Try to see if the component we're encoding is already percent-encoded
+    # so we can skip all '%' characters but still encode all others.
+    percent_encodings = len(PERCENT_MATCHER.findall(
+                            compat.to_str(uri_component, encoding)))
+
     uri_bytes = compat.to_bytes(uri_component, encoding)
+    is_percent_encoded = (percent_encodings > 0
+                          and percent_encodings == uri_bytes.count(b'%'))
 
     encoded_uri = bytearray()
 
     for i in range(0, len(uri_bytes)):
         # Will return a single character bytestring on both Python 2 & 3
         byte = uri_bytes[i:i+1]
         byte_ord = ord(byte)
-        if byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED:
+        if ((is_percent_encoded and byte == b'%')
+                or (byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED)):
             encoded_uri.extend(byte)
             continue
         encoded_uri.extend('%{0:02x}'.format(byte_ord).encode())

tests/base.py

@@ -119,6 +119,20 @@ def test_handles_relative_uri(self, relative_uri):
         assert uri.scheme is None
         assert uri.authority == relative_uri[2:]
 
+    def test_handles_percent_in_path(self, uri_path_with_percent):
+        """Test that self.test_class encodes the % character properly."""
+        uri = self.test_class.from_string(uri_path_with_percent)
+        print(uri.path)
+        assert uri.path == '/%25%20'
+
+    def test_handles_percent_in_query(self, uri_query_with_percent):
+        uri = self.test_class.from_string(uri_query_with_percent)
+        assert uri.query == 'a=%25'
+
+    def test_handles_percent_in_fragment(self, uri_fragment_with_percent):
+        uri = self.test_class.from_string(uri_fragment_with_percent)
+        assert uri.fragment == 'perc%25ent'
+
 
 class BaseTestUnsplits:
     test_class = None

tests/conftest.py

@@ -116,4 +116,20 @@ def absolute_path_uri():
 def invalid_uri(request):
     return 'https://%s' % request.param
 
+
+@pytest.fixture(params=valid_hosts)
+def uri_path_with_percent(request):
+    return 'https://%s/%% ' % request.param
+
+
+@pytest.fixture(params=valid_hosts)
+def uri_query_with_percent(request):
+    return 'https://%s?a=%%' % request.param
+
+
+@pytest.fixture(params=valid_hosts)
+def uri_fragment_with_percent(request):
+    return 'https://%s#perc%%ent' % request.param
+
+
 sys.path.insert(0, '.')

tests/test_builder.py

@@ -135,6 +135,7 @@ def test_add_path(path):
     ([('a', 'b+c')], 'a=b%2Bc'),
     ([('a', 'b'), ('c', 'd')], 'a=b&c=d'),
     ([('a', 'b'), ('username', '@d')], 'a=b&username=%40d'),
+    ([('percent', '%')], 'percent=%25'),
 ])
 def test_add_query_from(query_items, expected):
     """Verify the behaviour of add_query_from."""