Merge pull request #38 from SethMichaelLarson/percent-encode-percent

Percent-encode % characters

Author: sigmavirus24

.travis.yml

@@ -9,15 +9,19 @@ matrix:
   include:
     - python: 2.7
       env: TOXENV=py27
-    - python: 3.3
-      env: TOXENV=py33
     - python: 3.4
       env: TOXENV=py34
     - python: 3.5
       env: TOXENV=py35
+    - python: 3.6
+      env: TOXENV=py36
+    - python: 3.7
+      env: TOXENV=py37
+      dist: xenial
+      sudo: true
     - python: pypy
       env: TOXENV=pypy
-    - python: 3.5
+    - python: 3.6
       env: TOXENV=flake8
     #- env: TOXENV=docs
 

setup.py

@@ -25,7 +25,7 @@
     author_email='[email protected]',
     url='http://rfc3986.readthedocs.io',
     packages=packages,
-    package_dir={'': 'src/'},
+    package_dir={'': 'src'},
     package_data={'': ['LICENSE']},
     include_package_data=True,
     license='Apache 2.0',
@@ -37,9 +37,9 @@
         'Programming Language :: Python',
         'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
         'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
     ),
 )

src/rfc3986/abnf_regexp.py

@@ -20,16 +20,16 @@
 SUB_DELIMS = SUB_DELIMITERS = "!$&'()*+,;="
 SUB_DELIMITERS_SET = set(SUB_DELIMITERS)
 # Escape the '*' for use in regular expressions
-SUB_DELIMITERS_RE = "!$&'()\*+,;="
+SUB_DELIMITERS_RE = r"!$&'()\*+,;="
 RESERVED_CHARS_SET = GENERIC_DELIMITERS_SET.union(SUB_DELIMITERS_SET)
 ALPHA = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
 DIGIT = '0123456789'
 # https://tools.ietf.org/html/rfc3986#section-2.3
 UNRESERVED = UNRESERVED_CHARS = ALPHA + DIGIT + '._!-'
 UNRESERVED_CHARS_SET = set(UNRESERVED_CHARS)
-NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET).union('%')
+NON_PCT_ENCODED_SET = RESERVED_CHARS_SET.union(UNRESERVED_CHARS_SET)
 # We need to escape the '-' in this case:
-UNRESERVED_RE = 'A-Za-z0-9._~\-'
+UNRESERVED_RE = r'A-Za-z0-9._~\-'
 
 # Percent encoded character values
 PERCENT_ENCODED = PCT_ENCODED = '%[A-Fa-f0-9]{2}'
@@ -59,9 +59,9 @@
 # modified to ignore other matches that are not important to the parsing of
 # the reference so we can also simply use SRE_Match#groups.
 URL_PARSING_RE = (
-    '(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?'
-    '(?P<path>{path})(?:\?(?P<query>{query}))?'
-    '(?:#(?P<fragment>{fragment}))?'
+    r'(?:(?P<scheme>{scheme}):)?(?://(?P<authority>{authority}))?'
+    r'(?P<path>{path})(?:\?(?P<query>{query}))?'
+    r'(?:#(?P<fragment>{fragment}))?'
 ).format(**COMPONENT_PATTERN_DICT)
 
 
@@ -120,7 +120,7 @@
 ZONE_ID = '(?:[' + UNRESERVED_RE + ']|' + PCT_ENCODED + ')+'
 IPv6_ADDRZ_RE = IPv6_RE + '%25' + ZONE_ID
 
-IP_LITERAL_RE = '\[({0}|(?:{1})|{2})\]'.format(
+IP_LITERAL_RE = r'\[({0}|(?:{1})|{2})\]'.format(
     IPv6_RE,
     IPv6_ADDRZ_RE,
     IPv_FUTURE_RE,

src/rfc3986/misc.py

@@ -75,13 +75,13 @@
 # Scheme validation, see: http://tools.ietf.org/html/rfc3986#section-3.1
 SCHEME_MATCHER = re.compile('^{0}$'.format(abnf_regexp.SCHEME_RE))
 
-RELATIVE_REF_MATCHER = re.compile('^%s(\?%s)?(#%s)?$' % (
+RELATIVE_REF_MATCHER = re.compile(r'^%s(\?%s)?(#%s)?$' % (
     abnf_regexp.RELATIVE_PART_RE, abnf_regexp.QUERY_RE,
     abnf_regexp.FRAGMENT_RE,
 ))
 
 # See http://tools.ietf.org/html/rfc3986#section-4.3
-ABSOLUTE_URI_MATCHER = re.compile('^%s:%s(\?%s)?$' % (
+ABSOLUTE_URI_MATCHER = re.compile(r'^%s:%s(\?%s)?$' % (
     abnf_regexp.COMPONENT_PATTERN_DICT['scheme'],
     abnf_regexp.HIER_PART_RE,
     abnf_regexp.QUERY_RE[1:-1],

src/rfc3986/normalizers.py

@@ -129,15 +129,22 @@ def encode_component(uri_component, encoding):
     if uri_component is None:
         return uri_component
 
+    # Try to see if the component we're encoding is already percent-encoded
+    # so we can skip all '%' characters but still encode all others.
+    percent_encodings = len(PERCENT_MATCHER.findall(
+                            compat.to_str(uri_component, encoding)))
+
     uri_bytes = compat.to_bytes(uri_component, encoding)
+    is_percent_encoded = percent_encodings == uri_bytes.count(b'%')
 
     encoded_uri = bytearray()
 
     for i in range(0, len(uri_bytes)):
         # Will return a single character bytestring on both Python 2 & 3
         byte = uri_bytes[i:i+1]
         byte_ord = ord(byte)
-        if byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED:
+        if ((is_percent_encoded and byte == b'%')
+                or (byte_ord < 128 and byte.decode() in misc.NON_PCT_ENCODED)):
             encoded_uri.extend(byte)
             continue
         encoded_uri.extend('%{0:02x}'.format(byte_ord).encode())

tests/base.py

@@ -119,6 +119,20 @@ def test_handles_relative_uri(self, relative_uri):
         assert uri.scheme is None
         assert uri.authority == relative_uri[2:]
 
+    def test_handles_percent_in_path(self, uri_path_with_percent):
+        """Test that self.test_class encodes the % character properly."""
+        uri = self.test_class.from_string(uri_path_with_percent)
+        print(uri.path)
+        assert uri.path == '/%25%20'
+
+    def test_handles_percent_in_query(self, uri_query_with_percent):
+        uri = self.test_class.from_string(uri_query_with_percent)
+        assert uri.query == 'a=%25'
+
+    def test_handles_percent_in_fragment(self, uri_fragment_with_percent):
+        uri = self.test_class.from_string(uri_fragment_with_percent)
+        assert uri.fragment == 'perc%25ent'
+
 
 class BaseTestUnsplits:
     test_class = None

tests/conftest.py

@@ -116,4 +116,20 @@ def absolute_path_uri():
 def invalid_uri(request):
     return 'https://%s' % request.param
 
+
+@pytest.fixture(params=valid_hosts)
+def uri_path_with_percent(request):
+    return 'https://%s/%% ' % request.param
+
+
+@pytest.fixture(params=valid_hosts)
+def uri_query_with_percent(request):
+    return 'https://%s?a=%%' % request.param
+
+
+@pytest.fixture(params=valid_hosts)
+def uri_fragment_with_percent(request):
+    return 'https://%s#perc%%ent' % request.param
+
+
 sys.path.insert(0, '.')

tests/test_builder.py

@@ -135,6 +135,7 @@ def test_add_path(path):
     ([('a', 'b+c')], 'a=b%2Bc'),
     ([('a', 'b'), ('c', 'd')], 'a=b&c=d'),
     ([('a', 'b'), ('username', '@d')], 'a=b&username=%40d'),
+    ([('percent', '%')], 'percent=%25'),
 ])
 def test_add_query_from(query_items, expected):
     """Verify the behaviour of add_query_from."""

tests/test_normalizers.py

@@ -3,7 +3,8 @@
 
 from rfc3986.uri import URIReference
 from rfc3986.normalizers import (
-    normalize_scheme, normalize_percent_characters, remove_dot_segments
+    normalize_scheme, normalize_percent_characters,
+    remove_dot_segments, encode_component,
     )
 
 
@@ -76,3 +77,19 @@ def test_fragment_normalization():
     uri = URIReference(
         None, 'example.com', None, None, 'fiz%DF').normalize()
     assert uri.fragment == 'fiz%DF'
+
+
+@pytest.mark.parametrize(
+    ["component", "encoded_component"],
+    [
+    ('/%', '/%25'),
+    ('/%a', '/%25a'),
+    ('/%ag', '/%25ag'),
+    ('/%af', '/%af'),
+    ('/%20/%', '/%2520/%25'),
+    ('/%20%25', '/%20%25'),
+    ('/%21%22%23%ah%12%ff', '/%2521%2522%2523%25ah%2512%25ff'),
+    ]
+)
+def test_detect_percent_encoded_component(component, encoded_component):
+    assert encode_component(component, 'utf-8') == encoded_component