Fixed URL encoding & decoding issues.

Author: tijme

nyawc/helpers/URLHelper.py

@@ -86,7 +86,7 @@ def append_with_data(url, data):
         query = OrderedDict(parse_qsl(url_parts[4], keep_blank_values=True))
         query.update(data)
 
-        url_parts[4] = urlencode(query)
+        url_parts[4] = URLHelper.query_dict_to_string(query)
 
         return urlunparse(url_parts)
 
@@ -233,7 +233,7 @@ def get_ordered_params(url):
         if url not in URLHelper.__cache:
             URLHelper.__cache[url] = urlparse(url)
 
-        params = dict(parse_qsl(URLHelper.__cache[url].query, keep_blank_values=True))
+        params = URLHelper.query_string_to_dict(URLHelper.__cache[url].query)
 
         return OrderedDict(sorted(params.items()))
 
@@ -250,3 +250,54 @@ def remove_hash(url):
         """
 
         return url.split("#")[0]
+
+    @staticmethod
+    def query_dict_to_string(query):
+        """Convert an OrderedDict to a query string.
+
+        Args:
+            query (obj): The key value object with query params.
+
+        Returns:
+            str: The query string.
+
+        Note:
+            This method does the same as urllib.parse.urlencode except
+            that it doesn't actually encode the values.
+
+        """
+
+        query_params = []
+
+        for key, value in query.items():
+            query_params.append(key + "=" + value)
+
+        return "&".join(query_params)
+
+    @staticmethod
+    def query_string_to_dict(query):
+        """Convert a string to a query dict.
+
+        Args:
+            query (str): The query string.
+
+        Returns:
+            obj: The key value object with query params.
+
+        Note:
+            This method does the same as urllib.parse.parse_qsl except
+            that it doesn't actually decode the values.
+
+        """
+
+        query_params = {}
+
+        for key_value in query.split("&"):
+            key_value_pair = key_value.split("=", 1)
+
+            key = key_value_pair[0] if len(key_value_pair) >= 1 else ""
+            value = key_value_pair[1] if len(key_value_pair) == 2 else ""
+
+            query_params[key] = value
+
+        return query_params

test/test_helpers_url_helper.py

@@ -142,3 +142,12 @@ def test_get_ordered_params(self):
         val2 = URLHelper.get_ordered_params("http://sub.domain.ltd?c=c&b=b&a=a&d=d")
 
         self.assertEqual(val1, val2)
+
+    def test_append_with_data_encoded_and_decoded(self):
+        """Make sure values do not get decoded or encoded."""
+
+        val1 = URLHelper.append_with_data("http://example.tld/", {"val": "{{aaaa}}"})
+        val2 = URLHelper.append_with_data("http://example.tld/", {"val": "%7B%7Baaaa%7D%7D"})
+
+        self.assertEqual(val1, "http://example.tld/?val={{aaaa}}")
+        self.assertEqual(val2, "http://example.tld/?val=%7B%7Baaaa%7D%7D")

test/test_queue.py

@@ -56,3 +56,14 @@ def test_hash_different_query_order(self):
         queue.add_request(Request("https://www.example.ltd?a=a&b=b&c=c"))
 
         self.assertEqual(queue.count_total, 1)
+
+
+    def test_hash_different_encoded_and_decoded_values(self):
+        """Ensure encoded and decoded values have a different hash."""
+
+        queue = Queue(Options())
+
+        queue.add_request(Request("http://example.ltd?val={{aaaa}}"))
+        queue.add_request(Request("http://example.ltd?val=%7B%7Baaaa%7D%7D"))
+
+        self.assertEqual(queue.count_total, 2)