Add or replace URL parameters without removing duplicate values in query strings (#126)

Author: rennerocha

tests/test_url.py

@@ -321,6 +321,12 @@ def test_add_or_replace_parameter(self):
         self.assertEqual(add_or_replace_parameter(url, 'pageurl', 'test'),
                          'http://example.com/?version=1&pageurl=test&param2=value2')
 
+        url = 'http://domain/test?arg1=v1&arg2=v2&arg1=v3'
+        self.assertEqual(add_or_replace_parameter(url, 'arg4', 'v4'),
+                         'http://domain/test?arg1=v1&arg2=v2&arg1=v3&arg4=v4')
+        self.assertEqual(add_or_replace_parameter(url, 'arg1', 'v3'),
+                         'http://domain/test?arg1=v3&arg2=v2')
+
     def test_add_or_replace_parameters(self):
         url = 'http://domain/test'
         self.assertEqual(add_or_replace_parameters(url, {'arg': 'v'}),
@@ -330,6 +336,17 @@ def test_add_or_replace_parameters(self):
                          'http://domain/test?arg1=v1&arg2=v2&arg3=v3&arg4=v4')
         self.assertEqual(add_or_replace_parameters(url, {'arg4': 'v4', 'arg3': 'v3new'}),
                          'http://domain/test?arg1=v1&arg2=v2&arg3=v3new&arg4=v4')
+        url = 'http://domain/test?arg1=v1&arg2=v2&arg1=v3'
+        self.assertEqual(add_or_replace_parameters(url, {'arg4': 'v4'}),
+                         'http://domain/test?arg1=v1&arg2=v2&arg1=v3&arg4=v4')
+        self.assertEqual(add_or_replace_parameters(url, {'arg1': 'v3'}),
+                         'http://domain/test?arg1=v3&arg2=v2')
+
+    def test_add_or_replace_parameters_does_not_change_input_param(self):
+        url = 'http://domain/test?arg=original'
+        input_param = {'arg': 'value'}
+        new_url = add_or_replace_parameters(url, input_param)  # noqa
+        self.assertEqual(input_param, {'arg': 'value'})
 
     def test_url_query_cleaner(self):
         self.assertEqual('product.html',

w3lib/url.py

@@ -9,7 +9,7 @@
 import posixpath
 import warnings
 import string
-from collections import namedtuple, OrderedDict
+from collections import namedtuple
 import six
 from six.moves.urllib.parse import (urljoin, urlsplit, urlunsplit,
                                     urldefrag, urlencode, urlparse,
@@ -208,13 +208,21 @@ def url_query_cleaner(url, parameterlist=(), sep='&', kvsep='=', remove=False, u
         url += '#' + fragment
     return url
 
-
 def _add_or_replace_parameters(url, params):
     parsed = urlsplit(url)
-    args = parse_qsl(parsed.query, keep_blank_values=True)
-
-    new_args = OrderedDict(args)
-    new_args.update(params)
+    current_args = parse_qsl(parsed.query, keep_blank_values=True)
+
+    new_args = []
+    seen_params = set()
+    for name, value in current_args:
+        if name not in params:
+            new_args.append((name, value))
+        elif name not in seen_params:
+            new_args.append((name, params[name]))
+            seen_params.add(name)
+
+    not_modified_args = [(name, value) for name, value in params.items() if name not in seen_params]
+    new_args += not_modified_args
 
     query = urlencode(new_args)
     return urlunsplit(parsed._replace(query=query))