Allow extending lists with --override foo+=bar (#3088)

Author: stefanor

docs/changelog/3087.feature.rst

@@ -0,0 +1,3 @@
+``--override`` can now take options in the form of ``foo+=bar`` which
+will append ``bar`` to the end of an existing list/dict, rather than
+replacing it.

docs/config.rst

@@ -958,3 +958,31 @@ Other Substitutions
 
 * ``{}`` - replaced as ``os.pathsep``
 * ``{/}`` - replaced as ``os.sep``
+
+Overriding configuration from the command line
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can override options in the configuration file, from the command
+line.
+
+For example, given this config:
+
+.. code-block:: ini
+
+    [testenv]
+    deps = pytest
+    setenv =
+      foo=bar
+    commands = pytest tests
+
+You could enable ``ignore_errors`` by running::
+
+    tox --override testenv.ignore_errors=True
+
+You could add additional dependencies by running::
+
+    tox --override testenv.deps+=pytest-xdist,pytest-cov
+
+You could set additional environment variables by running::
+
+    tox --override testenv.setenv+=baz=quux

src/tox/config/loader/api.py

@@ -24,6 +24,12 @@ def __init__(self, value: str) -> None:
         if not equal:
             msg = f"override {value} has no = sign in it"
             raise ArgumentTypeError(msg)
+
+        self.append = False
+        if key.endswith("+"):  # key += value appends to a list
+            key = key[:-1]
+            self.append = True
+
         self.namespace, _, self.key = key.rpartition(".")
 
     def __repr__(self) -> str:
@@ -117,10 +123,25 @@ def load(  # noqa: PLR0913
         :param args: the config load arguments
         :return: the converted type
         """
-        if key in self.overrides:
-            return _STR_CONVERT.to(self.overrides[key].value, of_type, factory)
+        from tox.config.set_env import SetEnv
+
+        override = self.overrides.get(key)
+        if override and not override.append:
+            return _STR_CONVERT.to(override.value, of_type, factory)
         raw = self.load_raw(key, conf, args.env_name)
-        return self.build(key, of_type, factory, conf, raw, args)
+        converted = self.build(key, of_type, factory, conf, raw, args)
+        if override and override.append:
+            appends = _STR_CONVERT.to(override.value, of_type, factory)
+            if isinstance(converted, list) and isinstance(appends, list):
+                converted += appends
+            elif isinstance(converted, dict) and isinstance(appends, dict):
+                converted.update(appends)
+            elif isinstance(converted, SetEnv) and isinstance(appends, SetEnv):
+                converted.update(appends, override=True)
+            else:
+                msg = "Only able to append to lists and dicts"
+                raise ValueError(msg)
+        return converted
 
     def build(  # noqa: PLR0913
         self,

src/tox/config/set_env.py

@@ -95,10 +95,11 @@ def __iter__(self) -> Iterator[str]:
             self._raw.update(sub_raw)
             yield from sub_raw.keys()
 
-    def update(self, param: Mapping[str, str], *, override: bool = True) -> None:
-        for key, value in param.items():
+    def update(self, param: Mapping[str, str] | SetEnv, *, override: bool = True) -> None:
+        for key in param:
             # do not override something already set explicitly
             if override or (key not in self._raw and key not in self._materialized):
+                value = param.load(key) if isinstance(param, SetEnv) else param[key]
                 self._materialized[key] = value
                 self.changed = True
 

tests/config/loader/test_loader.py

@@ -28,6 +28,18 @@ def test_override_add(flag: str) -> None:
     assert value.key == "magic"
     assert value.value == "true"
     assert not value.namespace
+    assert value.append is False
+
+
+@pytest.mark.parametrize("flag", ["-x", "--override"])
+def test_override_append(flag: str) -> None:
+    parsed, _, __, ___, ____ = get_options(flag, "magic+=true")
+    assert len(parsed.override) == 1
+    value = parsed.override[0]
+    assert value.key == "magic"
+    assert value.value == "true"
+    assert not value.namespace
+    assert value.append is True
 
 
 def test_override_equals() -> None:

tests/config/test_main.py

@@ -2,7 +2,9 @@
 
 import os
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, List
+
+import pytest
 
 from tox.config.loader.api import Override
 from tox.config.loader.memory import MemoryLoader
@@ -64,6 +66,38 @@ def test_config_override_wins_memory_loader(tox_ini_conf: ToxIniCreator) -> None
     assert conf["c"] == "ok"
 
 
+def test_config_override_appends_to_list(tox_ini_conf: ToxIniCreator) -> None:
+    example = """
+    [testenv]
+    passenv = foo
+    """
+    conf = tox_ini_conf(example, override=[Override("testenv.passenv+=bar")]).get_env("testenv")
+    conf.add_config("passenv", of_type=List[str], default=[], desc="desc")
+    assert conf["passenv"] == ["foo", "bar"]
+
+
+def test_config_override_appends_to_setenv(tox_ini_conf: ToxIniCreator) -> None:
+    example = """
+    [testenv]
+    setenv =
+      foo = bar
+    """
+    conf = tox_ini_conf(example, override=[Override("testenv.setenv+=baz=quux")]).get_env("testenv")
+    assert conf["setenv"].load("foo") == "bar"
+    assert conf["setenv"].load("baz") == "quux"
+
+
+def test_config_override_cannot_append(tox_ini_conf: ToxIniCreator) -> None:
+    example = """
+    [testenv]
+    foo = 1
+    """
+    conf = tox_ini_conf(example, override=[Override("testenv.foo+=2")]).get_env("testenv")
+    conf.add_config("foo", of_type=int, default=0, desc="desc")
+    with pytest.raises(ValueError, match="Only able to append to lists and dicts"):
+        conf["foo"]
+
+
 def test_args_are_paths_when_disabled(tox_project: ToxProjectCreator) -> None:
     ini = "[testenv]\npackage=skip\ncommands={posargs}\nargs_are_paths=False"
     project = tox_project({"tox.ini": ini, "w": {"a.txt": "a"}})

tests/config/test_set_env.py

@@ -34,6 +34,22 @@ def test_set_env_explicit() -> None:
     assert "MISS" not in set_env
 
 
+def test_set_env_merge() -> None:
+    a = SetEnv("\nA=1\nB = 2\nC= 3\nD= 4", "py", "py", Path())
+    b = SetEnv("\nA=2\nE = 5", "py", "py", Path())
+    a.update(b, override=False)
+
+    keys = list(a)
+    assert keys == ["E", "A", "B", "C", "D"]
+    values = [a.load(k) for k in keys]
+    assert values == ["5", "1", "2", "3", "4"]
+
+    a.update(b, override=True)
+
+    values = [a.load(k) for k in keys]
+    assert values == ["5", "2", "2", "3", "4"]
+
+
 def test_set_env_bad_line() -> None:
     with pytest.raises(ValueError, match="A"):
         SetEnv("A", "py", "py", Path())