diff --git a/changelog/13737.feature.rst b/changelog/13737.feature.rst new file mode 100644 index 00000000000..deac2f65890 --- /dev/null +++ b/changelog/13737.feature.rst @@ -0,0 +1,4 @@ +Added the ``--require-unique-paramset-ids`` flag to pytest. + +When passed, this flag causes pytest to raise an exception upon detection of non-unique parameter set IDs, +rather than attempting to generate them automatically. diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index 443530a2006..140c6a093a5 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -2063,6 +2063,30 @@ passed multiple times. The expected format is ``name=value``. For example:: [pytest] xfail_strict = True +.. confval:: require_unique_paramset_ids + + When passed, this flag causes pytest to raise an exception upon detection of non-unique parameter set IDs, + rather than attempting to generate them automatically. + + Can be overridden by `--require-unique-paramset-ids`. + + .. code-block:: ini + + [pytest] + require_unique_paramset_ids = True + + .. code-block:: python + + import pytest + + + @pytest.mark.parametrize("x", [1, 2], ids=["a", "a"]) + def test_example(x): + assert x in (1, 2) + + will raise an exception due to the duplicate IDs "a". + when normal pytest behavior would be to handle this by generating unique IDs like "a-0", "a-1". + .. _`command-line-flags`: @@ -2220,6 +2244,11 @@ All the command-line flags can be obtained by running ``pytest --help``:: --doctest-continue-on-failure For a given doctest, continue to run after the first failure + --require-unique-paramset-ids + If pytest collects test ids with non-unique names, raise an + error rather than handling it. + Useful if you collect in one process, + and then execute tests in independent workers. test session debugging and configuration: -c, --config-file FILE diff --git a/src/_pytest/main.py b/src/_pytest/main.py index b1eb22f1f61..1eaeec1cb16 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -90,6 +90,21 @@ def pytest_addoption(parser: Parser) -> None: action="store_true", help="(Deprecated) alias to --strict-markers", ) + group.addoption( + "--require-unique-paramset-ids", + action="store_true", + default=False, + help="When passed, this flag causes pytest to raise an exception upon detection of non-unique parameter set IDs" + "rather than attempting to generate them automatically.", + ) + + parser.addini( + "require_unique_paramset_ids", + type="bool", + default=False, + help="When passed, this flag causes pytest to raise an exception upon detection of non-unique parameter set IDs" + "rather than attempting to generate them automatically.", + ) group = parser.getgroup("pytest-warnings") group.addoption( diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 3f9da026799..1bdb693ab01 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -50,6 +50,7 @@ from _pytest.config import Config from _pytest.config import hookimpl from _pytest.config.argparsing import Parser +from _pytest.config.exceptions import UsageError from _pytest.deprecated import check_ispytest from _pytest.fixtures import FixtureDef from _pytest.fixtures import FixtureRequest @@ -902,6 +903,23 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: resolved_ids = list(self._resolve_ids()) # All IDs must be unique! if len(resolved_ids) != len(set(resolved_ids)): + if self._require_unique_ids_enabled(): + duplicate_indexs = defaultdict(list) + for i, val in enumerate(resolved_ids): + duplicate_indexs[val].append(i) + + # Keep only duplicates + duplicates = {k: v for k, v in duplicate_indexs.items() if len(v) > 1} + raise UsageError(f""" + Because --require-unique-paramset-ids given, pytest won't + attempt to generate unique IDs for parameter sets. + argument values: {self.parametersets} + argument names: {self.argnames} + function name: {self.func_name} + test name: {self.nodeid} + resolved (with non-unique) IDs: {resolved_ids} + duplicates: {duplicates} + """) # Record the number of occurrences of each ID. id_counts = Counter(resolved_ids) # Map the ID to its next suffix. @@ -925,6 +943,14 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: ) return resolved_ids + def _require_unique_ids_enabled(self) -> bool: + if self.config: + cli_value = self.config.getoption("require_unique_paramset_ids") + if cli_value: + return bool(cli_value) + return bool(self.config.getini("require_unique_paramset_ids")) + return False + def _resolve_ids(self) -> Iterable[str | _HiddenParam]: """Resolve IDs for all ParameterSets (may contain duplicates).""" for idx, parameterset in enumerate(self.parametersets): diff --git a/testing/test_collection.py b/testing/test_collection.py index 0dcf2990ed3..514bbb2feb1 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -5,6 +5,7 @@ from pathlib import Path from pathlib import PurePath import pprint +import re import shutil import sys import tempfile @@ -2031,3 +2032,73 @@ def test_namespace_packages(pytester: Pytester, import_mode: str): " ", ] ) + + +class TestRequireUniqueParamsetIds: + CASES = [ + ("[(1, 1), (1, 1)]", {"1-1": [0, 1]}), + ("[(1, 1), (1, 2), (1, 1)]", {"1-1": [0, 2]}), + ("[(1, 1), (2, 2), (1, 1)]", {"1-1": [0, 2]}), + ("[(1, 1), (2, 2), (1, 2), (2, 1), (1, 1)]", {"1-1": [0, 4]}), + ] + + @staticmethod + def _make_testfile(pytester: Pytester, parametrize_args: str) -> None: + pytester.makepyfile( + f""" + import pytest + + @pytest.mark.parametrize('y, x', {parametrize_args}) + def test1(y, x): + pass + """ + ) + + @staticmethod + def _fnmatch_escape_repr(obj) -> str: + return re.sub(r"[*?[\]]", (lambda m: f"[{m.group()}]"), repr(obj)) + + def _assert_duplicate_msg(self, result, expected_indices): + # Collection errors usually go to stdout; fall back to stderr just in case. + stream = result.stdout + stream.fnmatch_lines( + [ + "E*Because --require-unique-paramset-ids given, pytest won't", + "E*attempt to generate unique IDs for parameter sets.", + "E*argument names: [[]'y', 'x'[]]", + "E*function name: test1", + "E*test name: *::test1", + f"E*duplicates: {self._fnmatch_escape_repr(expected_indices)}", + ] + ) + assert result.ret != 0 + + @pytest.mark.parametrize("parametrize_args, expected_indices", CASES) + def test_cli_enables(self, pytester: Pytester, parametrize_args, expected_indices): + self._make_testfile(pytester, parametrize_args) + result = pytester.runpytest("--require-unique-paramset-ids") + self._assert_duplicate_msg(result, expected_indices) + + @pytest.mark.parametrize("parametrize_args, expected_indices", CASES) + def test_ini_enables(self, pytester: Pytester, parametrize_args, expected_indices): + pytester.makeini( + """ + [pytest] + require_unique_paramset_ids = true + """ + ) + self._make_testfile(pytester, parametrize_args) + result = pytester.runpytest() + self._assert_duplicate_msg(result, expected_indices) + + def test_cli_overrides_ini_false(self, pytester: Pytester): + """CLI True should override ini False.""" + pytester.makeini( + """ + [pytest] + require_unique_paramset_ids = false + """ + ) + self._make_testfile(pytester, "[(1,1), (1,1)]") + result = pytester.runpytest("--require-unique-paramset-ids") + self._assert_duplicate_msg(result, {"1-1": [0, 1]})