python: add strict_parametrization_ids option

Fix #13737.

Author: gomri15

AUTHORS

@@ -346,6 +346,7 @@ Oliver Bestwalter
 Olivier Grisel
 Omar Kohl
 Omer Hadari
+Omri Golan
 Ondřej Súkup
 Oscar Benjamin
 Parth Patel

changelog/13737.feature.rst

@@ -0,0 +1,4 @@
+Added the :confval:`strict_parametrization_ids` configuration option.
+
+When set, pytest emits an error if it detects non-unique parameter set IDs,
+rather than automatically making the IDs unique by adding `0`, `1`, ... to them.

doc/en/reference/reference.rst

@@ -2082,6 +2082,44 @@ passed multiple times. The expected format is ``name=value``. For example::
         [pytest]
         xfail_strict = True
 
+.. confval:: strict_parametrization_ids
+
+    If set, pytest emits an error if it detects non-unique parameter set IDs.
+
+    If not set (the default), pytest automatically handles this by adding `0`, `1`, ... to duplicate IDs,
+    making them unique.
+
+    .. code-block:: ini
+
+          [pytest]
+          strict_parametrization_ids = True
+
+    For example,
+
+    .. code-block:: python
+
+        import pytest
+
+
+        @pytest.mark.parametrize("letter", ["a", "a"])
+        def test_letter_is_ascii(letter):
+            assert letter.isascii()
+
+    will emit an error because both cases (parameter sets) have the same auto-generated ID "a".
+
+    To fix the error, if you decide to keep the duplicates, explicitly assign unique IDs:
+
+    .. code-block:: python
+
+        import pytest
+
+
+        @pytest.mark.parametrize("letter", ["a", "a"], ids=["a0", "a1"])
+        def test_letter_is_ascii(letter):
+            assert letter.isascii()
+
+    See :func:`parametrize <pytest.Metafunc.parametrize>` and :func:`pytest.param` for other ways to set IDs.
+
 
 .. _`command-line-flags`:
 

src/_pytest/python.py

@@ -21,6 +21,7 @@
 import os
 from pathlib import Path
 import re
+import textwrap
 import types
 from typing import Any
 from typing import final
@@ -107,6 +108,12 @@ def pytest_addoption(parser: Parser) -> None:
         help="Disable string escape non-ASCII characters, might cause unwanted "
         "side effects(use at your own risk)",
     )
+    parser.addini(
+        "strict_parametrization_ids",
+        type="bool",
+        default=False,
+        help="Emit an error if non-unique parameter set IDs are detected",
+    )
 
 
 def pytest_generate_tests(metafunc: Metafunc) -> None:
@@ -878,8 +885,8 @@ class IdMaker:
     # Optionally, explicit IDs for ParameterSets by index.
     ids: Sequence[object | None] | None
     # Optionally, the pytest config.
-    # Used for controlling ASCII escaping, and for calling the
-    # :hook:`pytest_make_parametrize_id` hook.
+    # Used for controlling ASCII escaping, determining parametrization ID
+    # strictness, and for calling the :hook:`pytest_make_parametrize_id` hook.
     config: Config | None
     # Optionally, the ID of the node being parametrized.
     # Used only for clearer error messages.
@@ -892,6 +899,9 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]:
         """Make a unique identifier for each ParameterSet, that may be used to
         identify the parametrization in a node ID.
 
+        If strict_parametrization_ids is enabled, and duplicates are detected,
+        raises CollectError. Otherwise makes the IDs unique as follows:
+
         Format is <prm_1_token>-...-<prm_n_token>[counter], where prm_x_token is
         - user-provided id, if given
         - else an id derived from the value, applicable for certain types
@@ -904,6 +914,33 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]:
         if len(resolved_ids) != len(set(resolved_ids)):
             # Record the number of occurrences of each ID.
             id_counts = Counter(resolved_ids)
+
+            if self._strict_parametrization_ids_enabled():
+                parameters = ", ".join(self.argnames)
+                parametersets = ", ".join(
+                    [saferepr(list(param.values)) for param in self.parametersets]
+                )
+                ids = ", ".join(
+                    id if id is not HIDDEN_PARAM else "<hidden>" for id in resolved_ids
+                )
+                duplicates = ", ".join(
+                    id if id is not HIDDEN_PARAM else "<hidden>"
+                    for id, count in id_counts.items()
+                    if count > 1
+                )
+                msg = textwrap.dedent(f"""
+                    Duplicate parametrization IDs detected, but strict_parametrization_ids is set.
+
+                    Test name:      {self.nodeid}
+                    Parameters:     {parameters}
+                    Parameter sets: {parametersets}
+                    IDs:            {ids}
+                    Duplicates:     {duplicates}
+
+                    You can fix this problem using `@pytest.mark.parametrize(..., ids=...)` or `pytest.param(..., id=...)`.
+                """).strip()  # noqa: E501
+                raise nodes.Collector.CollectError(msg)
+
             # Map the ID to its next suffix.
             id_suffixes: dict[str, int] = defaultdict(int)
             # Suffix non-unique IDs to make them unique.
@@ -925,6 +962,11 @@ def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]:
         )
         return resolved_ids
 
+    def _strict_parametrization_ids_enabled(self) -> bool:
+        if self.config:
+            return bool(self.config.getini("strict_parametrization_ids"))
+        return False
+
     def _resolve_ids(self) -> Iterable[str | _HiddenParam]:
         """Resolve IDs for all ParameterSets (may contain duplicates)."""
         for idx, parameterset in enumerate(self.parametersets):

testing/test_collection.py

@@ -1,6 +1,7 @@
 # mypy: allow-untyped-defs
 from __future__ import annotations
 
+from collections.abc import Sequence
 import os
 from pathlib import Path
 from pathlib import PurePath
@@ -2702,3 +2703,94 @@ def test_1(): pass
             ],
             consecutive=True,
         )
+
+
+@pytest.mark.parametrize(
+    ["x_y", "expected_duplicates"],
+    [
+        (
+            [(1, 1), (1, 1)],
+            ["1-1"],
+        ),
+        (
+            [(1, 1), (1, 2), (1, 1)],
+            ["1-1"],
+        ),
+        (
+            [(1, 1), (2, 2), (1, 1)],
+            ["1-1"],
+        ),
+        (
+            [(1, 1), (2, 2), (1, 2), (2, 1), (1, 1), (2, 1)],
+            ["1-1", "2-1"],
+        ),
+    ],
+)
+def test_strict_parametrization_ids(
+    pytester: Pytester,
+    x_y: Sequence[tuple[int, int]],
+    expected_duplicates: Sequence[str],
+) -> None:
+    pytester.makeini(
+        """
+        [pytest]
+        strict_parametrization_ids = true
+        """
+    )
+    pytester.makepyfile(
+        f"""
+        import pytest
+
+        @pytest.mark.parametrize(["x", "y"], {x_y})
+        def test1(x, y):
+            pass
+        """
+    )
+
+    result = pytester.runpytest()
+
+    assert result.ret == ExitCode.INTERRUPTED
+    expected_parametersets = ", ".join(str(list(p)) for p in x_y)
+    expected_ids = ", ".join(f"{x}-{y}" for x, y in x_y)
+    result.stdout.fnmatch_lines(
+        [
+            "Duplicate parametrization IDs detected*",
+            "",
+            "Test name:      *::test1",
+            "Parameters:     x, y",
+            f"Parameter sets: {expected_parametersets}",
+            f"IDs:            {expected_ids}",
+            f"Duplicates:     {', '.join(expected_duplicates)}",
+            "",
+            "You can fix this problem using *",
+        ]
+    )
+
+
+def test_strict_parametrization_ids_with_hidden_param(pytester: Pytester) -> None:
+    pytester.makeini(
+        """
+        [pytest]
+        strict_parametrization_ids = true
+        """
+    )
+    pytester.makepyfile(
+        """
+        import pytest
+
+        @pytest.mark.parametrize(["x"], ["a", pytest.param("a", id=pytest.HIDDEN_PARAM), "a"])
+        def test1(x):
+            pass
+        """
+    )
+
+    result = pytester.runpytest()
+
+    assert result.ret == ExitCode.INTERRUPTED
+    result.stdout.fnmatch_lines(
+        [
+            "Duplicate parametrization IDs detected*",
+            "IDs:            a, <hidden>, a",
+            "Duplicates:     a",
+        ]
+    )