pytester: error on non-str pytester.plugins in subprocess mode instead of silently ignoring

In subprocess mode, adding a non-str plugin object to `pytester.plugins`
can't work. Previously, such plugins would just be silently ignored.
Silently ignoring an explicit setup doesn't seem right. Error instead.

Author: bluetech

changelog/13522.bugfix.rst

@@ -1 +1,5 @@
 Fixed :fixture:`pytester` in subprocess mode ignored all :attr`pytester.plugins <pytest.Pytester.plugins>` except the first.
+
+Fixed :fixture:`pytester` in subprocess mode silently ignored non-str :attr:`pytester.plugins <pytest.Pytester.plugins>`.
+Now it errors instead.
+If you are affected by this, specify the plugin by name, or switch the affected tests to use :func:`pytester.runpytest_inprocess <pytest.Pytester.runpytest_inprocess>` explicitly instead.

src/_pytest/pytester.py

@@ -682,9 +682,11 @@ def __init__(
         self._name = name
         self._path: Path = tmp_path_factory.mktemp(name, numbered=True)
         #: A list of plugins to use with :py:meth:`parseconfig` and
-        #: :py:meth:`runpytest`.  Initially this is an empty list but plugins can
-        #: be added to the list.  The type of items to add to the list depends on
-        #: the method using them so refer to them for details.
+        #: :py:meth:`runpytest`. Initially this is an empty list but plugins can
+        #: be added to the list.
+        #:
+        #: When running in subprocess mode, specify plugins by name (str) - adding
+        #: plugin objects directly is not supported.
         self.plugins: list[str | _PluggyPlugin] = []
         self._sys_path_snapshot = SysPathsSnapshot()
         self._sys_modules_snapshot = self.__take_sys_modules_snapshot()
@@ -1494,8 +1496,12 @@ def runpytest_subprocess(
         p = make_numbered_dir(root=self.path, prefix="runpytest-", mode=0o700)
         args = (f"--basetemp={p}", *args)
         for plugin in self.plugins:
-            if isinstance(plugin, str):
-                args = ("-p", plugin, *args)
+            if not isinstance(plugin, str):
+                raise ValueError(
+                    f"Specifying plugins as objects is not supported in pytester subprocess mode; "
+                    f"specify by name instead: {plugin}"
+                )
+            args = ("-p", plugin, *args)
         args = self._getpytestargs() + args
         return self.run(*args, timeout=timeout)
 

testing/test_pytester.py

@@ -834,3 +834,25 @@ def test_two():
     result.assert_outcomes(passed=1, deselected=1)
     # If deselected is not passed, it is not checked at all.
     result.assert_outcomes(passed=1)
+
+
+def test_pytester_subprocess_with_string_plugins(pytester: Pytester) -> None:
+    """Test that pytester.runpytest_subprocess is OK with named (string)
+    `.plugins`."""
+    pytester.plugins = ["pytester"]
+
+    result = pytester.runpytest_subprocess()
+    assert result.ret == ExitCode.NO_TESTS_COLLECTED
+
+
+def test_pytester_subprocess_with_non_string_plugins(pytester: Pytester) -> None:
+    """Test that pytester.runpytest_subprocess fails with a proper error given
+    non-string `.plugins`."""
+
+    class MyPlugin:
+        pass
+
+    pytester.plugins = [MyPlugin()]
+
+    with pytest.raises(ValueError, match="plugins as objects is not supported"):
+        pytester.runpytest_subprocess()