add --disable-plugin-autoload (#13253)

* add --disable-plugin-autoload

* update comments in test, don't check __spec__ on pypy (????), add changelog

* pemdas except not

* Apply suggestions from code review

Co-authored-by: Florian Bruhin <[email protected]>

* add parens

* Simplify plugin name in test_installed_plugin_rewrite

* Apply suggestions from code review

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: Florian Bruhin <[email protected]>
Co-authored-by: Bruno Oliveira <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 4 people

changelog/13253.feature.rst

@@ -0,0 +1 @@
+New flag: :ref:`--disable-plugin-autoload <disable_plugin_autoload>` which works as an alternative to :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD` when setting environment variables is inconvenient; and allows setting it in config files with :confval:`addopts`.

doc/en/how-to/plugins.rst

@@ -133,4 +133,29 @@ CI server), you can set ``PYTEST_ADDOPTS`` environment variable to
 
 See :ref:`findpluginname` for how to obtain the name of a plugin.
 
-.. _`builtin plugins`:
+.. _`disable_plugin_autoload`:
+
+Disabling plugins from autoloading
+----------------------------------
+
+If you want to disable plugins from loading automatically, instead of requiring you to
+manually specify each plugin with ``-p`` or :envvar:`PYTEST_PLUGINS`, you can use ``--disable-plugin-autoload`` or :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD`.
+
+.. code-block:: bash
+
+   export PYTEST_DISABLE_PLUGIN_AUTOLOAD=1
+   export PYTEST_PLUGINS=NAME,NAME2
+   pytest
+
+.. code-block:: bash
+
+   pytest --disable-plugin-autoload -p NAME,NAME2
+
+.. code-block:: ini
+
+    [pytest]
+    addopts = --disable-plugin-autoload -p NAME,NAME2
+
+.. versionadded:: 8.4
+
+   The ``--disable-plugin-autoload`` command-line flag.

doc/en/reference/reference.rst

@@ -1181,8 +1181,9 @@ as discussed in :ref:`temporary directory location and retention`.
 .. envvar:: PYTEST_DISABLE_PLUGIN_AUTOLOAD
 
 When set, disables plugin auto-loading through :std:doc:`entry point packaging
-metadata <packaging:guides/creating-and-discovering-plugins>`. Only explicitly
-specified plugins will be loaded.
+metadata <packaging:guides/creating-and-discovering-plugins>`. Only plugins
+explicitly specified in :envvar:`PYTEST_PLUGINS` or with ``-p`` will be loaded.
+See also :ref:`--disable-plugin-autoload <disable_plugin_autoload>`.
 
 .. envvar:: PYTEST_PLUGINS
 
@@ -1192,6 +1193,8 @@ Contains comma-separated list of modules that should be loaded as plugins:
 
     export PYTEST_PLUGINS=mymodule.plugin,xdist
 
+See also ``-p``.
+
 .. envvar:: PYTEST_THEME
 
 Sets a `pygment style <https://pygments.org/docs/styles/>`_ to use for the code output.

src/_pytest/config/__init__.py

@@ -70,6 +70,7 @@
 
 
 if TYPE_CHECKING:
+    from _pytest.assertions.rewrite import AssertionRewritingHook
     from _pytest.cacheprovider import Cache
     from _pytest.terminal import TerminalReporter
 
@@ -1271,6 +1272,10 @@ def _consider_importhook(self, args: Sequence[str]) -> None:
         """
         ns, unknown_args = self._parser.parse_known_and_unknown_args(args)
         mode = getattr(ns, "assertmode", "plain")
+
+        disable_autoload = getattr(ns, "disable_plugin_autoload", False) or bool(
+            os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
+        )
         if mode == "rewrite":
             import _pytest.assertion
 
@@ -1279,16 +1284,18 @@ def _consider_importhook(self, args: Sequence[str]) -> None:
             except SystemError:
                 mode = "plain"
             else:
-                self._mark_plugins_for_rewrite(hook)
+                self._mark_plugins_for_rewrite(hook, disable_autoload)
         self._warn_about_missing_assertion(mode)
 
-    def _mark_plugins_for_rewrite(self, hook) -> None:
+    def _mark_plugins_for_rewrite(
+        self, hook: AssertionRewritingHook, disable_autoload: bool
+    ) -> None:
         """Given an importhook, mark for rewrite any top-level
         modules or packages in the distribution package for
         all pytest plugins."""
         self.pluginmanager.rewrite_hook = hook
 
-        if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
+        if disable_autoload:
             # We don't autoload from distribution package entry points,
             # no need to continue.
             return
@@ -1393,10 +1400,15 @@ def _preparse(self, args: list[str], addopts: bool = True) -> None:
         self._consider_importhook(args)
         self._configure_python_path()
         self.pluginmanager.consider_preparse(args, exclude_only=False)
-        if not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"):
-            # Don't autoload from distribution package entry point. Only
-            # explicitly specified plugins are going to be loaded.
+        if (
+            not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
+            and not self.known_args_namespace.disable_plugin_autoload
+        ):
+            # Autoloading from distribution package entry point has
+            # not been disabled.
             self.pluginmanager.load_setuptools_entrypoints("pytest11")
+        # Otherwise only plugins explicitly specified in PYTEST_PLUGINS
+        # are going to be loaded.
         self.pluginmanager.consider_env()
 
         self.known_args_namespace = self._parser.parse_known_args(
@@ -1419,7 +1431,7 @@ def _preparse(self, args: list[str], addopts: bool = True) -> None:
         except ConftestImportFailure as e:
             if self.known_args_namespace.help or self.known_args_namespace.version:
                 # we don't want to prevent --help/--version to work
-                # so just let is pass and print a warning at the end
+                # so just let it pass and print a warning at the end
                 self.issue_config_time_warning(
                     PytestConfigWarning(f"could not load initial conftests: {e.path}"),
                     stacklevel=2,

src/_pytest/helpconfig.py

@@ -70,7 +70,14 @@ def pytest_addoption(parser: Parser) -> None:
         metavar="name",
         help="Early-load given plugin module name or entry point (multi-allowed). "
         "To avoid loading of plugins, use the `no:` prefix, e.g. "
-        "`no:doctest`.",
+        "`no:doctest`. See also --disable-plugin-autoload.",
+    )
+    group.addoption(
+        "--disable-plugin-autoload",
+        action="store_true",
+        default=False,
+        help="Disable plugin auto-loading through entry point packaging metadata. "
+        "Only plugins explicitly specified in -p or env var PYTEST_PLUGINS will be loaded.",
     )
     group.addoption(
         "--traceconfig",

testing/test_assertion.py

@@ -218,10 +218,36 @@ def test_foo(pytestconfig):
         assert result.ret == 0
 
     @pytest.mark.parametrize("mode", ["plain", "rewrite"])
+    @pytest.mark.parametrize("disable_plugin_autoload", ["env_var", "cli", ""])
+    @pytest.mark.parametrize("explicit_specify", ["env_var", "cli", ""])
     def test_installed_plugin_rewrite(
-        self, pytester: Pytester, mode, monkeypatch
+        self,
+        pytester: Pytester,
+        mode: str,
+        monkeypatch: pytest.MonkeyPatch,
+        disable_plugin_autoload: str,
+        explicit_specify: str,
     ) -> None:
-        monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
+        args = ["mainwrapper.py", "-s", f"--assert={mode}"]
+        if disable_plugin_autoload == "env_var":
+            monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
+        elif disable_plugin_autoload == "cli":
+            monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
+            args.append("--disable-plugin-autoload")
+        else:
+            assert disable_plugin_autoload == ""
+            monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False)
+
+        name = "spamplugin"
+
+        if explicit_specify == "env_var":
+            monkeypatch.setenv("PYTEST_PLUGINS", name)
+        elif explicit_specify == "cli":
+            args.append("-p")
+            args.append(name)
+        else:
+            assert explicit_specify == ""
+
         # Make sure the hook is installed early enough so that plugins
         # installed via distribution package are rewritten.
         pytester.mkdir("hampkg")
@@ -250,7 +276,7 @@ def check(values, value):
             import pytest
 
             class DummyEntryPoint(object):
-                name = 'spam'
+                name = 'spamplugin'
                 module_name = 'spam.py'
                 group = 'pytest11'
 
@@ -275,20 +301,29 @@ def test(check_first):
                 check_first([10, 30], 30)
 
             def test2(check_first2):
-                check_first([10, 30], 30)
+                check_first2([10, 30], 30)
             """,
         }
         pytester.makepyfile(**contents)
-        result = pytester.run(
-            sys.executable, "mainwrapper.py", "-s", f"--assert={mode}"
-        )
+        result = pytester.run(sys.executable, *args)
         if mode == "plain":
             expected = "E       AssertionError"
         elif mode == "rewrite":
             expected = "*assert 10 == 30*"
         else:
             assert 0
-        result.stdout.fnmatch_lines([expected])
+
+        if not disable_plugin_autoload or explicit_specify:
+            result.assert_outcomes(failed=2)
+            result.stdout.fnmatch_lines([expected, expected])
+        else:
+            result.assert_outcomes(errors=2)
+            result.stdout.fnmatch_lines(
+                [
+                    "E       fixture 'check_first' not found",
+                    "E       fixture 'check_first2' not found",
+                ]
+            )
 
     def test_rewrite_ast(self, pytester: Pytester) -> None:
         pytester.mkdir("pkg")

testing/test_config.py

@@ -6,6 +6,7 @@
 import importlib.metadata
 import os
 from pathlib import Path
+import platform
 import re
 import sys
 import textwrap
@@ -1314,14 +1315,13 @@ def distributions():
         )
 
 
-@pytest.mark.parametrize(
-    "parse_args,should_load", [(("-p", "mytestplugin"), True), ((), False)]
-)
+@pytest.mark.parametrize("disable_plugin_method", ["env_var", "flag", ""])
+@pytest.mark.parametrize("enable_plugin_method", ["env_var", "flag", ""])
 def test_disable_plugin_autoload(
     pytester: Pytester,
     monkeypatch: MonkeyPatch,
-    parse_args: tuple[str, str] | tuple[()],
-    should_load: bool,
+    enable_plugin_method: str,
+    disable_plugin_method: str,
 ) -> None:
     class DummyEntryPoint:
         project_name = name = "mytestplugin"
@@ -1342,23 +1342,60 @@ class PseudoPlugin:
         attrs_used = []
 
         def __getattr__(self, name):
-            assert name == "__loader__"
+            assert name in ("__loader__", "__spec__")
             self.attrs_used.append(name)
             return object()
 
     def distributions():
         return (Distribution(),)
 
-    monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
+    parse_args: list[str] = []
+
+    if disable_plugin_method == "env_var":
+        monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1")
+    elif disable_plugin_method == "flag":
+        monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
+        parse_args.append("--disable-plugin-autoload")
+    else:
+        assert disable_plugin_method == ""
+        monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD")
+
+    if enable_plugin_method == "env_var":
+        monkeypatch.setenv("PYTEST_PLUGINS", "mytestplugin")
+    elif enable_plugin_method == "flag":
+        parse_args.extend(["-p", "mytestplugin"])
+    else:
+        assert enable_plugin_method == ""
+
     monkeypatch.setattr(importlib.metadata, "distributions", distributions)
     monkeypatch.setitem(sys.modules, "mytestplugin", PseudoPlugin())
     config = pytester.parseconfig(*parse_args)
+
     has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None
-    assert has_loaded == should_load
-    if should_load:
-        assert PseudoPlugin.attrs_used == ["__loader__"]
-    else:
-        assert PseudoPlugin.attrs_used == []
+    # it should load if it's enabled, or we haven't disabled autoloading
+    assert has_loaded == (bool(enable_plugin_method) or not disable_plugin_method)
+
+    # The reason for the discrepancy between 'has_loaded' and __loader__ being accessed
+    # appears to be the monkeypatching of importlib.metadata.distributions; where
+    # files being empty means that _mark_plugins_for_rewrite doesn't find the plugin.
+    # But enable_method==flag ends up in mark_rewrite being called and __loader__
+    # being accessed.
+    assert ("__loader__" in PseudoPlugin.attrs_used) == (
+        has_loaded
+        and not (enable_plugin_method in ("env_var", "") and not disable_plugin_method)
+    )
+
+    # __spec__ is accessed in AssertionRewritingHook.exec_module, which would be
+    # eventually called if we did a full pytest run; but it's only accessed with
+    # enable_plugin_method=="env_var" because that will early-load it.
+    # Except when autoloads aren't disabled, in which case PytestPluginManager.import_plugin
+    # bails out before importing it.. because it knows it'll be loaded later?
+    # The above seems a bit weird, but I *think* it's true.
+    if platform.python_implementation() != "PyPy":
+        assert ("__spec__" in PseudoPlugin.attrs_used) == bool(
+            enable_plugin_method == "env_var" and disable_plugin_method
+        )
+    # __spec__ is present when testing locally on pypy, but not in CI ????
 
 
 def test_plugin_loading_order(pytester: Pytester) -> None: