From 194fe042ecf439c823c73b29da776ae0f0a748c9 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 10:13:05 +0200 Subject: [PATCH 01/14] config: replace post-parse usage of `known_args_namespace` with `option` After parse, we should use the final `option` Namespace. --- src/_pytest/config/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 36fe4171582..6e8d6914c0b 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1539,12 +1539,12 @@ def parse(self, args: list[str], addopts: bool = True) -> None: self._preparse(args, addopts=addopts) self._parser.after_preparse = True # type: ignore try: - parsed = self._parser.parse(args, namespace=self.option) + self._parser.parse(args, namespace=self.option) except PrintHelp: return self.args, self.args_source = self._decide_args( - args=getattr(parsed, FILE_OR_DIR), - pyargs=self.known_args_namespace.pyargs, + args=getattr(self.option, FILE_OR_DIR), + pyargs=self.option.pyargs, testpaths=self.getini("testpaths"), invocation_dir=self.invocation_params.dir, rootpath=self.rootpath, From c8b03ba58d5b2d4a5355f92c4b8518216133458d Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 10:29:49 +0200 Subject: [PATCH 02/14] config: replace confusing use of `parse_known_and_unknown_args` in `_preparse` The way our parser works, "unknown args" or only unknown *flags*. But `determine_setup` is not interested in those. --- src/_pytest/config/__init__.py | 8 +++----- src/_pytest/config/argparsing.py | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 6e8d6914c0b..46042363582 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1253,7 +1253,7 @@ def _consider_importhook(self, args: Sequence[str]) -> None: and find all the installed plugins to mark them for rewriting by the importhook. """ - ns, _unknown_args = self._parser.parse_known_and_unknown_args(args) + ns = self._parser.parse_known_args(args) mode = getattr(ns, "assertmode", "plain") disable_autoload = getattr(ns, "disable_plugin_autoload", False) or bool( @@ -1371,13 +1371,11 @@ def _preparse(self, args: list[str], addopts: bool = True) -> None: + args ) - ns, unknown_args = self._parser.parse_known_and_unknown_args( - args, namespace=copy.copy(self.option) - ) + ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option)) rootpath, inipath, inicfg, ignored_config_files = determine_setup( inifile=ns.inifilename, override_ini=ns.override_ini, - args=ns.file_or_dir + unknown_args, + args=ns.file_or_dir, rootdir_cmd_arg=ns.rootdir or None, invocation_dir=self.invocation_params.dir, ) diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index fd907409c3f..647f2b6705a 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -149,11 +149,11 @@ def parse_known_and_unknown_args( namespace: argparse.Namespace | None = None, ) -> tuple[argparse.Namespace, list[str]]: """Parse the known arguments at this point, and also return the - remaining unknown arguments. + remaining unknown flag arguments. :returns: A tuple containing an argparse namespace object for the known - arguments, and a list of the unknown arguments. + arguments, and a list of unknown flag arguments. """ optparser = self._getparser() strargs = [os.fspath(x) for x in args] From 52268d26b0f4aa0e91f646712cfe88b3095bc4e0 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 10:58:56 +0200 Subject: [PATCH 03/14] config: remove unneeded parse in `_consider_importhook` Can use the parse we did just before the call. --- src/_pytest/config/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 46042363582..0dbe5a24d2a 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1246,19 +1246,18 @@ def pytest_load_initial_conftests(self, early_config: Config) -> None: ), ) - def _consider_importhook(self, args: Sequence[str]) -> None: + def _consider_importhook(self) -> None: """Install the PEP 302 import hook if using assertion rewriting. Needs to parse the --assert= option from the commandline and find all the installed plugins to mark them for rewriting by the importhook. """ - ns = self._parser.parse_known_args(args) - mode = getattr(ns, "assertmode", "plain") + mode = getattr(self.known_args_namespace, "assertmode", "plain") - disable_autoload = getattr(ns, "disable_plugin_autoload", False) or bool( - os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") - ) + disable_autoload = getattr( + self.known_args_namespace, "disable_plugin_autoload", False + ) or bool(os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD")) if mode == "rewrite": import _pytest.assertion @@ -1406,7 +1405,7 @@ def _preparse(self, args: list[str], addopts: bool = True) -> None: args, namespace=copy.copy(self.option) ) self._checkversion() - self._consider_importhook(args) + self._consider_importhook() self._configure_python_path() self.pluginmanager.consider_preparse(args, exclude_only=False) if ( From 27e0bb38babc0b8b1d5b490975331440e4c2ac8a Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 11:00:31 +0200 Subject: [PATCH 04/14] config: remove a couple of unneeded `:type:` doc comments --- src/_pytest/config/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 0dbe5a24d2a..5714e3346f9 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1005,7 +1005,7 @@ class InvocationParams: plugins: Sequence[str | _PluggyPlugin] | None """Extra plugins, might be `None`.""" dir: pathlib.Path - """The directory from which :func:`pytest.main` was invoked. :type: pathlib.Path""" + """The directory from which :func:`pytest.main` was invoked.""" def __init__( self, @@ -1099,8 +1099,6 @@ def __init__( def rootpath(self) -> pathlib.Path: """The path to the :ref:`rootdir `. - :type: pathlib.Path - .. versionadded:: 6.1 """ return self._rootpath From 7ba4d698ad22c6b231f5d459ade2a863c4e94c79 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 11:08:45 +0200 Subject: [PATCH 05/14] config: reorder `Parser.parse` near its comrades and add a docstring --- src/_pytest/config/argparsing.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index 647f2b6705a..03f3f0a825f 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -101,18 +101,6 @@ def addoption(self, *opts: str, **attrs: Any) -> None: """ self._anonymous.addoption(*opts, **attrs) - def parse( - self, - args: Sequence[str | os.PathLike[str]], - namespace: argparse.Namespace | None = None, - ) -> argparse.Namespace: - from _pytest._argcomplete import try_argcomplete - - self.optparser = self._getparser() - try_argcomplete(self.optparser) - strargs = [os.fspath(x) for x in args] - return self.optparser.parse_intermixed_args(strargs, namespace=namespace) - def _getparser(self) -> PytestArgumentParser: from _pytest._argcomplete import filescompleter @@ -132,6 +120,25 @@ def _getparser(self) -> PytestArgumentParser: file_or_dir_arg.completer = filescompleter # type: ignore return optparser + def parse( + self, + args: Sequence[str | os.PathLike[str]], + namespace: argparse.Namespace | None = None, + ) -> argparse.Namespace: + """Parse the arguments. + + Unlike ``parse_known_args`` and ``parse_known_and_unknown_args``, + raises UsageError on unknown flags. + + :meta private: + """ + from _pytest._argcomplete import try_argcomplete + + self.optparser = self._getparser() + try_argcomplete(self.optparser) + strargs = [os.fspath(x) for x in args] + return self.optparser.parse_intermixed_args(strargs, namespace=namespace) + def parse_known_args( self, args: Sequence[str | os.PathLike[str]], From 115b8703b898e5782bb2512eb08390674745e8e2 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 11:09:45 +0200 Subject: [PATCH 06/14] config: use intermixed arg parsing also in `parse_known_and_unknown_args` Missed in fc2d649280a335fb23d58bbd08a2e2b5f5bee50f; we should be consistent here. --- src/_pytest/config/argparsing.py | 13 ++++++++++++- testing/test_parseopt.py | 10 +++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index 03f3f0a825f..73e7e140c22 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from collections.abc import Sequence import os +import sys from typing import Any from typing import final from typing import Literal @@ -164,7 +165,17 @@ def parse_known_and_unknown_args( """ optparser = self._getparser() strargs = [os.fspath(x) for x in args] - return optparser.parse_known_args(strargs, namespace=namespace) + if sys.version_info < (3, 12): + # Older argparse have a bugged parse_known_intermixed_args. + namespace, unknown = self.optparser.parse_known_args(strargs, namespace) + assert namespace is not None + file_or_dir = getattr(namespace, FILE_OR_DIR) + unknown_flags: list[str] = [] + for arg in unknown: + (unknown_flags if arg.startswith("-") else file_or_dir).append(arg) + return namespace, unknown_flags + else: + return optparser.parse_known_intermixed_args(strargs, namespace) def addini( self, diff --git a/testing/test_parseopt.py b/testing/test_parseopt.py index d3c39d55820..b4d4193e976 100644 --- a/testing/test_parseopt.py +++ b/testing/test_parseopt.py @@ -142,17 +142,17 @@ def test_parse_known_args(self, parser: parseopt.Parser) -> None: parser.parse_known_args([Path(".")]) parser.addoption("--hello", action="store_true") ns = parser.parse_known_args(["x", "--y", "--hello", "this"]) - assert ns.hello - assert ns.file_or_dir == ["x"] + assert ns.hello is True + assert ns.file_or_dir == ["x", "this"] def test_parse_known_and_unknown_args(self, parser: parseopt.Parser) -> None: parser.addoption("--hello", action="store_true") ns, unknown = parser.parse_known_and_unknown_args( ["x", "--y", "--hello", "this"] ) - assert ns.hello - assert ns.file_or_dir == ["x"] - assert unknown == ["--y", "this"] + assert ns.hello is True + assert ns.file_or_dir == ["x", "this"] + assert unknown == ["--y"] def test_parse_will_set_default(self, parser: parseopt.Parser) -> None: parser.addoption("--hello", dest="hello", default="x", action="store") From 3b4071eba0c30c4de133ecb9e44e8e0df6f2057c Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Wed, 29 Oct 2025 17:12:27 +0200 Subject: [PATCH 07/14] helpconfig: add missing envvars to --help --- src/_pytest/helpconfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index 932e35ceacc..8e3608a5c61 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -239,6 +239,9 @@ def showhelp(config: Config) -> None: ("PYTEST_PLUGINS", "Comma-separated plugins to load during startup"), ("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "Set to disable plugin auto-loading"), ("PYTEST_DEBUG", "Set to enable debug tracing of pytest's internals"), + ("PYTEST_DEBUG_TEMPROOT", "Override the system temporary directory"), + ("PYTEST_THEME", "The Pygments style to use for code output"), + ("PYTEST_THEME_MODE", "Set the PYTEST_THEME to be either 'dark' or 'light'"), ] for name, help in vars: tw.line(f" {name:<24} {help}") From bda322a4932d9d55638202896017239a8ea7fdd2 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 17:18:15 +0200 Subject: [PATCH 08/14] config: remove unneeded lazy imports And a couple of minor cleanups. --- src/_pytest/config/__init__.py | 6 +----- src/_pytest/config/argparsing.py | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 5714e3346f9..8e47bb0a28c 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1041,9 +1041,6 @@ def __init__( *, invocation_params: InvocationParams | None = None, ) -> None: - from .argparsing import FILE_OR_DIR - from .argparsing import Parser - if invocation_params is None: invocation_params = self.InvocationParams( args=(), plugins=None, dir=pathlib.Path.cwd() @@ -1061,9 +1058,8 @@ def __init__( :type: InvocationParams """ - _a = FILE_OR_DIR self._parser = Parser( - usage=f"%(prog)s [options] [{_a}] [{_a}] [...]", + usage=f"%(prog)s [options] [{FILE_OR_DIR}] [{FILE_OR_DIR}] [...]", processopt=self._processopt, _ispytest=True, ) diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index 73e7e140c22..91c18545466 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -12,8 +12,8 @@ from typing import Literal from typing import NoReturn +from .exceptions import UsageError import _pytest._io -from _pytest.config.exceptions import UsageError from _pytest.deprecated import check_ispytest From e94518ddef13db739e9ab2c35d11e4f8904f6d13 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Thu, 30 Oct 2025 21:44:07 +0200 Subject: [PATCH 09/14] config: use one `ArgumentParser` instead of creating new one for each parse Previously, `Parser` would create a new `ArgumentParser` for each parsing call (of which there are several during pytest initialization). Besides being a bit wasteful, it makes things hard to follow, especially since one `ArgumentParser` was saved on the `Parser` (in `optparser`). It makes a few refactorings I am planning difficult to pull off. So now, `Parser` instantiates one `ArgumentParser` in `__init__` and just adds groups/arguments to it immediately. There's more synchronization needed, but since arguments can only be added, never removed, it's not bad. One gotcha, in order to control `--help` group orders we must access argparse internals since it doesn't provide a way to do it. I checked and the relevant code hasn't changed since argparse was introduced 15 years ago. So hopefully it will continue to be fine. --- src/_pytest/config/__init__.py | 2 +- src/_pytest/config/argparsing.py | 76 ++++++++++++++++---------------- testing/test_config.py | 6 +-- testing/test_parseopt.py | 21 +++++---- 4 files changed, 53 insertions(+), 52 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 8e47bb0a28c..14ac820649a 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1157,7 +1157,7 @@ def pytest_cmdline_parse( elif ( getattr(self.option, "help", False) or "--help" in args or "-h" in args ): - self._parser._getparser().print_help() + self._parser.optparser.print_help() sys.stdout.write( "\nNOTE: displaying only minimal help due to UsageError.\n\n" ) diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index 91c18545466..d202c2f1b56 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -36,8 +36,6 @@ class Parser: there's an error processing the command line arguments. """ - prog: str | None = None - def __init__( self, usage: str | None = None, @@ -46,14 +44,31 @@ def __init__( _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) - self._anonymous = OptionGroup("Custom options", parser=self, _ispytest=True) - self._groups: list[OptionGroup] = [] + + from _pytest._argcomplete import filescompleter + self._processopt = processopt - self._usage = usage + self.extra_info: dict[str, Any] = {} + self.optparser = PytestArgumentParser(self, usage, self.extra_info) + anonymous_arggroup = self.optparser.add_argument_group("Custom options") + self._anonymous = OptionGroup( + anonymous_arggroup, "_anonymous", self, _ispytest=True + ) + self._groups = [self._anonymous] + file_or_dir_arg = self.optparser.add_argument(FILE_OR_DIR, nargs="*") + file_or_dir_arg.completer = filescompleter # type: ignore + self._inidict: dict[str, tuple[str, str, Any]] = {} # Maps alias -> canonical name. self._ini_aliases: dict[str, str] = {} - self.extra_info: dict[str, Any] = {} + + @property + def prog(self) -> str: + return self.optparser.prog + + @prog.setter + def prog(self, value: str) -> None: + self.optparser.prog = value def processoption(self, option: Argument) -> None: if self._processopt: @@ -78,12 +93,17 @@ def getgroup( for group in self._groups: if group.name == name: return group - group = OptionGroup(name, description, parser=self, _ispytest=True) + + arggroup = self.optparser.add_argument_group(description or name) + group = OptionGroup(arggroup, name, self, _ispytest=True) i = 0 for i, grp in enumerate(self._groups): if grp.name == after: break self._groups.insert(i + 1, group) + # argparse doesn't provide a way to control `--help` order, so must + # access its internals ☹. + self.optparser._action_groups.insert(i + 1, self.optparser._action_groups.pop()) return group def addoption(self, *opts: str, **attrs: Any) -> None: @@ -102,25 +122,6 @@ def addoption(self, *opts: str, **attrs: Any) -> None: """ self._anonymous.addoption(*opts, **attrs) - def _getparser(self) -> PytestArgumentParser: - from _pytest._argcomplete import filescompleter - - optparser = PytestArgumentParser(self, self.extra_info, prog=self.prog) - groups = [*self._groups, self._anonymous] - for group in groups: - if group.options: - desc = group.description or group.name - arggroup = optparser.add_argument_group(desc) - for option in group.options: - n = option.names() - a = option.attrs() - arggroup.add_argument(*n, **a) - file_or_dir_arg = optparser.add_argument(FILE_OR_DIR, nargs="*") - # bash like autocompletion for dirs (appending '/') - # Type ignored because typeshed doesn't know about argcomplete. - file_or_dir_arg.completer = filescompleter # type: ignore - return optparser - def parse( self, args: Sequence[str | os.PathLike[str]], @@ -135,7 +136,6 @@ def parse( """ from _pytest._argcomplete import try_argcomplete - self.optparser = self._getparser() try_argcomplete(self.optparser) strargs = [os.fspath(x) for x in args] return self.optparser.parse_intermixed_args(strargs, namespace=namespace) @@ -163,7 +163,6 @@ def parse_known_and_unknown_args( A tuple containing an argparse namespace object for the known arguments, and a list of unknown flag arguments. """ - optparser = self._getparser() strargs = [os.fspath(x) for x in args] if sys.version_info < (3, 12): # Older argparse have a bugged parse_known_intermixed_args. @@ -175,7 +174,7 @@ def parse_known_and_unknown_args( (unknown_flags if arg.startswith("-") else file_or_dir).append(arg) return namespace, unknown_flags else: - return optparser.parse_known_intermixed_args(strargs, namespace) + return self.optparser.parse_known_intermixed_args(strargs, namespace) def addini( self, @@ -392,15 +391,14 @@ class OptionGroup: def __init__( self, + arggroup: argparse._ArgumentGroup, name: str, - description: str = "", - parser: Parser | None = None, - *, + parser: Parser | None, _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) + self._arggroup = arggroup self.name = name - self.description = description self.options: list[Argument] = [] self.parser = parser @@ -435,8 +433,11 @@ def _addoption_instance(self, option: Argument, shortupper: bool = False) -> Non for opt in option._short_opts: if opt[0] == "-" and opt[1].islower(): raise ValueError("lowercase shortoptions reserved") + if self.parser: self.parser.processoption(option) + + self._arggroup.add_argument(*option.names(), **option.attrs()) self.options.append(option) @@ -444,13 +445,12 @@ class PytestArgumentParser(argparse.ArgumentParser): def __init__( self, parser: Parser, - extra_info: dict[str, Any] | None = None, - prog: str | None = None, + usage: str | None, + extra_info: dict[str, str], ) -> None: self._parser = parser super().__init__( - prog=prog, - usage=parser._usage, + usage=usage, add_help=False, formatter_class=DropShorterLongHelpFormatter, allow_abbrev=False, @@ -458,7 +458,7 @@ def __init__( ) # extra_info is a dict of (param -> value) to display if there's # an usage error to provide more contextual information to the user. - self.extra_info = extra_info if extra_info else {} + self.extra_info = extra_info def error(self, message: str) -> NoReturn: """Transform argparse error message into UsageError.""" diff --git a/testing/test_config.py b/testing/test_config.py index f1221cea9d0..c0d6f847ef3 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -2304,10 +2304,9 @@ def test_addopts_from_ini_not_concatenated(self, pytester: Pytester) -> None: """ ) result = pytester.runpytest("cache_dir=ignored") - config = pytester._request.config result.stderr.fnmatch_lines( [ - f"{config._parser.optparser.prog}: error: argument -o/--override-ini: expected one argument", + "*: error: argument -o/--override-ini: expected one argument", " config source: via addopts config", ] ) @@ -2410,8 +2409,7 @@ def pytest_addoption(parser): result.stderr.fnmatch_lines( [ "ERROR: usage: *", - f"{pytester._request.config._parser.optparser.prog}: error: " - f"argument --invalid-option-should-allow-for-help: expected one argument", + "*: error: argument --invalid-option-should-allow-for-help: expected one argument", ] ) # Does not display full/default help. diff --git a/testing/test_parseopt.py b/testing/test_parseopt.py index b4d4193e976..30370d3d673 100644 --- a/testing/test_parseopt.py +++ b/testing/test_parseopt.py @@ -28,9 +28,10 @@ def test_no_help_by_default(self) -> None: def test_custom_prog(self, parser: parseopt.Parser) -> None: """Custom prog can be set for `argparse.ArgumentParser`.""" - assert parser._getparser().prog == argparse.ArgumentParser().prog + assert parser.optparser.prog == argparse.ArgumentParser().prog parser.prog = "custom-prog" - assert parser._getparser().prog == "custom-prog" + assert parser.prog == "custom-prog" + assert parser.optparser.prog == "custom-prog" def test_argument(self) -> None: with pytest.raises(parseopt.ArgumentError): @@ -71,14 +72,12 @@ def test_argument_processopt(self) -> None: assert res["dest"] == "abc" def test_group_add_and_get(self, parser: parseopt.Parser) -> None: - group = parser.getgroup("hello", description="desc") + group = parser.getgroup("hello") assert group.name == "hello" - assert group.description == "desc" def test_getgroup_simple(self, parser: parseopt.Parser) -> None: - group = parser.getgroup("hello", description="desc") + group = parser.getgroup("hello") assert group.name == "hello" - assert group.description == "desc" group2 = parser.getgroup("hello") assert group2 is group @@ -88,16 +87,20 @@ def test_group_ordering(self, parser: parseopt.Parser) -> None: parser.getgroup("3", after="1") groups = parser._groups groups_names = [x.name for x in groups] - assert groups_names == list("132") + assert groups_names == ["_anonymous", "1", "3", "2"] def test_group_addoption(self) -> None: - group = parseopt.OptionGroup("hello", _ispytest=True) + optparser = argparse.ArgumentParser() + arggroup = optparser.add_argument_group("hello") + group = parseopt.OptionGroup(arggroup, "hello", None, _ispytest=True) group.addoption("--option1", action="store_true") assert len(group.options) == 1 assert isinstance(group.options[0], parseopt.Argument) def test_group_addoption_conflict(self) -> None: - group = parseopt.OptionGroup("hello again", _ispytest=True) + optparser = argparse.ArgumentParser() + arggroup = optparser.add_argument_group("hello again") + group = parseopt.OptionGroup(arggroup, "hello again", None, _ispytest=True) group.addoption("--option1", "--option-1", action="store_true") with pytest.raises(ValueError) as err: group.addoption("--option1", "--option-one", action="store_true") From 5e3b7d5e1f0c9dc7f426dbecaef76efcea4a3770 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Fri, 31 Oct 2025 10:14:27 +0200 Subject: [PATCH 10/14] config: remove `after_preparse` The `after_preparse` hack was used to decide between two behaviors of `--help`: 1. Just set `help` on the Namespace and continue parsing 2. Interrupt the parsing The `after_preparse` was used to make 1 happen before `_preparse` and 2 after. But that's not what we want, the timing shouldn't really matter, only the "mode" in which we parse: - `Parser.parse_known_[and_unknown_]arguments` -- permissive mode, want behavior 1. - `Parser.parse` -- actual parsing, want behavior 2. Make it so. --- src/_pytest/config/__init__.py | 1 - src/_pytest/config/argparsing.py | 10 ++++++++-- src/_pytest/helpconfig.py | 20 ++++++++++---------- testing/test_helpconfig.py | 8 ++++++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 14ac820649a..8034cec970f 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1528,7 +1528,6 @@ def parse(self, args: list[str], addopts: bool = True) -> None: kwargs=dict(pluginmanager=self.pluginmanager) ) self._preparse(args, addopts=addopts) - self._parser.after_preparse = True # type: ignore try: self._parser.parse(args, namespace=self.option) except PrintHelp: diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index d202c2f1b56..995408800a8 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -130,7 +130,7 @@ def parse( """Parse the arguments. Unlike ``parse_known_args`` and ``parse_known_and_unknown_args``, - raises UsageError on unknown flags. + raises PrintHelp on `--help` and UsageError on unknown flags :meta private: """ @@ -138,7 +138,13 @@ def parse( try_argcomplete(self.optparser) strargs = [os.fspath(x) for x in args] - return self.optparser.parse_intermixed_args(strargs, namespace=namespace) + if namespace is None: + namespace = argparse.Namespace() + try: + namespace._raise_print_help = True + return self.optparser.parse_intermixed_args(strargs, namespace=namespace) + finally: + del namespace._raise_print_help def parse_known_args( self, diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index 8e3608a5c61..6a22c9f58ac 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -14,19 +14,21 @@ from _pytest.config import ExitCode from _pytest.config import PrintHelp from _pytest.config.argparsing import Parser -from _pytest.config.argparsing import PytestArgumentParser from _pytest.terminal import TerminalReporter import pytest class HelpAction(argparse.Action): - """An argparse Action that will raise an exception in order to skip the - rest of the argument parsing when --help is passed. + """An argparse Action that will raise a PrintHelp exception in order to skip + the rest of the argument parsing when --help is passed. - This prevents argparse from quitting due to missing required arguments - when any are defined, for example by ``pytest_addoption``. - This is similar to the way that the builtin argparse --help option is - implemented by raising SystemExit. + This prevents argparse from raising UsageError when `--help` is used along + with missing required arguments when any are defined, for example by + ``pytest_addoption``. This is similar to the way that the builtin argparse + --help option is implemented by raising SystemExit. + + To opt in to this behavior, the parse caller must set + `namespace._raise_print_help = True`. Otherwise it just sets the option. """ def __init__( @@ -50,9 +52,7 @@ def __call__( ) -> None: setattr(namespace, self.dest, self.const) - # We should only skip the rest of the parsing after preparse is done. - assert isinstance(parser, PytestArgumentParser) - if getattr(parser._parser, "after_preparse", False): + if getattr(namespace, "_raise_print_help", False): raise PrintHelp diff --git a/testing/test_helpconfig.py b/testing/test_helpconfig.py index 455ed5276a8..b01a6fa1559 100644 --- a/testing/test_helpconfig.py +++ b/testing/test_helpconfig.py @@ -83,6 +83,14 @@ def pytest_addoption(parser): result.stdout.fnmatch_lines(lines, consecutive=True) +def test_parse_known_args_doesnt_quit_on_help(pytester: Pytester) -> None: + """`parse_known_args` shouldn't exit on `--help`, unlike `parse`.""" + config = pytester.parseconfig() + # Doesn't raise or exit! + config._parser.parse_known_args(["--help"]) + config._parser.parse_known_and_unknown_args(["--help"]) + + def test_hookvalidation_unknown(pytester: Pytester) -> None: pytester.makeconftest( """ From d0e607670de85e674c219ea189c2e46440896083 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Fri, 31 Oct 2025 10:51:53 +0200 Subject: [PATCH 11/14] main: remove ineffective "general" group description Since this is not the first `getgroup` call on this name, the description is just ignored. It's also not really accurate, as the "general" group contains a bunch of stuff. So let's just keep "general" as the description for now. --- src/_pytest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 54944677891..9bc930df8e8 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -56,7 +56,7 @@ def pytest_addoption(parser: Parser) -> None: - group = parser.getgroup("general", "Running and selection options") + group = parser.getgroup("general") group._addoption( # private to use reserved lower-case short option "-x", "--exitfirst", From 5a85b90a442eb4fd77cf2a3309d3280e8793d1a9 Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Fri, 31 Oct 2025 11:40:46 +0200 Subject: [PATCH 12/14] config: inline `_preparse` No that "preparse" phase is no longer a thing, with the removal of the historic `pytest_cmdline_preparse` hook and `after_preparse` hack, let's inline so we can see the full parse flow in linear code. --- src/_pytest/config/__init__.py | 176 +++++++++++++++++---------------- testing/test_config.py | 6 +- testing/test_warnings.py | 8 +- 3 files changed, 95 insertions(+), 95 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 8034cec970f..74cd83b8872 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1355,92 +1355,6 @@ def _decide_args( result = [str(invocation_dir)] return result, source - def _preparse(self, args: list[str], addopts: bool = True) -> None: - if addopts: - env_addopts = os.environ.get("PYTEST_ADDOPTS", "") - if len(env_addopts): - args[:] = ( - self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") - + args - ) - - ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option)) - rootpath, inipath, inicfg, ignored_config_files = determine_setup( - inifile=ns.inifilename, - override_ini=ns.override_ini, - args=ns.file_or_dir, - rootdir_cmd_arg=ns.rootdir or None, - invocation_dir=self.invocation_params.dir, - ) - self._rootpath = rootpath - self._inipath = inipath - self._ignored_config_files = ignored_config_files - self.inicfg = inicfg - self._parser.extra_info["rootdir"] = str(self.rootpath) - self._parser.extra_info["inifile"] = str(self.inipath) - self._parser.addini("addopts", "Extra command line options", "args") - self._parser.addini("minversion", "Minimally required pytest version") - self._parser.addini( - "pythonpath", type="paths", help="Add paths to sys.path", default=[] - ) - self._parser.addini( - "required_plugins", - "Plugins that must be present for pytest to run", - type="args", - default=[], - ) - - if addopts: - args[:] = ( - self._validate_args(self.getini("addopts"), "via addopts config") + args - ) - - self.known_args_namespace = self._parser.parse_known_args( - args, namespace=copy.copy(self.option) - ) - self._checkversion() - self._consider_importhook() - self._configure_python_path() - self.pluginmanager.consider_preparse(args, exclude_only=False) - if ( - not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") - and not self.known_args_namespace.disable_plugin_autoload - ): - # Autoloading from distribution package entry point has - # not been disabled. - self.pluginmanager.load_setuptools_entrypoints("pytest11") - # Otherwise only plugins explicitly specified in PYTEST_PLUGINS - # are going to be loaded. - self.pluginmanager.consider_env() - - self.known_args_namespace = self._parser.parse_known_args( - args, namespace=copy.copy(self.known_args_namespace) - ) - - self._validate_plugins() - self._warn_about_skipped_plugins() - - if self.known_args_namespace.confcutdir is None: - if self.inipath is not None: - confcutdir = str(self.inipath.parent) - else: - confcutdir = str(self.rootpath) - self.known_args_namespace.confcutdir = confcutdir - try: - self.hook.pytest_load_initial_conftests( - early_config=self, args=args, parser=self._parser - ) - except ConftestImportFailure as e: - if self.known_args_namespace.help or self.known_args_namespace.version: - # we don't want to prevent --help/--version to work - # so just let it pass and print a warning at the end - self.issue_config_time_warning( - PytestConfigWarning(f"could not load initial conftests: {e.path}"), - stacklevel=2, - ) - else: - raise - @hookimpl(wrapper=True) def pytest_collection(self) -> Generator[None, object, object]: # Validate invalid configuration keys after collection is done so we @@ -1524,14 +1438,102 @@ def parse(self, args: list[str], addopts: bool = True) -> None: assert self.args == [], ( "can only parse cmdline args at most once per Config object" ) + self.hook.pytest_addhooks.call_historic( kwargs=dict(pluginmanager=self.pluginmanager) ) - self._preparse(args, addopts=addopts) + + if addopts: + env_addopts = os.environ.get("PYTEST_ADDOPTS", "") + if len(env_addopts): + args[:] = ( + self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") + + args + ) + + ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option)) + rootpath, inipath, inicfg, ignored_config_files = determine_setup( + inifile=ns.inifilename, + override_ini=ns.override_ini, + args=ns.file_or_dir, + rootdir_cmd_arg=ns.rootdir or None, + invocation_dir=self.invocation_params.dir, + ) + self._rootpath = rootpath + self._inipath = inipath + self._ignored_config_files = ignored_config_files + self.inicfg = inicfg + self._parser.extra_info["rootdir"] = str(self.rootpath) + self._parser.extra_info["inifile"] = str(self.inipath) + + self._parser.addini("addopts", "Extra command line options", "args") + self._parser.addini("minversion", "Minimally required pytest version") + self._parser.addini( + "pythonpath", type="paths", help="Add paths to sys.path", default=[] + ) + self._parser.addini( + "required_plugins", + "Plugins that must be present for pytest to run", + type="args", + default=[], + ) + + if addopts: + args[:] = ( + self._validate_args(self.getini("addopts"), "via addopts config") + args + ) + + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.option) + ) + self._checkversion() + self._consider_importhook() + self._configure_python_path() + self.pluginmanager.consider_preparse(args, exclude_only=False) + if ( + not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + and not self.known_args_namespace.disable_plugin_autoload + ): + # Autoloading from distribution package entry point has + # not been disabled. + self.pluginmanager.load_setuptools_entrypoints("pytest11") + # Otherwise only plugins explicitly specified in PYTEST_PLUGINS + # are going to be loaded. + self.pluginmanager.consider_env() + + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.known_args_namespace) + ) + + self._validate_plugins() + self._warn_about_skipped_plugins() + + if self.known_args_namespace.confcutdir is None: + if self.inipath is not None: + confcutdir = str(self.inipath.parent) + else: + confcutdir = str(self.rootpath) + self.known_args_namespace.confcutdir = confcutdir + try: + self.hook.pytest_load_initial_conftests( + early_config=self, args=args, parser=self._parser + ) + except ConftestImportFailure as e: + if self.known_args_namespace.help or self.known_args_namespace.version: + # we don't want to prevent --help/--version to work + # so just let it pass and print a warning at the end + self.issue_config_time_warning( + PytestConfigWarning(f"could not load initial conftests: {e.path}"), + stacklevel=2, + ) + else: + raise + try: self._parser.parse(args, namespace=self.option) except PrintHelp: return + self.args, self.args_source = self._decide_args( args=getattr(self.option, FILE_OR_DIR), pyargs=self.option.pyargs, diff --git a/testing/test_config.py b/testing/test_config.py index c0d6f847ef3..98555e04452 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -2276,7 +2276,7 @@ def test_addopts_before_initini( cache_dir = ".custom_cache" monkeypatch.setenv("PYTEST_ADDOPTS", f"-o cache_dir={cache_dir}") config = _config_for_test - config._preparse([], addopts=True) + config.parse([], addopts=True) assert config.inicfg.get("cache_dir") == ConfigValue( cache_dir, origin="override", mode="ini" ) @@ -2288,7 +2288,7 @@ def test_addopts_from_env_not_concatenated( monkeypatch.setenv("PYTEST_ADDOPTS", "-o") config = _config_for_test with pytest.raises(UsageError) as excinfo: - config._preparse(["cache_dir=ignored"], addopts=True) + config.parse(["cache_dir=ignored"], addopts=True) assert ( "error: argument -o/--override-ini: expected one argument" in excinfo.value.args[0] @@ -2317,7 +2317,7 @@ def test_override_ini_does_not_contain_paths( ) -> None: """Check that -o no longer swallows all options after it (#3103)""" config = _config_for_test - config._preparse(["-o", "cache_dir=/cache", "/some/test/path"]) + config.parse(["-o", "cache_dir=/cache", "/some/test/path"]) assert config.inicfg.get("cache_dir") == ConfigValue( "/cache", origin="override", mode="ini" ) diff --git a/testing/test_warnings.py b/testing/test_warnings.py index d13ed72a2d4..e3221da7569 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -742,10 +742,8 @@ def test_issue4445_rewrite(self, pytester: Pytester, capwarn) -> None: assert func == "" # the above conftest.py assert lineno == 4 - def test_issue4445_preparse(self, pytester: Pytester, capwarn) -> None: - """#4445: Make sure the warning points to a reasonable location - See origin of _issue_warning_captured at: _pytest.config.__init__.py:910 - """ + def test_issue4445_initial_conftest(self, pytester: Pytester, capwarn) -> None: + """#4445: Make sure the warning points to a reasonable location.""" pytester.makeconftest( """ import nothing @@ -761,7 +759,7 @@ def test_issue4445_preparse(self, pytester: Pytester, capwarn) -> None: assert "could not load initial conftests" in str(warning.message) assert f"config{os.sep}__init__.py" in file - assert func == "_preparse" + assert func == "parse" @pytest.mark.filterwarnings("default") def test_conftest_warning_captured(self, pytester: Pytester) -> None: From bf78ef34f00951a74b0c8a8fed2765cf67db4f7a Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Fri, 31 Oct 2025 12:03:04 +0200 Subject: [PATCH 13/14] config: remove an unneeded Namespace copy --- src/_pytest/config/__init__.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 74cd83b8872..54e4ec6e2bc 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1501,9 +1501,7 @@ def parse(self, args: list[str], addopts: bool = True) -> None: # are going to be loaded. self.pluginmanager.consider_env() - self.known_args_namespace = self._parser.parse_known_args( - args, namespace=copy.copy(self.known_args_namespace) - ) + self._parser.parse_known_args(args, namespace=self.known_args_namespace) self._validate_plugins() self._warn_about_skipped_plugins() From 2079b916eae146430b5ac9322074d986f7825f9b Mon Sep 17 00:00:00 2001 From: Ran Benita Date: Fri, 31 Oct 2025 12:18:11 +0200 Subject: [PATCH 14/14] config: move uninteresting error handling code to own functions --- src/_pytest/config/__init__.py | 58 +++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 54e4ec6e2bc..595face1812 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -142,6 +142,29 @@ def filter_traceback_for_conftest_import_failure( return filter_traceback(entry) and "importlib" not in str(entry.path).split(os.sep) +def print_conftest_import_error(e: ConftestImportFailure, file: TextIO) -> None: + exc_info = ExceptionInfo.from_exception(e.cause) + tw = TerminalWriter(file) + tw.line(f"ImportError while loading conftest '{e.path}'.", red=True) + exc_info.traceback = exc_info.traceback.filter( + filter_traceback_for_conftest_import_failure + ) + exc_repr = ( + exc_info.getrepr(style="short", chain=False) + if exc_info.traceback + else exc_info.exconly() + ) + formatted_tb = str(exc_repr) + for line in formatted_tb.splitlines(): + tw.line(line.rstrip(), red=True) + + +def print_usage_error(e: UsageError, file: TextIO) -> None: + tw = TerminalWriter(file) + for msg in e.args: + tw.line(f"ERROR: {msg}\n", red=True) + + def main( args: list[str] | os.PathLike[str] | None = None, plugins: Sequence[str | _PluggyPlugin] | None = None, @@ -167,34 +190,19 @@ def main( try: config = _prepareconfig(new_args, plugins) except ConftestImportFailure as e: - exc_info = ExceptionInfo.from_exception(e.cause) - tw = TerminalWriter(sys.stderr) - tw.line(f"ImportError while loading conftest '{e.path}'.", red=True) - exc_info.traceback = exc_info.traceback.filter( - filter_traceback_for_conftest_import_failure - ) - exc_repr = ( - exc_info.getrepr(style="short", chain=False) - if exc_info.traceback - else exc_info.exconly() - ) - formatted_tb = str(exc_repr) - for line in formatted_tb.splitlines(): - tw.line(line.rstrip(), red=True) + print_conftest_import_error(e, file=sys.stderr) return ExitCode.USAGE_ERROR - else: + + try: + ret: ExitCode | int = config.hook.pytest_cmdline_main(config=config) try: - ret: ExitCode | int = config.hook.pytest_cmdline_main(config=config) - try: - return ExitCode(ret) - except ValueError: - return ret - finally: - config._ensure_unconfigure() + return ExitCode(ret) + except ValueError: + return ret + finally: + config._ensure_unconfigure() except UsageError as e: - tw = TerminalWriter(sys.stderr) - for msg in e.args: - tw.line(f"ERROR: {msg}\n", red=True) + print_usage_error(e, file=sys.stderr) return ExitCode.USAGE_ERROR finally: if old_pytest_version is None: