diff --git a/changelog/11671.feature.rst b/changelog/11671.feature.rst new file mode 100644 index 00000000000..9e401112ad0 --- /dev/null +++ b/changelog/11671.feature.rst @@ -0,0 +1 @@ +Added `RaisesGroup` (also available as `raises_group`) and `Matcher`, as an equivalent to `raises` for expecting `ExceptionGroup`. It includes the ability to specity multiple different expected exceptions, the structure of nested exception groups, and/or closely emulating `except_star`. diff --git a/src/_pytest/_raises_group.py b/src/_pytest/_raises_group.py new file mode 100644 index 00000000000..68303c4a3fe --- /dev/null +++ b/src/_pytest/_raises_group.py @@ -0,0 +1,1003 @@ +from __future__ import annotations + +from abc import ABC +from abc import abstractmethod +import re +from re import Pattern +import sys +from textwrap import indent +from typing import cast +from typing import final +from typing import Generic +from typing import Literal +from typing import overload +from typing import TYPE_CHECKING + +from _pytest._code import ExceptionInfo +from _pytest.outcomes import fail + + +if TYPE_CHECKING: + from collections.abc import Callable + from collections.abc import Sequence + + # for some reason Sphinx does not play well with 'from types import TracebackType' + import types + + from typing_extensions import TypeGuard + from typing_extensions import TypeVar + + # this conditional definition is because we want to allow a TypeVar default + MatchE = TypeVar( + "MatchE", + bound=BaseException, + default=BaseException, + covariant=True, + ) +else: + from typing import TypeVar + + MatchE = TypeVar("MatchE", bound=BaseException, covariant=True) + +# RaisesGroup doesn't work with a default. +BaseExcT_co = TypeVar("BaseExcT_co", bound=BaseException, covariant=True) +BaseExcT_1 = TypeVar("BaseExcT_1", bound=BaseException) +BaseExcT_2 = TypeVar("BaseExcT_2", bound=BaseException) +ExcT_1 = TypeVar("ExcT_1", bound=Exception) +ExcT_2 = TypeVar("ExcT_2", bound=Exception) + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + + +# this differs slightly from pytest.ExceptionInfo._stringify_exception +# as we don't want '(1 sub-exception)' when matching group strings +def _stringify_exception(exc: BaseException) -> str: + return "\n".join( + [ + exc.message if isinstance(exc, BaseExceptionGroup) else str(exc), + *getattr(exc, "__notes__", []), + ], + ) + + +# String patterns default to including the unicode flag. +_REGEX_NO_FLAGS = re.compile(r"").flags + + +def _match_pattern(match: Pattern[str]) -> str | Pattern[str]: + """Helper function to remove redundant `re.compile` calls when printing regex""" + return match.pattern if match.flags == _REGEX_NO_FLAGS else match + + +def repr_callable(fun: Callable[[BaseExcT_1], bool]) -> str: + """Get the repr of a ``check`` parameter. + + Split out so it can be monkeypatched (e.g. by hypothesis) + """ + return repr(fun) + + +def _exception_type_name(e: type[BaseException]) -> str: + return repr(e.__name__) + + +def _check_raw_type( + expected_type: type[BaseException] | None, + exception: BaseException, +) -> str | None: + if expected_type is None: + return None + + if not isinstance( + exception, + expected_type, + ): + actual_type_str = _exception_type_name(type(exception)) + expected_type_str = _exception_type_name(expected_type) + if isinstance(exception, BaseExceptionGroup) and not issubclass( + expected_type, BaseExceptionGroup + ): + return f"Unexpected nested {actual_type_str}, expected {expected_type_str}" + return f"{actual_type_str} is not of type {expected_type_str}" + return None + + +class AbstractMatcher(ABC, Generic[BaseExcT_co]): + """ABC with common functionality shared between Matcher and RaisesGroup""" + + def __init__( + self, + match: str | Pattern[str] | None, + check: Callable[[BaseExcT_co], bool] | None, + ) -> None: + if isinstance(match, str): + self.match: Pattern[str] | None = re.compile(match) + else: + self.match = match + self.check = check + self._fail_reason: str | None = None + + # used to suppress repeated printing of `repr(self.check)` + self._nested: bool = False + + @property + def fail_reason(self) -> str | None: + """Set after a call to `matches` to give a human-readable reason for why the match failed. + When used as a context manager the string will be given as the text of an + `Failed`""" + return self._fail_reason + + def _check_check( + self: AbstractMatcher[BaseExcT_1], + exception: BaseExcT_1, + ) -> bool: + if self.check is None: + return True + + if self.check(exception): + return True + + check_repr = "" if self._nested else " " + repr_callable(self.check) + self._fail_reason = f"check{check_repr} did not return True" + return False + + def _check_match(self, e: BaseException) -> bool: + if self.match is None or re.search( + self.match, + stringified_exception := _stringify_exception(e), + ): + return True + + maybe_specify_type = ( + f" of {_exception_type_name(type(e))}" + if isinstance(e, BaseExceptionGroup) + else "" + ) + self._fail_reason = ( + f"Regex pattern {_match_pattern(self.match)!r}" + f" did not match {stringified_exception!r}{maybe_specify_type}" + ) + if _match_pattern(self.match) == stringified_exception: + self._fail_reason += "\n Did you mean to `re.escape()` the regex?" + return False + + @abstractmethod + def matches( + self: AbstractMatcher[BaseExcT_1], exc_val: BaseException + ) -> TypeGuard[BaseExcT_1]: + """Check if an exception matches the requirements of this AbstractMatcher. + If it fails, `AbstractMatcher.fail_reason` should be set. + """ + + +@final +class Matcher(AbstractMatcher[MatchE]): + """Helper class to be used together with RaisesGroups when you want to specify requirements on sub-exceptions. + Only specifying the type is redundant, and it's also unnecessary when the type is a + nested `RaisesGroup` since it supports the same arguments. + The type is checked with `isinstance`, and does not need to be an exact match. + If that is wanted you can use the ``check`` parameter. + :meth:`Matcher.matches` can also be used standalone to check individual exceptions. + + Examples:: + + with RaisesGroups(Matcher(ValueError, match="string")) + ... + with RaisesGroups(Matcher(check=lambda x: x.args == (3, "hello"))): + ... + with RaisesGroups(Matcher(check=lambda x: type(x) is ValueError)): + ... + + Tip: if you install ``hypothesis`` and import it in ``conftest.py`` you will get + readable ``repr``s of ``check`` callables in the output. + """ + + # Trio bundled hypothesis monkeypatching, we will probably instead assume that + # hypothesis will handle that in their pytest plugin by the time this is released. + # Alternatively we could add a version of get_pretty_function_description ourselves + # https://github.com/HypothesisWorks/hypothesis/blob/8ced2f59f5c7bea3344e35d2d53e1f8f8eb9fcd8/hypothesis-python/src/hypothesis/internal/reflection.py#L439 + + # At least one of the three parameters must be passed. + @overload + def __init__( + self: Matcher[MatchE], + exception_type: type[MatchE], + match: str | Pattern[str] = ..., + check: Callable[[MatchE], bool] = ..., + ) -> None: ... + + @overload + def __init__( + self: Matcher[BaseException], # Give E a value. + *, + match: str | Pattern[str], + # If exception_type is not provided, check() must do any typechecks itself. + check: Callable[[BaseException], bool] = ..., + ) -> None: ... + + @overload + def __init__(self, *, check: Callable[[BaseException], bool]) -> None: ... + + def __init__( + self, + exception_type: type[MatchE] | None = None, + match: str | Pattern[str] | None = None, + check: Callable[[MatchE], bool] | None = None, + ): + super().__init__(match, check) + if exception_type is None and match is None and check is None: + raise ValueError("You must specify at least one parameter to match on.") + if exception_type is not None and not issubclass(exception_type, BaseException): + raise TypeError( + f"exception_type {exception_type} must be a subclass of BaseException", + ) + self.exception_type = exception_type + + def matches( + self, + exception: BaseException, + ) -> TypeGuard[MatchE]: + """Check if an exception matches the requirements of this Matcher. + If it fails, `Matcher.fail_reason` will be set. + + Examples:: + + assert Matcher(ValueError).matches(my_exception): + # is equivalent to + assert isinstance(my_exception, ValueError) + + # this can be useful when checking e.g. the ``__cause__`` of an exception. + with pytest.raises(ValueError) as excinfo: + ... + assert Matcher(SyntaxError, match="foo").matches(excinfo.value.__cause__) + # above line is equivalent to + assert isinstance(excinfo.value.__cause__, SyntaxError) + assert re.search("foo", str(excinfo.value.__cause__) + + """ + if not self._check_type(exception): + return False + + if not self._check_match(exception): + return False + + return self._check_check(exception) + + def __repr__(self) -> str: + parameters = [] + if self.exception_type is not None: + parameters.append(self.exception_type.__name__) + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + parameters.append( + f"match={_match_pattern(self.match)!r}", + ) + if self.check is not None: + parameters.append(f"check={repr_callable(self.check)}") + return f"Matcher({', '.join(parameters)})" + + def _check_type(self, exception: BaseException) -> TypeGuard[MatchE]: + self._fail_reason = _check_raw_type(self.exception_type, exception) + return self._fail_reason is None + + +@final +class RaisesGroup(AbstractMatcher[BaseExceptionGroup[BaseExcT_co]]): + """Contextmanager for checking for an expected `ExceptionGroup`. + This works similar to ``pytest.raises``, but allows for specifying the structure of an `ExceptionGroup`. + `ExceptionInfo.group_contains` also tries to handle exception groups, + but it is very bad at checking that you *didn't* get exceptions you didn't expect. + + + The catching behaviour differs from :ref:`except* ` in multiple + different ways, being much stricter by default. + By using ``allow_unwrapped=True`` and ``flatten_subgroups=True`` you can match + ``except*`` fully when expecting a single exception. + + #. All specified exceptions must be present, *and no others*. + + * If you expect a variable number of exceptions you need to use ``pytest.raises(ExceptionGroup)`` and manually + check the contained exceptions. Consider making use of :func:`Matcher.matches`. + + #. It will only catch exceptions wrapped in an exceptiongroup by default. + + * With ``allow_unwrapped=True`` you can specify a single expected exception or `Matcher` and it will match + the exception even if it is not inside an `ExceptionGroup`. + If you expect one of several different exception types you need to use a `Matcher` object. + + #. By default it cares about the full structure with nested `ExceptionGroup`'s. You can specify nested + `ExceptionGroup`'s by passing `RaisesGroup` objects as expected exceptions. + + * With ``flatten_subgroups=True`` it will "flatten" the raised `ExceptionGroup`, + extracting all exceptions inside any nested :class:`ExceptionGroup`, before matching. + + It does not care about the order of the exceptions, so + ``RaisesGroups(ValueError, TypeError)`` + is equivalent to + ``RaisesGroups(TypeError, ValueError)``. + + Examples:: + + with RaisesGroups(ValueError): + raise ExceptionGroup("", (ValueError(),)) + with RaisesGroups( + ValueError, ValueError, Matcher(TypeError, match="expected int") + ): + ... + with RaisesGroups( + KeyboardInterrupt, + match="hello", + check=lambda x: type(x) is BaseExceptionGroup, + ): + ... + with RaisesGroups(RaisesGroups(ValueError)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # flatten_subgroups + with RaisesGroups(ValueError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # allow_unwrapped + with RaisesGroups(ValueError, allow_unwrapped=True): + raise ValueError + + + `RaisesGroup.matches` can also be used directly to check a standalone exception group. + + + The matching algorithm is greedy, which means cases such as this may fail:: + + with RaisesGroups(ValueError, Matcher(ValueError, match="hello")): + raise ExceptionGroup("", (ValueError("hello"), ValueError("goodbye"))) + + even though it generally does not care about the order of the exceptions in the group. + To avoid the above you should specify the first ValueError with a Matcher as well. + + Tip: if you install ``hypothesis`` and import it in ``conftest.py`` you will get + readable ``repr``s of ``check`` callables in the output. + """ + + # allow_unwrapped=True requires: singular exception, exception not being + # RaisesGroup instance, match is None, check is None + @overload + def __init__( + self, + exception: type[BaseExcT_co] | Matcher[BaseExcT_co], + *, + allow_unwrapped: Literal[True], + flatten_subgroups: bool = False, + ) -> None: ... + + # flatten_subgroups = True also requires no nested RaisesGroup + @overload + def __init__( + self, + exception: type[BaseExcT_co] | Matcher[BaseExcT_co], + *other_exceptions: type[BaseExcT_co] | Matcher[BaseExcT_co], + flatten_subgroups: Literal[True], + match: str | Pattern[str] | None = None, + check: Callable[[BaseExceptionGroup[BaseExcT_co]], bool] | None = None, + ) -> None: ... + + # simplify the typevars if possible (the following 3 are equivalent but go simpler->complicated) + # ... the first handles RaisesGroup[ValueError], the second RaisesGroup[ExceptionGroup[ValueError]], + # the third RaisesGroup[ValueError | ExceptionGroup[ValueError]]. + # ... otherwise, we will get results like RaisesGroup[ValueError | ExceptionGroup[Never]] (I think) + # (technically correct but misleading) + @overload + def __init__( + self: RaisesGroup[ExcT_1], + exception: type[ExcT_1] | Matcher[ExcT_1], + *other_exceptions: type[ExcT_1] | Matcher[ExcT_1], + match: str | Pattern[str] | None = None, + check: Callable[[ExceptionGroup[ExcT_1]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[ExceptionGroup[ExcT_2]], + exception: RaisesGroup[ExcT_2], + *other_exceptions: RaisesGroup[ExcT_2], + match: str | Pattern[str] | None = None, + check: Callable[[ExceptionGroup[ExceptionGroup[ExcT_2]]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[ExcT_1 | ExceptionGroup[ExcT_2]], + exception: type[ExcT_1] | Matcher[ExcT_1] | RaisesGroup[ExcT_2], + *other_exceptions: type[ExcT_1] | Matcher[ExcT_1] | RaisesGroup[ExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[[ExceptionGroup[ExcT_1 | ExceptionGroup[ExcT_2]]], bool] | None + ) = None, + ) -> None: ... + + # same as the above 3 but handling BaseException + @overload + def __init__( + self: RaisesGroup[BaseExcT_1], + exception: type[BaseExcT_1] | Matcher[BaseExcT_1], + *other_exceptions: type[BaseExcT_1] | Matcher[BaseExcT_1], + match: str | Pattern[str] | None = None, + check: Callable[[BaseExceptionGroup[BaseExcT_1]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[BaseExceptionGroup[BaseExcT_2]], + exception: RaisesGroup[BaseExcT_2], + *other_exceptions: RaisesGroup[BaseExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[[BaseExceptionGroup[BaseExceptionGroup[BaseExcT_2]]], bool] | None + ) = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]], + exception: type[BaseExcT_1] | Matcher[BaseExcT_1] | RaisesGroup[BaseExcT_2], + *other_exceptions: type[BaseExcT_1] + | Matcher[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[ + [BaseExceptionGroup[BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]]], + bool, + ] + | None + ) = None, + ) -> None: ... + + def __init__( + self: RaisesGroup[ExcT_1 | BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]], + exception: type[BaseExcT_1] | Matcher[BaseExcT_1] | RaisesGroup[BaseExcT_2], + *other_exceptions: type[BaseExcT_1] + | Matcher[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + allow_unwrapped: bool = False, + flatten_subgroups: bool = False, + match: str | Pattern[str] | None = None, + check: ( + Callable[[BaseExceptionGroup[BaseExcT_1]], bool] + | Callable[[ExceptionGroup[ExcT_1]], bool] + | None + ) = None, + ): + # The type hint on the `self` and `check` parameters uses different formats + # that are *very* hard to reconcile while adhering to the overloads, so we cast + # it to avoid an error when passing it to super().__init__ + check = cast( + "Callable[[" + "BaseExceptionGroup[ExcT_1|BaseExcT_1|BaseExceptionGroup[BaseExcT_2]]" + "], bool]", + check, + ) + super().__init__(match, check) + self.expected_exceptions: tuple[ + type[BaseExcT_co] | Matcher[BaseExcT_co] | RaisesGroup[BaseException], ... + ] = ( + exception, + *other_exceptions, + ) + self.allow_unwrapped = allow_unwrapped + self.flatten_subgroups: bool = flatten_subgroups + self.is_baseexceptiongroup = False + + if allow_unwrapped and other_exceptions: + raise ValueError( + "You cannot specify multiple exceptions with `allow_unwrapped=True.`" + " If you want to match one of multiple possible exceptions you should" + " use a `Matcher`." + " E.g. `Matcher(check=lambda e: isinstance(e, (...)))`", + ) + if allow_unwrapped and isinstance(exception, RaisesGroup): + raise ValueError( + "`allow_unwrapped=True` has no effect when expecting a `RaisesGroup`." + " You might want it in the expected `RaisesGroup`, or" + " `flatten_subgroups=True` if you don't care about the structure.", + ) + if allow_unwrapped and (match is not None or check is not None): + raise ValueError( + "`allow_unwrapped=True` bypasses the `match` and `check` parameters" + " if the exception is unwrapped. If you intended to match/check the" + " exception you should use a `Matcher` object. If you want to match/check" + " the exceptiongroup when the exception *is* wrapped you need to" + " do e.g. `if isinstance(exc.value, ExceptionGroup):" + " assert RaisesGroup(...).matches(exc.value)` afterwards.", + ) + + # verify `expected_exceptions` and set `self.is_baseexceptiongroup` + for exc in self.expected_exceptions: + if isinstance(exc, RaisesGroup): + if self.flatten_subgroups: + raise ValueError( + "You cannot specify a nested structure inside a RaisesGroup with" + " `flatten_subgroups=True`. The parameter will flatten subgroups" + " in the raised exceptiongroup before matching, which would never" + " match a nested structure.", + ) + self.is_baseexceptiongroup |= exc.is_baseexceptiongroup + exc._nested = True + elif isinstance(exc, Matcher): + if exc.exception_type is not None: + # Matcher __init__ assures it's a subclass of BaseException + self.is_baseexceptiongroup |= not issubclass( + exc.exception_type, + Exception, + ) + exc._nested = True + elif isinstance(exc, type) and issubclass(exc, BaseException): + self.is_baseexceptiongroup |= not issubclass(exc, Exception) + else: + raise TypeError( + f'Invalid argument "{exc!r}" must be exception type, Matcher, or' + " RaisesGroup.", + ) + + @overload + def __enter__( + self: RaisesGroup[ExcT_1], + ) -> ExceptionInfo[ExceptionGroup[ExcT_1]]: ... + @overload + def __enter__( + self: RaisesGroup[BaseExcT_1], + ) -> ExceptionInfo[BaseExceptionGroup[BaseExcT_1]]: ... + + def __enter__(self) -> ExceptionInfo[BaseExceptionGroup[BaseException]]: + self.excinfo: ExceptionInfo[BaseExceptionGroup[BaseExcT_co]] = ( + ExceptionInfo.for_later() + ) + return self.excinfo + + def __repr__(self) -> str: + reqs = [ + e.__name__ if isinstance(e, type) else repr(e) + for e in self.expected_exceptions + ] + if self.allow_unwrapped: + reqs.append(f"allow_unwrapped={self.allow_unwrapped}") + if self.flatten_subgroups: + reqs.append(f"flatten_subgroups={self.flatten_subgroups}") + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + reqs.append(f"match={_match_pattern(self.match)!r}") + if self.check is not None: + reqs.append(f"check={repr_callable(self.check)}") + return f"RaisesGroup({', '.join(reqs)})" + + def _unroll_exceptions( + self, + exceptions: Sequence[BaseException], + ) -> Sequence[BaseException]: + """Used if `flatten_subgroups=True`.""" + res: list[BaseException] = [] + for exc in exceptions: + if isinstance(exc, BaseExceptionGroup): + res.extend(self._unroll_exceptions(exc.exceptions)) + + else: + res.append(exc) + return res + + @overload + def matches( + self: RaisesGroup[ExcT_1], + exc_val: BaseException | None, + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def matches( + self: RaisesGroup[BaseExcT_1], + exc_val: BaseException | None, + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def matches( + self, + exc_val: BaseException | None, + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_co]]: + """Check if an exception matches the requirements of this RaisesGroup. + If it fails, `RaisesGroup.fail_reason` will be set. + + Example:: + + with pytest.raises(TypeError) as excinfo: + ... + assert RaisesGroups(ValueError).matches(excinfo.value.__cause__) + # the above line is equivalent to + myexc = excinfo.value.__cause + assert isinstance(myexc, BaseExceptionGroup) + assert len(myexc.exceptions) == 1 + assert isinstance(myexc.exceptions[0], ValueError) + """ + self._fail_reason = None + if exc_val is None: + self._fail_reason = "exception is None" + return False + if not isinstance(exc_val, BaseExceptionGroup): + # we opt to only print type of the exception here, as the repr would + # likely be quite long + not_group_msg = f"{type(exc_val).__name__!r} is not an exception group" + if len(self.expected_exceptions) > 1: + self._fail_reason = not_group_msg + return False + # if we have 1 expected exception, check if it would work even if + # allow_unwrapped is not set + res = self._check_expected(self.expected_exceptions[0], exc_val) + if res is None and self.allow_unwrapped: + return True + + if res is None: + self._fail_reason = ( + f"{not_group_msg}, but would match with `allow_unwrapped=True`" + ) + elif self.allow_unwrapped: + self._fail_reason = res + else: + self._fail_reason = not_group_msg + return False + + actual_exceptions: Sequence[BaseException] = exc_val.exceptions + if self.flatten_subgroups: + actual_exceptions = self._unroll_exceptions(actual_exceptions) + + if not self._check_match(exc_val): + self._fail_reason = cast(str, self._fail_reason) + old_reason = self._fail_reason + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + and isinstance(actual := actual_exceptions[0], expected) + and self._check_match(actual) + ): + assert self.match is not None, "can't be None if _check_match failed" + assert self._fail_reason is old_reason is not None + self._fail_reason += ( + f", but matched the expected {self._repr_expected(expected)}." + f" You might want RaisesGroup(Matcher({expected.__name__}, match={_match_pattern(self.match)!r}))" + ) + else: + self._fail_reason = old_reason + return False + + # do the full check on expected exceptions + if not self._check_exceptions( + exc_val, + actual_exceptions, + ): + self._fail_reason = cast(str, self._fail_reason) + assert self._fail_reason is not None + old_reason = self._fail_reason + # if we're not expecting a nested structure, and there is one, do a second + # pass where we try flattening it + if ( + not self.flatten_subgroups + and not any( + isinstance(e, RaisesGroup) for e in self.expected_exceptions + ) + and any(isinstance(e, BaseExceptionGroup) for e in actual_exceptions) + and self._check_exceptions( + exc_val, + self._unroll_exceptions(exc_val.exceptions), + ) + ): + # only indent if it's a single-line reason. In a multi-line there's already + # indented lines that this does not belong to. + indent = " " if "\n" not in self._fail_reason else "" + self._fail_reason = ( + old_reason + + f"\n{indent}Did you mean to use `flatten_subgroups=True`?" + ) + else: + self._fail_reason = old_reason + return False + + # Only run `self.check` once we know `exc_val` is of the correct type. + if not self._check_check(exc_val): + reason = cast(str, self._fail_reason) + f" on the {type(exc_val).__name__}" + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + # we explicitly break typing here :) + and self._check_check(actual_exceptions[0]) # type: ignore[arg-type] + ): + self._fail_reason = reason + ( + f", but did return True for the expected {self._repr_expected(expected)}." + f" You might want RaisesGroup(Matcher({expected.__name__}, check=<...>))" + ) + else: + self._fail_reason = reason + return False + + return True + + @staticmethod + def _check_expected( + expected_type: ( + type[BaseException] | Matcher[BaseException] | RaisesGroup[BaseException] + ), + exception: BaseException, + ) -> str | None: + """Helper method for `RaisesGroup.matches` and `RaisesGroup._check_exceptions` + to check one of potentially several expected exceptions.""" + if isinstance(expected_type, type): + return _check_raw_type(expected_type, exception) + res = expected_type.matches(exception) + if res: + return None + assert expected_type.fail_reason is not None + if expected_type.fail_reason.startswith("\n"): + return f"\n{expected_type!r}: {indent(expected_type.fail_reason, ' ')}" + return f"{expected_type!r}: {expected_type.fail_reason}" + + @staticmethod + def _repr_expected(e: type[BaseException] | AbstractMatcher[BaseException]) -> str: + """Get the repr of an expected type/Matcher/RaisesGroup, but we only want + the name if it's a type""" + if isinstance(e, type): + return _exception_type_name(e) + return repr(e) + + @overload + def _check_exceptions( + self: RaisesGroup[ExcT_1], + _exc_val: Exception, + actual_exceptions: Sequence[Exception], + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def _check_exceptions( + self: RaisesGroup[BaseExcT_1], + _exc_val: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def _check_exceptions( + self, + _exc_val: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_co]]: + """Helper method for RaisesGroup.matches that attempts to pair up expected and actual exceptions""" + # full table with all results + results = ResultHolder(self.expected_exceptions, actual_exceptions) + + # (indexes of) raised exceptions that haven't (yet) found an expected + remaining_actual = list(range(len(actual_exceptions))) + # (indexes of) expected exceptions that haven't found a matching raised + failed_expected: list[int] = [] + # successful greedy matches + matches: dict[int, int] = {} + + # loop over expected exceptions first to get a more predictable result + for i_exp, expected in enumerate(self.expected_exceptions): + for i_rem in remaining_actual: + res = self._check_expected(expected, actual_exceptions[i_rem]) + results.set_result(i_exp, i_rem, res) + if res is None: + remaining_actual.remove(i_rem) + matches[i_exp] = i_rem + break + else: + failed_expected.append(i_exp) + + # All exceptions matched up successfully + if not remaining_actual and not failed_expected: + return True + + # in case of a single expected and single raised we simplify the output + if 1 == len(actual_exceptions) == len(self.expected_exceptions): + assert not matches + self._fail_reason = res + return False + + # The test case is failing, so we can do a slow and exhaustive check to find + # duplicate matches etc that will be helpful in debugging + for i_exp, expected in enumerate(self.expected_exceptions): + for i_actual, actual in enumerate(actual_exceptions): + if results.has_result(i_exp, i_actual): + continue + results.set_result( + i_exp, i_actual, self._check_expected(expected, actual) + ) + + successful_str = ( + f"{len(matches)} matched exception{'s' if len(matches) > 1 else ''}. " + if matches + else "" + ) + + # all expected were found + if not failed_expected and results.no_match_for_actual(remaining_actual): + self._fail_reason = ( + f"{successful_str}Unexpected exception(s):" + f" {[actual_exceptions[i] for i in remaining_actual]!r}" + ) + return False + # all raised exceptions were expected + if not remaining_actual and results.no_match_for_expected(failed_expected): + no_match_for_str = ", ".join( + self._repr_expected(self.expected_exceptions[i]) + for i in failed_expected + ) + self._fail_reason = f"{successful_str}Too few exceptions raised, found no match for: [{no_match_for_str}]" + return False + + # if there's only one remaining and one failed, and the unmatched didn't match anything else, + # we elect to only print why the remaining and the failed didn't match. + if ( + 1 == len(remaining_actual) == len(failed_expected) + and results.no_match_for_actual(remaining_actual) + and results.no_match_for_expected(failed_expected) + ): + self._fail_reason = f"{successful_str}{results.get_result(failed_expected[0], remaining_actual[0])}" + return False + + # there's both expected and raised exceptions without matches + s = "" + if matches: + s += f"\n{successful_str}" + indent_1 = " " * 2 + indent_2 = " " * 4 + + if not remaining_actual: + s += "\nToo few exceptions raised!" + elif not failed_expected: + s += "\nUnexpected exception(s)!" + + if failed_expected: + s += "\nThe following expected exceptions did not find a match:" + rev_matches = {v: k for k, v in matches.items()} + for i_failed in failed_expected: + s += ( + f"\n{indent_1}{self._repr_expected(self.expected_exceptions[i_failed])}" + ) + for i_actual, actual in enumerate(actual_exceptions): + if results.get_result(i_exp, i_actual) is None: + # we print full repr of match target + s += ( + f"\n{indent_2}It matches {actual!r} which was paired with " + + self._repr_expected( + self.expected_exceptions[rev_matches[i_actual]] + ) + ) + + if remaining_actual: + s += "\nThe following raised exceptions did not find a match" + for i_actual in remaining_actual: + s += f"\n{indent_1}{actual_exceptions[i_actual]!r}:" + for i_exp, expected in enumerate(self.expected_exceptions): + res = results.get_result(i_exp, i_actual) + if i_exp in failed_expected: + assert res is not None + if res[0] != "\n": + s += "\n" + s += indent(res, indent_2) + if res is None: + # we print full repr of match target + s += ( + f"\n{indent_2}It matches {self._repr_expected(expected)} " + f"which was paired with {actual_exceptions[matches[i_exp]]!r}" + ) + + if len(self.expected_exceptions) == len(actual_exceptions) and possible_match( + results + ): + s += ( + "\nThere exist a possible match when attempting an exhaustive check," + " but RaisesGroup uses a greedy algorithm. " + "Please make your expected exceptions more stringent with `Matcher` etc" + " so the greedy algorithm can function." + ) + self._fail_reason = s + return False + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> bool: + __tracebackhide__ = True + if exc_type is None: + fail(f"DID NOT RAISE any exception, expected {self.expected_type()}") + + assert self.excinfo is not None, ( + "Internal error - should have been constructed in __enter__" + ) + + group_str = ( + "(group)" + if self.allow_unwrapped and not issubclass(exc_type, BaseExceptionGroup) + else "group" + ) + + if not self.matches(exc_val): + fail(f"Raised exception {group_str} did not match: {self._fail_reason}") + + # Cast to narrow the exception type now that it's verified.... + # even though the TypeGuard in self.matches should be narrowing + exc_info = cast( + "tuple[type[BaseExceptionGroup[BaseExcT_co]], BaseExceptionGroup[BaseExcT_co], types.TracebackType]", + (exc_type, exc_val, exc_tb), + ) + self.excinfo.fill_unfilled(exc_info) + return True + + def expected_type(self) -> str: + subexcs = [] + for e in self.expected_exceptions: + if isinstance(e, Matcher): + subexcs.append(str(e)) + elif isinstance(e, RaisesGroup): + subexcs.append(e.expected_type()) + elif isinstance(e, type): + subexcs.append(e.__name__) + else: # pragma: no cover + raise AssertionError("unknown type") + group_type = "Base" if self.is_baseexceptiongroup else "" + return f"{group_type}ExceptionGroup({', '.join(subexcs)})" + + +@final +class NotChecked: + """Singleton for unchecked values in ResultHolder""" + + +class ResultHolder: + """Container for results of checking exceptions. + Used in RaisesGroup._check_exceptions and possible_match. + """ + + def __init__( + self, + expected_exceptions: tuple[ + type[BaseException] | AbstractMatcher[BaseException], ... + ], + actual_exceptions: Sequence[BaseException], + ) -> None: + self.results: list[list[str | type[NotChecked] | None]] = [ + [NotChecked for _ in expected_exceptions] for _ in actual_exceptions + ] + + def set_result(self, expected: int, actual: int, result: str | None) -> None: + self.results[actual][expected] = result + + def get_result(self, expected: int, actual: int) -> str | None: + res = self.results[actual][expected] + assert res is not NotChecked + # mypy doesn't support identity checking against anything but None + return res # type: ignore[return-value] + + def has_result(self, expected: int, actual: int) -> bool: + return self.results[actual][expected] is not NotChecked + + def no_match_for_expected(self, expected: list[int]) -> bool: + for i in expected: + for actual_results in self.results: + assert actual_results[i] is not NotChecked + if actual_results[i] is None: + return False + return True + + def no_match_for_actual(self, actual: list[int]) -> bool: + for i in actual: + for res in self.results[i]: + assert res is not NotChecked + if res is None: + return False + return True + + +def possible_match(results: ResultHolder, used: set[int] | None = None) -> bool: + if used is None: + used = set() + curr_row = len(used) + if curr_row == len(results.results): + return True + + for i, val in enumerate(results.results[curr_row]): + if val is None and i not in used and possible_match(results, used | {i}): + return True + return False diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index 70096d6593e..ca2c66fee03 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -6,6 +6,9 @@ from _pytest import __version__ from _pytest import version_tuple from _pytest._code import ExceptionInfo +from _pytest._raises_group import Matcher +from _pytest._raises_group import RaisesGroup +from _pytest._raises_group import RaisesGroup as raises_group from _pytest.assertion import register_assert_rewrite from _pytest.cacheprovider import Cache from _pytest.capture import CaptureFixture @@ -113,6 +116,7 @@ "Mark", "MarkDecorator", "MarkGenerator", + "Matcher", "Metafunc", "Module", "MonkeyPatch", @@ -133,6 +137,7 @@ "PytestUnraisableExceptionWarning", "PytestWarning", "Pytester", + "RaisesGroup", "RecordedHookCall", "RunResult", "Session", @@ -162,6 +167,7 @@ "mark", "param", "raises", + "raises_group", "register_assert_rewrite", "set_trace", "skip", diff --git a/testing/python/raises_group.py b/testing/python/raises_group.py new file mode 100644 index 00000000000..2ba613ba9b5 --- /dev/null +++ b/testing/python/raises_group.py @@ -0,0 +1,1119 @@ +from __future__ import annotations + +# several expected multi-line strings contain long lines. We don't wanna break them up +# as that makes it confusing to see where the line breaks are. +# ruff: noqa: E501 +import re +import sys +from typing import TYPE_CHECKING + +from _pytest._raises_group import Matcher +from _pytest._raises_group import RaisesGroup +from _pytest._raises_group import repr_callable +from _pytest.outcomes import Failed +import pytest + + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + +if TYPE_CHECKING: + from _pytest.python_api import RaisesContext + + +def wrap_escape(s: str) -> str: + return "^" + re.escape(s) + "$" + + +def fails_raises_group(msg: str, add_prefix: bool = True) -> RaisesContext[Failed]: + assert msg[-1] != "\n", ( + "developer error, expected string should not end with newline" + ) + prefix = "Raised exception group did not match: " if add_prefix else "" + return pytest.raises(Failed, match=wrap_escape(prefix + msg)) + + +def test_raises_group() -> None: + with pytest.raises( + TypeError, + match=wrap_escape( + f'Invalid argument "{ValueError()!r}" must be exception type, Matcher, or RaisesGroup.', + ), + ): + RaisesGroup(ValueError()) # type: ignore[call-overload] + with RaisesGroup(ValueError): + raise ExceptionGroup("foo", (ValueError(),)) + + with ( + fails_raises_group("'SyntaxError' is not of type 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("foo", (SyntaxError(),)) + + # multiple exceptions + with RaisesGroup(ValueError, SyntaxError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # order doesn't matter + with RaisesGroup(SyntaxError, ValueError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # nested exceptions + with RaisesGroup(RaisesGroup(ValueError)): + raise ExceptionGroup("foo", (ExceptionGroup("bar", (ValueError(),)),)) + + with RaisesGroup( + SyntaxError, + RaisesGroup(ValueError), + RaisesGroup(RuntimeError), + ): + raise ExceptionGroup( + "foo", + ( + SyntaxError(), + ExceptionGroup("bar", (ValueError(),)), + ExceptionGroup("", (RuntimeError(),)), + ), + ) + + +def test_incorrect_number_exceptions() -> None: + # We previously gave an error saying the number of exceptions was wrong, + # but we now instead indicate excess/missing exceptions + with ( + fails_raises_group( + "1 matched exception. Unexpected exception(s): [RuntimeError()]" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (RuntimeError(), ValueError())) + + # will error if there's missing exceptions + with ( + fails_raises_group( + "1 matched exception. Too few exceptions raised, found no match for: ['SyntaxError']" + ), + RaisesGroup(ValueError, SyntaxError), + ): + raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Too few exceptions raised!\n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Unexpected exception(s)!\n" + "The following raised exceptions did not find a match\n" + " ValueError():\n" + " It matches 'ValueError' which was paired with ValueError()" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (ValueError(), ValueError())) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " SyntaxError():\n" + " 'SyntaxError' is not of type 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), SyntaxError()]) + + +def test_flatten_subgroups() -> None: + # loose semantics, as with expect* + with RaisesGroup(ValueError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + with RaisesGroup(ValueError, TypeError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(), TypeError())),)) + with RaisesGroup(ValueError, TypeError, flatten_subgroups=True): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()]), TypeError()]) + + # mixed loose is possible if you want it to be at least N deep + with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): + raise ExceptionGroup( + "", + (ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)),), + ) + + # but not the other way around + with pytest.raises( + ValueError, + match=r"^You cannot specify a nested structure inside a RaisesGroup with", + ): + RaisesGroup(RaisesGroup(ValueError), flatten_subgroups=True) # type: ignore[call-overload] + + # flatten_subgroups is not sufficient to catch fully unwrapped + with ( + fails_raises_group( + "'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError, flatten_subgroups=True), + ): + raise ValueError + with ( + fails_raises_group( + "RaisesGroup(ValueError, flatten_subgroups=True): 'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)), + ): + raise ExceptionGroup("", (ValueError(),)) + + # helpful suggestion if flatten_subgroups would make it pass + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'TypeError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'TypeError'\n" + "Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + # but doesn't consider check (otherwise we'd break typing guarantees) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'TypeError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'TypeError'\n" + "Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup( + ValueError, + TypeError, + check=lambda eg: len(eg.exceptions) == 1, + ), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + # correct number of exceptions, and flatten_subgroups would make it pass + # This now doesn't print a repr of the caught exception at all, but that can be found in the traceback + with ( + fails_raises_group( + "Raised exception group did not match: Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + # correct number of exceptions, but flatten_subgroups wouldn't help, so we don't suggest it + with ( + fails_raises_group("Unexpected nested 'ExceptionGroup', expected 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [TypeError()])]) + + # flatten_subgroups can be suggested if nested. This will implicitly ask the user to + # do `RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True))` which is unlikely + # to be what they actually want - but I don't think it's worth trying to special-case + with ( + fails_raises_group( + "RaisesGroup(ValueError): Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + + # Don't mention "unexpected nested" if expecting an ExceptionGroup. + # Although it should perhaps be an error to specify `RaisesGroup(ExceptionGroup)` in + # favor of doing `RaisesGroup(RaisesGroup(...))`. + with ( + fails_raises_group("'BaseExceptionGroup' is not of type 'ExceptionGroup'"), + RaisesGroup(ExceptionGroup), + ): + raise BaseExceptionGroup("", [BaseExceptionGroup("", [KeyboardInterrupt()])]) + + +def test_catch_unwrapped_exceptions() -> None: + # Catches lone exceptions with strict=False + # just as except* would + with RaisesGroup(ValueError, allow_unwrapped=True): + raise ValueError + + # expecting multiple unwrapped exceptions is not possible + with pytest.raises( + ValueError, + match=r"^You cannot specify multiple exceptions with", + ): + RaisesGroup(SyntaxError, ValueError, allow_unwrapped=True) # type: ignore[call-overload] + # if users want one of several exception types they need to use a Matcher + # (which the error message suggests) + with RaisesGroup( + Matcher(check=lambda e: isinstance(e, (SyntaxError, ValueError))), + allow_unwrapped=True, + ): + raise ValueError + + # Unwrapped nested `RaisesGroup` is likely a user error, so we raise an error. + with pytest.raises(ValueError, match="has no effect when expecting"): + RaisesGroup(RaisesGroup(ValueError), allow_unwrapped=True) # type: ignore[call-overload] + + # But it *can* be used to check for nesting level +- 1 if they move it to + # the nested RaisesGroup. Users should probably use `Matcher`s instead though. + with RaisesGroup(RaisesGroup(ValueError, allow_unwrapped=True)): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + with RaisesGroup(RaisesGroup(ValueError, allow_unwrapped=True)): + raise ExceptionGroup("", [ValueError()]) + + # with allow_unwrapped=False (default) it will not be caught + with ( + fails_raises_group( + "'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError), + ): + raise ValueError("value error text") + + # allow_unwrapped on its own won't match against nested groups + with ( + fails_raises_group( + "Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise ExceptionGroup("foo", [ExceptionGroup("bar", [ValueError()])]) + + # you need both allow_unwrapped and flatten_subgroups to fully emulate except* + with RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + + # code coverage + with ( + fails_raises_group( + "Raised exception (group) did not match: 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise TypeError("this text doesn't show up in the error message") + with ( + fails_raises_group( + "Raised exception (group) did not match: Matcher(ValueError): 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup(Matcher(ValueError), allow_unwrapped=True), + ): + raise TypeError + + # check we don't suggest unwrapping with nested RaisesGroup + with ( + fails_raises_group("'ValueError' is not an exception group"), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ValueError + + +def test_match() -> None: + # supports match string + with RaisesGroup(ValueError, match="bar"): + raise ExceptionGroup("bar", (ValueError(),)) + + # now also works with ^$ + with RaisesGroup(ValueError, match="^bar$"): + raise ExceptionGroup("bar", (ValueError(),)) + + # it also includes notes + with RaisesGroup(ValueError, match="my note"): + e = ExceptionGroup("bar", (ValueError(),)) + e.add_note("my note") + raise e + + # and technically you can match it all with ^$ + # but you're probably better off using a Matcher at that point + with RaisesGroup(ValueError, match="^bar\nmy note$"): + e = ExceptionGroup("bar", (ValueError(),)) + e.add_note("my note") + raise e + + with ( + fails_raises_group( + "Regex pattern 'foo' did not match 'bar' of 'ExceptionGroup'" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", (ValueError(),)) + + # Suggest a fix for easy pitfall of adding match to the RaisesGroup instead of + # using a Matcher. + # This requires a single expected & raised exception, the expected is a type, + # and `isinstance(raised, expected_type)`. + with ( + fails_raises_group( + "Regex pattern 'foo' did not match 'bar' of 'ExceptionGroup', but matched the expected 'ValueError'. You might want RaisesGroup(Matcher(ValueError, match='foo'))" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", [ValueError("foo")]) + + +def test_check() -> None: + exc = ExceptionGroup("", (ValueError(),)) + + def is_exc(e: ExceptionGroup[ValueError]) -> bool: + return e is exc + + is_exc_repr = repr_callable(is_exc) + with RaisesGroup(ValueError, check=is_exc): + raise exc + + with ( + fails_raises_group( + f"check {is_exc_repr} did not return True on the ExceptionGroup" + ), + RaisesGroup(ValueError, check=is_exc), + ): + raise ExceptionGroup("", (ValueError(),)) + + def is_value_error(e: BaseException) -> bool: + return isinstance(e, ValueError) + + # helpful suggestion if the user thinks the check is for the sub-exception + with ( + fails_raises_group( + f"check {is_value_error} did not return True on the ExceptionGroup, but did return True for the expected 'ValueError'. You might want RaisesGroup(Matcher(ValueError, check=<...>))" + ), + RaisesGroup(ValueError, check=is_value_error), + ): + raise ExceptionGroup("", (ValueError(),)) + + +def test_unwrapped_match_check() -> None: + def my_check(e: object) -> bool: # pragma: no cover + return True + + msg = ( + "`allow_unwrapped=True` bypasses the `match` and `check` parameters" + " if the exception is unwrapped. If you intended to match/check the" + " exception you should use a `Matcher` object. If you want to match/check" + " the exceptiongroup when the exception *is* wrapped you need to" + " do e.g. `if isinstance(exc.value, ExceptionGroup):" + " assert RaisesGroup(...).matches(exc.value)` afterwards." + ) + with pytest.raises(ValueError, match=re.escape(msg)): + RaisesGroup(ValueError, allow_unwrapped=True, match="foo") # type: ignore[call-overload] + with pytest.raises(ValueError, match=re.escape(msg)): + RaisesGroup(ValueError, allow_unwrapped=True, check=my_check) # type: ignore[call-overload] + + # Users should instead use a Matcher + rg = RaisesGroup(Matcher(ValueError, match="^foo$"), allow_unwrapped=True) + with rg: + raise ValueError("foo") + with rg: + raise ExceptionGroup("", [ValueError("foo")]) + + # or if they wanted to match/check the group, do a conditional `.matches()` + with RaisesGroup(ValueError, allow_unwrapped=True) as exc: + raise ExceptionGroup("bar", [ValueError("foo")]) + if isinstance(exc.value, ExceptionGroup): # pragma: no branch + assert RaisesGroup(ValueError, match="bar").matches(exc.value) + + +def test_RaisesGroup_matches() -> None: + rg = RaisesGroup(ValueError) + assert not rg.matches(None) + assert not rg.matches(ValueError()) + assert rg.matches(ExceptionGroup("", (ValueError(),))) + + +def test_message() -> None: + def check_message( + message: str, + body: RaisesGroup[BaseException], + ) -> None: + with ( + pytest.raises( + Failed, + match=f"^DID NOT RAISE any exception, expected {re.escape(message)}$", + ), + body, + ): + ... + + # basic + check_message("ExceptionGroup(ValueError)", RaisesGroup(ValueError)) + # multiple exceptions + check_message( + "ExceptionGroup(ValueError, ValueError)", + RaisesGroup(ValueError, ValueError), + ) + # nested + check_message( + "ExceptionGroup(ExceptionGroup(ValueError))", + RaisesGroup(RaisesGroup(ValueError)), + ) + + # Matcher + check_message( + "ExceptionGroup(Matcher(ValueError, match='my_str'))", + RaisesGroup(Matcher(ValueError, "my_str")), + ) + check_message( + "ExceptionGroup(Matcher(match='my_str'))", + RaisesGroup(Matcher(match="my_str")), + ) + + # BaseExceptionGroup + check_message( + "BaseExceptionGroup(KeyboardInterrupt)", + RaisesGroup(KeyboardInterrupt), + ) + # BaseExceptionGroup with type inside Matcher + check_message( + "BaseExceptionGroup(Matcher(KeyboardInterrupt))", + RaisesGroup(Matcher(KeyboardInterrupt)), + ) + # Base-ness transfers to parent containers + check_message( + "BaseExceptionGroup(BaseExceptionGroup(KeyboardInterrupt))", + RaisesGroup(RaisesGroup(KeyboardInterrupt)), + ) + # but not to child containers + check_message( + "BaseExceptionGroup(BaseExceptionGroup(KeyboardInterrupt), ExceptionGroup(ValueError))", + RaisesGroup(RaisesGroup(KeyboardInterrupt), RaisesGroup(ValueError)), + ) + + +def test_assert_message() -> None: + # the message does not need to list all parameters to RaisesGroup, nor all exceptions + # in the exception group, as those are both visible in the traceback. + # first fails to match + with ( + fails_raises_group("'TypeError' is not of type 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("a", [TypeError()]) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(ValueError, match='a')\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [RuntimeError()]):\n" + " RaisesGroup(ValueError): 'RuntimeError' is not of type 'ValueError'\n" + " RaisesGroup(ValueError, match='a'): Regex pattern 'a' did not match '' of 'ExceptionGroup'\n" + " RuntimeError():\n" + " RaisesGroup(ValueError): 'RuntimeError' is not an exception group\n" + " RaisesGroup(ValueError, match='a'): 'RuntimeError' is not an exception group", + add_prefix=False, # to see the full structure + ), + RaisesGroup(RaisesGroup(ValueError), RaisesGroup(ValueError, match="a")), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [RuntimeError()]), RuntimeError()], + ) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "2 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(RuntimeError)\n" + " RaisesGroup(ValueError)\n" + "The following raised exceptions did not find a match\n" + " RuntimeError():\n" + # " 'RuntimeError' is not of type 'ValueError'\n" + # " Matcher(TypeError): 'RuntimeError' is not of type 'TypeError'\n" + " RaisesGroup(RuntimeError): 'RuntimeError' is not an exception group, but would match with `allow_unwrapped=True`\n" + " RaisesGroup(ValueError): 'RuntimeError' is not an exception group\n" + " ValueError('bar'):\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " RaisesGroup(RuntimeError): 'ValueError' is not an exception group\n" + " RaisesGroup(ValueError): 'ValueError' is not an exception group, but would match with `allow_unwrapped=True`", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + ValueError, + Matcher(TypeError), + RaisesGroup(RuntimeError), + RaisesGroup(ValueError), + ), + ): + raise ExceptionGroup( + "a", + [RuntimeError(), TypeError(), ValueError("foo"), ValueError("bar")], + ) + + with ( + fails_raises_group( + "1 matched exception. 'AssertionError' is not of type 'TypeError'" + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("a", [ValueError(), AssertionError()]) + + with ( + fails_raises_group( + "Matcher(ValueError): 'TypeError' is not of type 'ValueError'" + ), + RaisesGroup(Matcher(ValueError)), + ): + raise ExceptionGroup("a", [TypeError()]) + + # suggest escaping + with ( + fails_raises_group( + "Raised exception group did not match: Regex pattern 'h(ell)o' did not match 'h(ell)o' of 'ExceptionGroup'\n" + " Did you mean to `re.escape()` the regex?", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, match="h(ell)o"), + ): + raise ExceptionGroup("h(ell)o", [ValueError()]) + with ( + fails_raises_group( + "Matcher(match='h(ell)o'): Regex pattern 'h(ell)o' did not match 'h(ell)o'\n" + " Did you mean to `re.escape()` the regex?", + ), + RaisesGroup(Matcher(match="h(ell)o")), + ): + raise ExceptionGroup("", [ValueError("h(ell)o")]) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " 'ValueError'\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, ValueError, ValueError, ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + + +def test_message_indent() -> None: + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [TypeError(), RuntimeError()]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'\n" + " 'TypeError' is not of type 'ValueError'\n" + " RuntimeError():\n" + " 'RuntimeError' is not of type 'ValueError'\n" + " 'RuntimeError' is not of type 'ValueError'\n" + # TODO: this line is not great, should maybe follow the same format as the other and say + # 'ValueError': Unexpected nested 'ExceptionGroup' (?) + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " TypeError():\n" + " RaisesGroup(ValueError, ValueError): 'TypeError' is not an exception group\n" + " 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ValueError, + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + TypeError(), + ], + ) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'\n" + " 'TypeError' is not of type 'ValueError'\n" + " RuntimeError():\n" + " 'RuntimeError' is not of type 'ValueError'\n" + " 'RuntimeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + ], + ) + + +def test_suggestion_on_nested_and_brief_error() -> None: + # Make sure "Did you mean" suggestion gets indented iff it follows a single-line error + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError): Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'", + ), + RaisesGroup(RaisesGroup(ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + # if indented here it would look like another raised exception + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'" + ), + RaisesGroup(RaisesGroup(ValueError, ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ValueError(), ExceptionGroup("", [ValueError()])])], + ) + + # re.escape always comes after single-line errors + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(Exception, match='^hello')\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('^hello', [Exception()]):\n" + " RaisesGroup(Exception, match='^hello'): Regex pattern '^hello' did not match '^hello' of 'ExceptionGroup'\n" + " Did you mean to `re.escape()` the regex?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'" + ), + RaisesGroup(RaisesGroup(Exception, match="^hello"), ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("^hello", [Exception()])]) + + +def test_assert_message_nested() -> None: + # we only get one instance of aaaaaaaaaa... and bbbbbb..., but we do get multiple instances of ccccc... and dddddd.. + # but I think this now only prints the full repr when that is necessary to disambiguate exceptions + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(RaisesGroup(ValueError))\n" + " RaisesGroup(Matcher(TypeError, match='foo'))\n" + " RaisesGroup(TypeError, ValueError)\n" + "The following raised exceptions did not find a match\n" + " TypeError('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(RaisesGroup(ValueError)): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): 'TypeError' is not an exception group\n" + " RaisesGroup(TypeError, ValueError): 'TypeError' is not an exception group\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')]):\n" + " RaisesGroup(ValueError): 'TypeError' is not of type 'ValueError'\n" + " RaisesGroup(RaisesGroup(ValueError)): RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'\n" + " RaisesGroup(TypeError, ValueError): 1 matched exception. Too few exceptions raised, found no match for: ['ValueError']\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('cccccccccccccccccccccccccccccc'), TypeError('dddddddddddddddddddddddddddddd')]):\n" + " RaisesGroup(ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " 'TypeError' is not of type 'ValueError'\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " 'TypeError' is not of type 'ValueError'\n" + " RaisesGroup(RaisesGroup(ValueError)): \n" + " The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): \n" + " The following expected exceptions did not find a match:\n" + " Matcher(TypeError, match='foo')\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'cccccccccccccccccccccccccccccc'\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'dddddddddddddddddddddddddddddd'\n" + " RaisesGroup(TypeError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " It matches 'TypeError' which was paired with TypeError('cccccccccccccccccccccccccccccc')\n" + " 'TypeError' is not of type 'ValueError'", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + RaisesGroup(ValueError), + RaisesGroup(RaisesGroup(ValueError)), + RaisesGroup(Matcher(TypeError, match="foo")), + RaisesGroup(TypeError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + TypeError("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ExceptionGroup( + "Exceptions from Trio nursery", + [TypeError("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")], + ), + ExceptionGroup( + "Exceptions from Trio nursery", + [ + TypeError("cccccccccccccccccccccccccccccc"), + TypeError("dddddddddddddddddddddddddddddd"), + ], + ), + ], + ) + + +@pytest.mark.skipif( + "hypothesis" in sys.modules, + reason="hypothesis may have monkeypatched _check_repr", +) +def test_check_no_patched_repr() -> None: + # We make `_check_repr` monkeypatchable to avoid this very ugly and verbose + # repr. The other tests that use `check` make use of `_check_repr` so they'll + # continue passing in case it is patched - but we have this one test that + # demonstrates just how nasty it gets otherwise. + match_str = ( + r"^Raised exception group did not match: \n" + r"The following expected exceptions did not find a match:\n" + r" Matcher\(check=. at .*>\)\n" + r" 'TypeError'\n" + r"The following raised exceptions did not find a match\n" + r" ValueError\('foo'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'\n" + r" ValueError\('bar'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'$" + ) + with ( + pytest.raises(Failed, match=match_str), + RaisesGroup(Matcher(check=lambda x: False), TypeError), + ): + raise ExceptionGroup("", [ValueError("foo"), ValueError("bar")]) + + +def test_misordering_example() -> None: + with ( + fails_raises_group( + "\n" + "3 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " Matcher(ValueError, match='foo')\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ValueError('bar'):\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " Matcher(ValueError, match='foo'): Regex pattern 'foo' did not match 'bar'\n" + "There exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `Matcher` etc so the greedy algorithm can function." + ), + RaisesGroup( + ValueError, ValueError, ValueError, Matcher(ValueError, match="foo") + ), + ): + raise ExceptionGroup( + "", + [ + ValueError("foo"), + ValueError("foo"), + ValueError("foo"), + ValueError("bar"), + ], + ) + + +def test_brief_error_on_one_fail() -> None: + """If only one raised and one expected fail to match up, we print a full table iff + the raised exception would match one of the expected that previously got matched""" + # no also-matched + with ( + fails_raises_group( + "1 matched exception. 'TypeError' is not of type 'RuntimeError'" + ), + RaisesGroup(ValueError, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # raised would match an expected + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'RuntimeError'\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " It matches 'Exception' which was paired with ValueError()\n" + " 'TypeError' is not of type 'RuntimeError'" + ), + RaisesGroup(Exception, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # expected would match a raised + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + +def test_identity_oopsies() -> None: + # it's both possible to have several instances of the same exception in the same group + # and to expect multiple of the same type + # this previously messed up the logic + + with ( + fails_raises_group( + "3 matched exceptions. 'RuntimeError' is not of type 'TypeError'" + ), + RaisesGroup(ValueError, ValueError, ValueError, TypeError), + ): + raise ExceptionGroup( + "", [ValueError(), ValueError(), ValueError(), RuntimeError()] + ) + + e = ValueError("foo") + m = Matcher(match="bar") + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " Matcher(match='bar')\n" + " Matcher(match='bar')\n" + " Matcher(match='bar')\n" + "The following raised exceptions did not find a match\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'" + ), + RaisesGroup(m, m, m), + ): + raise ExceptionGroup("", [e, e, e]) + + +def test_matcher() -> None: + with pytest.raises( + ValueError, + match=r"^You must specify at least one parameter to match on.$", + ): + Matcher() # type: ignore[call-overload] + with pytest.raises( + TypeError, + match=f"^exception_type {re.escape(repr(object))} must be a subclass of BaseException$", + ): + Matcher(object) # type: ignore[type-var] + + with RaisesGroup(Matcher(ValueError)): + raise ExceptionGroup("", (ValueError(),)) + with ( + fails_raises_group( + "Matcher(TypeError): 'ValueError' is not of type 'TypeError'" + ), + RaisesGroup(Matcher(TypeError)), + ): + raise ExceptionGroup("", (ValueError(),)) + + +def test_matcher_match() -> None: + with RaisesGroup(Matcher(ValueError, "foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + with ( + fails_raises_group( + "Matcher(ValueError, match='foo'): Regex pattern 'foo' did not match 'bar'" + ), + RaisesGroup(Matcher(ValueError, "foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) + + # Can be used without specifying the type + with RaisesGroup(Matcher(match="foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + with ( + fails_raises_group( + "Matcher(match='foo'): Regex pattern 'foo' did not match 'bar'" + ), + RaisesGroup(Matcher(match="foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) + + # check ^$ + with RaisesGroup(Matcher(ValueError, match="^bar$")): + raise ExceptionGroup("", [ValueError("bar")]) + with ( + fails_raises_group( + "Matcher(ValueError, match='^bar$'): Regex pattern '^bar$' did not match 'barr'" + ), + RaisesGroup(Matcher(ValueError, match="^bar$")), + ): + raise ExceptionGroup("", [ValueError("barr")]) + + +def test_Matcher_check() -> None: + def check_oserror_and_errno_is_5(e: BaseException) -> bool: + return isinstance(e, OSError) and e.errno == 5 + + with RaisesGroup(Matcher(check=check_oserror_and_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # specifying exception_type narrows the parameter type to the callable + def check_errno_is_5(e: OSError) -> bool: + return e.errno == 5 + + with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # avoid printing overly verbose repr multiple times + with ( + fails_raises_group( + f"Matcher(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(Matcher(OSError, check=check_errno_is_5)), + ): + raise ExceptionGroup("", (OSError(6, ""),)) + + # in nested cases you still get it multiple times though + # to address this you'd need logic in Matcher.__repr__ and RaisesGroup.__repr__ + with ( + fails_raises_group( + f"RaisesGroup(Matcher(OSError, check={check_errno_is_5!r})): Matcher(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(RaisesGroup(Matcher(OSError, check=check_errno_is_5))), + ): + raise ExceptionGroup("", [ExceptionGroup("", [OSError(6, "")])]) + + +def test_matcher_tostring() -> None: + assert str(Matcher(ValueError)) == "Matcher(ValueError)" + assert str(Matcher(match="[a-z]")) == "Matcher(match='[a-z]')" + pattern_no_flags = re.compile(r"noflag", 0) + assert str(Matcher(match=pattern_no_flags)) == "Matcher(match='noflag')" + pattern_flags = re.compile(r"noflag", re.IGNORECASE) + assert str(Matcher(match=pattern_flags)) == f"Matcher(match={pattern_flags!r})" + assert ( + str(Matcher(ValueError, match="re", check=bool)) + == f"Matcher(ValueError, match='re', check={bool!r})" + ) + + +def test_raisesgroup_tostring() -> None: + def check_str_and_repr(s: str) -> None: + evaled = eval(s) + assert s == str(evaled) == repr(evaled) + + check_str_and_repr("RaisesGroup(ValueError)") + check_str_and_repr("RaisesGroup(RaisesGroup(ValueError))") + check_str_and_repr("RaisesGroup(Matcher(ValueError))") + check_str_and_repr("RaisesGroup(ValueError, allow_unwrapped=True)") + check_str_and_repr("RaisesGroup(ValueError, match='aoeu')") + + assert ( + str(RaisesGroup(ValueError, match="[a-z]", check=bool)) + == f"RaisesGroup(ValueError, match='[a-z]', check={bool!r})" + ) diff --git a/testing/typing_raises_group.py b/testing/typing_raises_group.py new file mode 100644 index 00000000000..2dc35031dac --- /dev/null +++ b/testing/typing_raises_group.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import sys +from typing import Callable +from typing import Union + +from typing_extensions import assert_type + +from _pytest._raises_group import Matcher +from _pytest._raises_group import RaisesGroup + + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + +# split into functions to isolate the different scopes + + +def check_matcher_typevar_default(e: Matcher) -> None: + assert e.exception_type is not None + _exc: type[BaseException] = e.exception_type + # this would previously pass, as the type would be `Any` + e.exception_type().blah() # type: ignore + + +def check_basic_contextmanager() -> None: + with RaisesGroup(ValueError) as e: + raise ExceptionGroup("foo", (ValueError(),)) + assert_type(e.value, ExceptionGroup[ValueError]) + + +def check_basic_matches() -> None: + # check that matches gets rid of the naked ValueError in the union + exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup("", (ValueError(),)) + if RaisesGroup(ValueError).matches(exc): + assert_type(exc, ExceptionGroup[ValueError]) + + # also check that BaseExceptionGroup shows up for BaseExceptions + if RaisesGroup(KeyboardInterrupt).matches(exc): + assert_type(exc, BaseExceptionGroup[KeyboardInterrupt]) + + +def check_matches_with_different_exception_type() -> None: + e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( + "", + (KeyboardInterrupt(),), + ) + + # note: it might be tempting to have this warn. + # however, that isn't possible with current typing + if RaisesGroup(ValueError).matches(e): + assert_type(e, ExceptionGroup[ValueError]) + + +def check_matcher_init() -> None: + def check_exc(exc: BaseException) -> bool: + return isinstance(exc, ValueError) + + # Check various combinations of constructor signatures. + # At least 1 arg must be provided. + Matcher() # type: ignore + Matcher(ValueError) + Matcher(ValueError, "regex") + Matcher(ValueError, "regex", check_exc) + Matcher(exception_type=ValueError) + Matcher(match="regex") + Matcher(check=check_exc) + Matcher(ValueError, match="regex") + Matcher(match="regex", check=check_exc) + + def check_filenotfound(exc: FileNotFoundError) -> bool: + return not exc.filename.endswith(".tmp") + + # If exception_type is provided, that narrows the `check` method's argument. + Matcher(FileNotFoundError, check=check_filenotfound) + Matcher(ValueError, check=check_filenotfound) # type: ignore + Matcher(check=check_filenotfound) # type: ignore + Matcher(FileNotFoundError, match="regex", check=check_filenotfound) + + +def raisesgroup_check_type_narrowing() -> None: + """Check type narrowing on the `check` argument to `RaisesGroup`. + All `type: ignore`s are correctly pointing out type errors. + """ + + def handle_exc(e: BaseExceptionGroup[BaseException]) -> bool: + return True + + def handle_kbi(e: BaseExceptionGroup[KeyboardInterrupt]) -> bool: + return True + + def handle_value(e: BaseExceptionGroup[ValueError]) -> bool: + return True + + RaisesGroup(BaseException, check=handle_exc) + RaisesGroup(BaseException, check=handle_kbi) # type: ignore + + RaisesGroup(Exception, check=handle_exc) + RaisesGroup(Exception, check=handle_value) # type: ignore + + RaisesGroup(KeyboardInterrupt, check=handle_exc) + RaisesGroup(KeyboardInterrupt, check=handle_kbi) + RaisesGroup(KeyboardInterrupt, check=handle_value) # type: ignore + + RaisesGroup(ValueError, check=handle_exc) + RaisesGroup(ValueError, check=handle_kbi) # type: ignore + RaisesGroup(ValueError, check=handle_value) + + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_exc) + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_kbi) # type: ignore + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_value) # type: ignore + + +def raisesgroup_narrow_baseexceptiongroup() -> None: + """Check type narrowing specifically for the container exceptiongroup.""" + + def handle_group(e: ExceptionGroup[Exception]) -> bool: + return True + + def handle_group_value(e: ExceptionGroup[ValueError]) -> bool: + return True + + RaisesGroup(ValueError, check=handle_group_value) + + RaisesGroup(Exception, check=handle_group) + + +def check_matcher_transparent() -> None: + with RaisesGroup(Matcher(ValueError)) as e: + ... + _: BaseExceptionGroup[ValueError] = e.value + assert_type(e.value, ExceptionGroup[ValueError]) + + +def check_nested_raisesgroups_contextmanager() -> None: + with RaisesGroup(RaisesGroup(ValueError)) as excinfo: + raise ExceptionGroup("foo", (ValueError(),)) + + _: BaseExceptionGroup[BaseExceptionGroup[ValueError]] = excinfo.value + + assert_type( + excinfo.value, + ExceptionGroup[ExceptionGroup[ValueError]], + ) + + assert_type( + excinfo.value.exceptions[0], + # this union is because of how typeshed defines .exceptions + Union[ + ExceptionGroup[ValueError], + ExceptionGroup[ExceptionGroup[ValueError]], + ], + ) + + +def check_nested_raisesgroups_matches() -> None: + """Check nested RaisesGroups with .matches""" + exc: ExceptionGroup[ExceptionGroup[ValueError]] = ExceptionGroup( + "", + (ExceptionGroup("", (ValueError(),)),), + ) + + if RaisesGroup(RaisesGroup(ValueError)).matches(exc): + assert_type(exc, ExceptionGroup[ExceptionGroup[ValueError]]) + + +def check_multiple_exceptions_1() -> None: + a = RaisesGroup(ValueError, ValueError) + b = RaisesGroup(Matcher(ValueError), Matcher(ValueError)) + c = RaisesGroup(ValueError, Matcher(ValueError)) + + d: RaisesGroup[ValueError] + d = a + d = b + d = c + assert d + + +def check_multiple_exceptions_2() -> None: + # This previously failed due to lack of covariance in the TypeVar + a = RaisesGroup(Matcher(ValueError), Matcher(TypeError)) + b = RaisesGroup(Matcher(ValueError), TypeError) + c = RaisesGroup(ValueError, TypeError) + + d: RaisesGroup[Exception] + d = a + d = b + d = c + assert d + + +def check_raisesgroup_overloads() -> None: + # allow_unwrapped=True does not allow: + # multiple exceptions + RaisesGroup(ValueError, TypeError, allow_unwrapped=True) # type: ignore + # nested RaisesGroup + RaisesGroup(RaisesGroup(ValueError), allow_unwrapped=True) # type: ignore + # specifying match + RaisesGroup(ValueError, match="foo", allow_unwrapped=True) # type: ignore + # specifying check + RaisesGroup(ValueError, check=bool, allow_unwrapped=True) # type: ignore + # allowed variants + RaisesGroup(ValueError, allow_unwrapped=True) + RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True) + RaisesGroup(Matcher(ValueError), allow_unwrapped=True) + + # flatten_subgroups=True does not allow nested RaisesGroup + RaisesGroup(RaisesGroup(ValueError), flatten_subgroups=True) # type: ignore + # but rest is plenty fine + RaisesGroup(ValueError, TypeError, flatten_subgroups=True) + RaisesGroup(ValueError, match="foo", flatten_subgroups=True) + RaisesGroup(ValueError, check=bool, flatten_subgroups=True) + RaisesGroup(ValueError, flatten_subgroups=True) + RaisesGroup(Matcher(ValueError), flatten_subgroups=True) + + # if they're both false we can of course specify nested raisesgroup + RaisesGroup(RaisesGroup(ValueError)) + + +def check_triple_nested_raisesgroup() -> None: + with RaisesGroup(RaisesGroup(RaisesGroup(ValueError))) as e: + assert_type(e.value, ExceptionGroup[ExceptionGroup[ExceptionGroup[ValueError]]]) + + +def check_check_typing() -> None: + # `BaseExceptiongroup` should perhaps be `ExceptionGroup`, but close enough + assert_type( + RaisesGroup(ValueError).check, + Union[ + Callable[[BaseExceptionGroup[ValueError]], bool], + None, + ], + )