Improve type-checking in OutcomeException

Fix #5578

Author: nicoddemus

changelog/5578.bugfix.rst

@@ -0,0 +1,3 @@
+Improve type checking for some exception-raising functions (``pytest.xfail``, ``pytest.skip``, etc)
+so they provide better error messages when users meant to use marks (for example ``@pytest.xfail``
+instead of ``@pytest.mark.xfail``).

src/_pytest/outcomes.py

@@ -13,16 +13,19 @@ class OutcomeException(BaseException):
     """
 
     def __init__(self, msg=None, pytrace=True):
+        if msg is not None and not isinstance(msg, str):
+            error_msg = (
+                "{} expected string as 'msg' parameter, got '{}' instead.\n"
+                "Perhaps you meant to use a mark?"
+            )
+            raise TypeError(error_msg.format(type(self).__name__, type(msg).__name__))
         BaseException.__init__(self, msg)
         self.msg = msg
         self.pytrace = pytrace
 
     def __repr__(self):
         if self.msg:
-            val = self.msg
-            if isinstance(val, bytes):
-                val = val.decode("UTF-8", errors="replace")
-            return val
+            return self.msg
         return "<{} instance>".format(self.__class__.__name__)
 
     __str__ = __repr__

testing/test_runner.py

@@ -11,6 +11,7 @@
 from _pytest import outcomes
 from _pytest import reports
 from _pytest import runner
+from _pytest.outcomes import OutcomeException
 
 
 class TestSetupState:
@@ -990,3 +991,18 @@ def test_func():
         rep = reports[1]
         assert rep.capstdout == ""
         assert rep.capstderr == ""
+
+
+def test_outcome_exception_bad_msg():
+    """Check that OutcomeExceptions validate their input to prevent confusing errors (#5578)"""
+
+    def func():
+        pass
+
+    expected = (
+        "OutcomeException expected string as 'msg' parameter, got 'function' instead.\n"
+        "Perhaps you meant to use a mark?"
+    )
+    with pytest.raises(TypeError) as excinfo:
+        OutcomeException(func)
+    assert str(excinfo.value) == expected

testing/test_skipping.py

@@ -1066,7 +1066,8 @@ def test_module_level_skip_error(testdir):
     testdir.makepyfile(
         """
         import pytest
-        @pytest.skip
+        pytest.skip("skip_module_level")
+
         def test_func():
             assert True
     """