Merge pull request #1816 from nicoddemus/merge-master-into-features

Merge master into features

Author: nicoddemus

AUTHORS

@@ -28,6 +28,7 @@ Carl Friedrich Bolz
 Charles Cloud
 Charnjit SiNGH (CCSJ)
 Chris Lamb
+Christian Boelsen
 Christian Theunert
 Christian Tismer
 Christopher Gilling

CHANGELOG.rst

@@ -3,6 +3,7 @@
 
 **Incompatible changes**
 
+
 A number of incompatible changes were made in this release, with the intent of removing features deprecated for a long
 time or change existing behaviors in order to make them less surprising/more useful.
 
@@ -159,6 +160,14 @@ time or change existing behaviors in order to make them less surprising/more use
 * Plugins now benefit from assertion rewriting.  Thanks
   `@sober7`_, `@nicoddemus`_ and `@flub`_ for the PR.
 
+* Change ``report.outcome`` for ``xpassed`` tests to ``"passed"`` in non-strict
+  mode and ``"failed"`` in strict mode. Thanks to `@hackebrot`_ for the PR
+  (`#1795`_) and `@gprasad84`_ for report (`#1546`_).
+
+* Tests marked with ``xfail(strict=False)`` (the default) now appear in
+  JUnitXML reports as passing tests instead of skipped.
+  Thanks to `@hackebrot`_ for the PR (`#1795`_).
+
 * Highlight path of the file location in the error report to make it easier to copy/paste.
   Thanks `@suzaku`_ for the PR (`#1778`_).
 
@@ -175,6 +184,9 @@ time or change existing behaviors in order to make them less surprising/more use
   fixture is declared in a test function.
   Thanks `@nicoddemus`_ for the PR.
 
+* ``pytest_terminal_summary`` hook now receives the ``exitstatus``
+  of the test session as argument. Thanks `@blueyed`_ for the PR (`#1809`_).
+
 * Parametrize ids can accept ``None`` as specific test id, in which case the
   automatically generated id for that argument will be used.
   Thanks `@palaviv`_ for the complete PR (`#1468`_).
@@ -279,7 +291,10 @@ time or change existing behaviors in order to make them less surprising/more use
   and there are more modern and better alternatives (see `#830`_).
   Thanks `@nicoddemus`_ for the PR.
 
-*
+* Improve error message with fixture lookup errors: add an 'E' to the first
+  line and '>' to the rest. Fixes `#717`_. Thanks `@blueyed`_ for reporting and
+  a PR, `@eolo999`_ for the initial PR and `@tomviner`_ for his guidance during
+  EuroPython2016 sprint.
 
 *
 
@@ -322,11 +337,11 @@ time or change existing behaviors in order to make them less surprising/more use
 * Fixed scope overriding inside metafunc.parametrize (`#634`_).
   Thanks to `@Stranger6667`_ for the PR.
 
-*
-
-*
+* Fixed the total tests tally in junit xml output (`#1798`_).
+  Thanks to `@cryporchild`_ for the PR.
 
-*
+* Fixed off-by-one error with lines from ``request.node.warn``.
+  Thanks to `@blueyed`_ for the PR.
 
 *
 
@@ -354,6 +369,7 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _#1526: https://github.com/pytest-dev/pytest/pull/1526
 .. _#1539: https://github.com/pytest-dev/pytest/issues/1539
 .. _#1544: https://github.com/pytest-dev/pytest/issues/1544
+.. _#1546: https://github.com/pytest-dev/pytest/issues/1546
 .. _#1553: https://github.com/pytest-dev/pytest/issues/1553
 .. _#1562: https://github.com/pytest-dev/pytest/issues/1562
 .. _#1579: https://github.com/pytest-dev/pytest/issues/1579
@@ -377,6 +393,9 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _#1740: https://github.com/pytest-dev/pytest/issues/1740
 .. _#1749: https://github.com/pytest-dev/pytest/issues/1749
 .. _#1778: https://github.com/pytest-dev/pytest/pull/1778
+.. _#1795: https://github.com/pytest-dev/pytest/pull/1795
+.. _#1798: https://github.com/pytest-dev/pytest/pull/1798
+.. _#1809: https://github.com/pytest-dev/pytest/pull/1809
 .. _#372: https://github.com/pytest-dev/pytest/issues/372
 .. _#457: https://github.com/pytest-dev/pytest/issues/457
 .. _#460: https://github.com/pytest-dev/pytest/pull/460
@@ -393,13 +412,15 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _@BeyondEvil: https://github.com/BeyondEvil
 .. _@blueyed: https://github.com/blueyed
 .. _@ceridwen: https://github.com/ceridwen
+.. _@cryporchild: https://github.com/cryporchild
 .. _@csaftoiu: https://github.com/csaftoiu
 .. _@d6e: https://github.com/d6e
 .. _@davehunt: https://github.com/davehunt
 .. _@DRMacIver: https://github.com/DRMacIver
 .. _@eolo999: https://github.com/eolo999
 .. _@fengxx: https://github.com/fengxx
 .. _@flub: https://github.com/flub
+.. _@gprasad84: https://github.com/gprasad84
 .. _@graingert: https://github.com/graingert
 .. _@hartym: https://github.com/hartym
 .. _@JonathonSonesen: https://github.com/JonathonSonesen

_pytest/compat.py

@@ -173,6 +173,7 @@ def get_real_func(obj):
         obj = obj.func
     return obj
 
+
 def getfslineno(obj):
     # xxx let decorators etc specify a sane ordering
     obj = get_real_func(obj)
@@ -182,6 +183,7 @@ def getfslineno(obj):
     assert isinstance(fslineno[1], int), obj
     return fslineno
 
+
 def getimfunc(func):
     try:
         return func.__func__
@@ -191,6 +193,7 @@ def getimfunc(func):
         except AttributeError:
             return func
 
+
 def safe_getattr(object, name, default):
     """ Like getattr but return default upon any Exception.
 
@@ -201,3 +204,13 @@ def safe_getattr(object, name, default):
         return getattr(object, name, default)
     except Exception:
         return default
+
+
+def _is_unittest_unexpected_success_a_failure():
+    """Return if the test suite should fail if a @expectedFailure unittest test PASSES.
+
+    From https://docs.python.org/3/library/unittest.html?highlight=unittest#unittest.TestResult.wasSuccessful:
+        Changed in version 3.4: Returns False if there were any
+        unexpectedSuccesses from tests marked with the expectedFailure() decorator.
+    """
+    return sys.version_info >= (3, 4)

_pytest/fixtures.py

@@ -1,4 +1,7 @@
 import sys
+
+from py._code.code import FormattedExcinfo
+
 import py
 import pytest
 import warnings
@@ -649,6 +652,7 @@ def formatrepr(self):
 
         return FixtureLookupErrorRepr(fspath, lineno, tblines, msg, self.argname)
 
+
 class FixtureLookupErrorRepr(TerminalRepr):
     def __init__(self, filename, firstlineno, tblines, errorstring, argname):
         self.tblines = tblines
@@ -658,16 +662,16 @@ def __init__(self, filename, firstlineno, tblines, errorstring, argname):
         self.argname = argname
 
     def toterminal(self, tw):
-        #tw.line("FixtureLookupError: %s" %(self.argname), red=True)
+        # tw.line("FixtureLookupError: %s" %(self.argname), red=True)
         for tbline in self.tblines:
             tw.line(tbline.rstrip())
         lines = self.errorstring.split("\n")
-        for line in lines:
-            if line == lines[0]:
-                prefix = 'E       '
-            else:
-                prefix = '        '
-            tw.line(prefix + line.strip(), red=True)
+        if lines:
+            tw.line('{0}       {1}'.format(FormattedExcinfo.fail_marker,
+                                           lines[0].strip()), red=True)
+            for line in lines[1:]:
+                tw.line('{0}       {1}'.format(FormattedExcinfo.flow_marker,
+                                               line.strip()), red=True)
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 

_pytest/junitxml.py

@@ -370,7 +370,7 @@ def pytest_sessionfinish(self):
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
 
-        numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped']
+        numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error']
 
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
 

_pytest/main.py

@@ -292,7 +292,7 @@ def warn(self, code, message):
         if fslocation is None:
             fslocation = getattr(self, "fspath", None)
         else:
-            fslocation = "%s:%s" % fslocation[:2]
+            fslocation = "%s:%s" % (fslocation[0], fslocation[1] + 1)
 
         self.ihook.pytest_logwarning.call_historic(kwargs=dict(
             code=code, message=message,

_pytest/python.py

@@ -1538,4 +1538,3 @@ def runtest(self):
     def setup(self):
         super(Function, self).setup()
         fixtures.fillfixtures(self)
-

_pytest/skipping.py

@@ -224,9 +224,16 @@ def pytest_runtest_makereport(item, call):
     evalskip = getattr(item, '_evalskip', None)
     # unitttest special case, see setting of _unexpectedsuccess
     if hasattr(item, '_unexpectedsuccess') and rep.when == "call":
-        # we need to translate into how pytest encodes xpass
-        rep.wasxfail = "reason: " + repr(item._unexpectedsuccess)
-        rep.outcome = "failed"
+        from _pytest.compat import _is_unittest_unexpected_success_a_failure
+        if item._unexpectedsuccess:
+            rep.longrepr = "Unexpected success: {0}".format(item._unexpectedsuccess)
+        else:
+            rep.longrepr = "Unexpected success"
+        if _is_unittest_unexpected_success_a_failure():
+            rep.outcome = "failed"
+        else:
+            rep.outcome = "passed"
+            rep.wasxfail = rep.longrepr
     elif item.config.option.runxfail:
         pass   # don't interefere
     elif call.excinfo and call.excinfo.errisinstance(pytest.xfail.Exception):
@@ -241,8 +248,15 @@ def pytest_runtest_makereport(item, call):
                 rep.outcome = "skipped"
                 rep.wasxfail = evalxfail.getexplanation()
         elif call.when == "call":
-            rep.outcome = "failed"  # xpass outcome
-            rep.wasxfail = evalxfail.getexplanation()
+            strict_default = item.config.getini('xfail_strict')
+            is_strict_xfail = evalxfail.get('strict', strict_default)
+            explanation = evalxfail.getexplanation()
+            if is_strict_xfail:
+                rep.outcome = "failed"
+                rep.longrepr = "[XPASS(strict)] {0}".format(explanation)
+            else:
+                rep.outcome = "passed"
+                rep.wasxfail = explanation
     elif evalskip is not None and rep.skipped and type(rep.longrepr) is tuple:
         # skipped by mark.skipif; change the location of the failure
         # to point to the item definition, otherwise it will display
@@ -256,7 +270,7 @@ def pytest_report_teststatus(report):
     if hasattr(report, "wasxfail"):
         if report.skipped:
             return "xfailed", "x", "xfail"
-        elif report.failed:
+        elif report.passed:
             return "xpassed", "X", ("XPASS", {'yellow': True})
 
 # called by the terminalreporter instance/plugin

testing/python/fixture.py

@@ -425,11 +425,11 @@ def test_lookup_error(unknown):
         """)
         result = testdir.runpytest()
         result.stdout.fnmatch_lines([
-            "*ERROR*test_lookup_error*",
-            "*def test_lookup_error(unknown):*",
-            "*fixture*unknown*not found*",
-            # check if fixtures appear sorted
-            "*available fixtures:*a_fixture,*b_fixture,*c_fixture,*d_fixture*monkeypatch,*",
+            "*ERROR at setup of test_lookup_error*",
+            "  def test_lookup_error(unknown):*",
+            "E       fixture 'unknown' not found",
+            ">       available fixtures:*a_fixture,*b_fixture,*c_fixture,*d_fixture*monkeypatch,*",  # sorted
+            ">       use 'py*test --fixtures *' for help on them.",
             "*1 error*",
         ])
         assert "INTERNAL" not in result.stdout.str()

testing/python/metafunc.py

@@ -1191,22 +1191,23 @@ def test_increment(n, expected):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=2, skipped=1)
 
-    def test_xfail_passing_is_xpass(self, testdir):
+    @pytest.mark.parametrize('strict', [True, False])
+    def test_xfail_passing_is_xpass(self, testdir, strict):
         s = """
             import pytest
 
             @pytest.mark.parametrize(("n", "expected"), [
                 (1, 2),
-                pytest.mark.xfail("sys.version > 0", reason="some bug")((2, 3)),
+                pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})((2, 3)),
                 (3, 4),
             ])
             def test_increment(n, expected):
                 assert n + 1 == expected
-        """
+        """.format(strict=strict)
         testdir.makepyfile(s)
         reprec = testdir.inline_run()
-        # xpass is fail, obviously :)
-        reprec.assertoutcome(passed=2, failed=1)
+        passed, failed = (2, 1) if strict else (3, 0)
+        reprec.assertoutcome(passed=passed, failed=failed)
 
     def test_parametrize_called_in_generate_tests(self, testdir):
         s = """