Merge branch 'master' into merge-master-into-features

Preparing for 3.0

Author: nicoddemus

AUTHORS

@@ -28,6 +28,7 @@ Carl Friedrich Bolz
 Charles Cloud
 Charnjit SiNGH (CCSJ)
 Chris Lamb
+Christian Boelsen
 Christian Theunert
 Christian Tismer
 Christopher Gilling

CHANGELOG.rst

@@ -3,8 +3,15 @@
 
 **Incompatible changes**
 
+<<<<<<< HEAD
 A number of incompatible changes were made in this release, with the intent of removing features deprecated for a long
 time or change existing behaviors in order to make them less surprising/more useful.
+=======
+* Improve error message with fixture lookup errors: add an 'E' to the first
+  line and '>' to the rest. Fixes `#717`_. Thanks `@blueyed`_ for reporting and
+  a PR, `@eolo999`_ for the initial PR and `@tomviner`_ for his guidance during
+  EuroPython2016 sprint.
+>>>>>>> master
 
 * Reinterpretation mode has now been removed.  Only plain and rewrite
   mode are available, consequently the ``--assert=reinterp`` option is
@@ -159,6 +166,14 @@ time or change existing behaviors in order to make them less surprising/more use
 * Plugins now benefit from assertion rewriting.  Thanks
   `@sober7`_, `@nicoddemus`_ and `@flub`_ for the PR.
 
+* Change ``report.outcome`` for ``xpassed`` tests to ``"passed"`` in non-strict
+  mode and ``"failed"`` in strict mode. Thanks to `@hackebrot`_ for the PR
+  (`#1795`_) and `@gprasad84`_ for report (`#1546`_).
+
+* Tests marked with ``xfail(strict=False)`` (the default) now appear in
+  JUnitXML reports as passing tests instead of skipped.
+  Thanks to `@hackebrot`_ for the PR (`#1795`_).
+
 * Highlight path of the file location in the error report to make it easier to copy/paste.
   Thanks `@suzaku`_ for the PR (`#1778`_).
 
@@ -322,12 +337,17 @@ time or change existing behaviors in order to make them less surprising/more use
 * Fixed scope overriding inside metafunc.parametrize (`#634`_).
   Thanks to `@Stranger6667`_ for the PR.
 
-*
+* Fixed the total tests tally in junit xml output (`#1798`_).
+  Thanks to `@cryporchild`_ for the PR.
 
-*
+* ``pytest_terminal_summary`` hook now receives the ``exitstatus``
+  of the test session as argument. Thanks `@blueyed`_ for the PR (`#1809`_).
 
 *
 
+* Fixed off-by-one error with lines from ``request.node.warn``.
+  Thanks to `@blueyed`_ for the PR.
+
 *
 
 .. _#1210: https://github.com/pytest-dev/pytest/issues/1210
@@ -354,6 +374,7 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _#1526: https://github.com/pytest-dev/pytest/pull/1526
 .. _#1539: https://github.com/pytest-dev/pytest/issues/1539
 .. _#1544: https://github.com/pytest-dev/pytest/issues/1544
+.. _#1546: https://github.com/pytest-dev/pytest/issues/1546
 .. _#1553: https://github.com/pytest-dev/pytest/issues/1553
 .. _#1562: https://github.com/pytest-dev/pytest/issues/1562
 .. _#1579: https://github.com/pytest-dev/pytest/issues/1579
@@ -377,6 +398,9 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _#1740: https://github.com/pytest-dev/pytest/issues/1740
 .. _#1749: https://github.com/pytest-dev/pytest/issues/1749
 .. _#1778: https://github.com/pytest-dev/pytest/pull/1778
+.. _#1795: https://github.com/pytest-dev/pytest/pull/1795
+.. _#1798: https://github.com/pytest-dev/pytest/pull/1798
+.. _#1809: https://github.com/pytest-dev/pytest/pull/1809
 .. _#372: https://github.com/pytest-dev/pytest/issues/372
 .. _#457: https://github.com/pytest-dev/pytest/issues/457
 .. _#460: https://github.com/pytest-dev/pytest/pull/460
@@ -393,13 +417,15 @@ time or change existing behaviors in order to make them less surprising/more use
 .. _@BeyondEvil: https://github.com/BeyondEvil
 .. _@blueyed: https://github.com/blueyed
 .. _@ceridwen: https://github.com/ceridwen
+.. _@cryporchild: https://github.com/cryporchild
 .. _@csaftoiu: https://github.com/csaftoiu
 .. _@d6e: https://github.com/d6e
 .. _@davehunt: https://github.com/davehunt
 .. _@DRMacIver: https://github.com/DRMacIver
 .. _@eolo999: https://github.com/eolo999
 .. _@fengxx: https://github.com/fengxx
 .. _@flub: https://github.com/flub
+.. _@gprasad84: https://github.com/gprasad84
 .. _@graingert: https://github.com/graingert
 .. _@hartym: https://github.com/hartym
 .. _@JonathonSonesen: https://github.com/JonathonSonesen

_pytest/fixtures.py

@@ -1,4 +1,7 @@
 import sys
+
+from py._code.code import FormattedExcinfo
+
 import py
 import pytest
 import warnings
@@ -649,6 +652,7 @@ def formatrepr(self):
 
         return FixtureLookupErrorRepr(fspath, lineno, tblines, msg, self.argname)
 
+
 class FixtureLookupErrorRepr(TerminalRepr):
     def __init__(self, filename, firstlineno, tblines, errorstring, argname):
         self.tblines = tblines
@@ -658,16 +662,16 @@ def __init__(self, filename, firstlineno, tblines, errorstring, argname):
         self.argname = argname
 
     def toterminal(self, tw):
-        #tw.line("FixtureLookupError: %s" %(self.argname), red=True)
+        # tw.line("FixtureLookupError: %s" %(self.argname), red=True)
         for tbline in self.tblines:
             tw.line(tbline.rstrip())
         lines = self.errorstring.split("\n")
-        for line in lines:
-            if line == lines[0]:
-                prefix = 'E       '
-            else:
-                prefix = '        '
-            tw.line(prefix + line.strip(), red=True)
+        if lines:
+            tw.line('{0}       {1}'.format(FormattedExcinfo.fail_marker,
+                                           lines[0].strip()), red=True)
+            for line in lines[1:]:
+                tw.line('{0}       {1}'.format(FormattedExcinfo.flow_marker,
+                                               line.strip()), red=True)
         tw.line()
         tw.line("%s:%d" % (self.filename, self.firstlineno+1))
 

_pytest/junitxml.py

@@ -370,7 +370,7 @@ def pytest_sessionfinish(self):
         suite_stop_time = time.time()
         suite_time_delta = suite_stop_time - self.suite_start_time
 
-        numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped']
+        numtests = self.stats['passed'] + self.stats['failure'] + self.stats['skipped'] + self.stats['error']
 
         logfile.write('<?xml version="1.0" encoding="utf-8"?>')
 

_pytest/main.py

@@ -292,7 +292,7 @@ def warn(self, code, message):
         if fslocation is None:
             fslocation = getattr(self, "fspath", None)
         else:
-            fslocation = "%s:%s" % fslocation[:2]
+            fslocation = "%s:%s" % (fslocation[0], fslocation[1] + 1)
 
         self.ihook.pytest_logwarning.call_historic(kwargs=dict(
             code=code, message=message,

_pytest/python.py

@@ -1538,4 +1538,3 @@ def runtest(self):
     def setup(self):
         super(Function, self).setup()
         fixtures.fillfixtures(self)
-

_pytest/skipping.py

@@ -216,6 +216,18 @@ def check_strict_xfail(pyfuncitem):
             pytest.fail('[XPASS(strict)] ' + explanation, pytrace=False)
 
 
+def _is_unittest_unexpected_success_a_failure():
+    """Return if the test suite should fail if a @expectedFailure unittest test PASSES.
+
+    From https://docs.python.org/3/library/unittest.html?highlight=unittest#unittest.TestResult.wasSuccessful:
+        Changed in version 3.4: Returns False if there were any
+        unexpectedSuccesses from tests marked with the expectedFailure() decorator.
+
+    TODO: this should be moved to the "compat" module.
+    """
+    return sys.version_info >= (3, 4)
+
+
 @pytest.hookimpl(hookwrapper=True)
 def pytest_runtest_makereport(item, call):
     outcome = yield
@@ -224,9 +236,15 @@ def pytest_runtest_makereport(item, call):
     evalskip = getattr(item, '_evalskip', None)
     # unitttest special case, see setting of _unexpectedsuccess
     if hasattr(item, '_unexpectedsuccess') and rep.when == "call":
-        # we need to translate into how pytest encodes xpass
-        rep.wasxfail = "reason: " + repr(item._unexpectedsuccess)
-        rep.outcome = "failed"
+        if item._unexpectedsuccess:
+            rep.longrepr = "Unexpected success: {0}".format(item._unexpectedsuccess)
+        else:
+            rep.longrepr = "Unexpected success"
+        if _is_unittest_unexpected_success_a_failure():
+            rep.outcome = "failed"
+        else:
+            rep.outcome = "passed"
+            rep.wasxfail = rep.longrepr
     elif item.config.option.runxfail:
         pass   # don't interefere
     elif call.excinfo and call.excinfo.errisinstance(pytest.xfail.Exception):
@@ -241,8 +259,15 @@ def pytest_runtest_makereport(item, call):
                 rep.outcome = "skipped"
                 rep.wasxfail = evalxfail.getexplanation()
         elif call.when == "call":
-            rep.outcome = "failed"  # xpass outcome
-            rep.wasxfail = evalxfail.getexplanation()
+            strict_default = item.config.getini('xfail_strict')
+            is_strict_xfail = evalxfail.get('strict', strict_default)
+            explanation = evalxfail.getexplanation()
+            if is_strict_xfail:
+                rep.outcome = "failed"
+                rep.longrepr = "[XPASS(strict)] {0}".format(explanation)
+            else:
+                rep.outcome = "passed"
+                rep.wasxfail = explanation
     elif evalskip is not None and rep.skipped and type(rep.longrepr) is tuple:
         # skipped by mark.skipif; change the location of the failure
         # to point to the item definition, otherwise it will display
@@ -256,7 +281,7 @@ def pytest_report_teststatus(report):
     if hasattr(report, "wasxfail"):
         if report.skipped:
             return "xfailed", "x", "xfail"
-        elif report.failed:
+        elif report.passed:
             return "xpassed", "X", ("XPASS", {'yellow': True})
 
 # called by the terminalreporter instance/plugin

testing/python/fixture.py

@@ -425,11 +425,11 @@ def test_lookup_error(unknown):
         """)
         result = testdir.runpytest()
         result.stdout.fnmatch_lines([
-            "*ERROR*test_lookup_error*",
-            "*def test_lookup_error(unknown):*",
-            "*fixture*unknown*not found*",
-            # check if fixtures appear sorted
-            "*available fixtures:*a_fixture,*b_fixture,*c_fixture,*d_fixture*monkeypatch,*",
+            "*ERROR at setup of test_lookup_error*",
+            "  def test_lookup_error(unknown):*",
+            "E       fixture 'unknown' not found",
+            ">       available fixtures:*a_fixture,*b_fixture,*c_fixture,*d_fixture*monkeypatch,*",  # sorted
+            ">       use 'py*test --fixtures *' for help on them.",
             "*1 error*",
         ])
         assert "INTERNAL" not in result.stdout.str()

testing/python/metafunc.py

@@ -1191,22 +1191,23 @@ def test_increment(n, expected):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=2, skipped=1)
 
-    def test_xfail_passing_is_xpass(self, testdir):
+    @pytest.mark.parametrize('strict', [True, False])
+    def test_xfail_passing_is_xpass(self, testdir, strict):
         s = """
             import pytest
 
             @pytest.mark.parametrize(("n", "expected"), [
                 (1, 2),
-                pytest.mark.xfail("sys.version > 0", reason="some bug")((2, 3)),
+                pytest.mark.xfail("sys.version_info > (0, 0, 0)", reason="some bug", strict={strict})((2, 3)),
                 (3, 4),
             ])
             def test_increment(n, expected):
                 assert n + 1 == expected
-        """
+        """.format(strict=strict)
         testdir.makepyfile(s)
         reprec = testdir.inline_run()
-        # xpass is fail, obviously :)
-        reprec.assertoutcome(passed=2, failed=1)
+        passed, failed = (2, 1) if strict else (3, 0)
+        reprec.assertoutcome(passed=passed, failed=failed)
 
     def test_parametrize_called_in_generate_tests(self, testdir):
         s = """

testing/test_config.py

@@ -548,13 +548,14 @@ def test_proper(pytestconfig):
         reprec = testdir.inline_run()
         reprec.assertoutcome(passed=1)
 
-    def test_warn_on_test_item_from_request(self, testdir):
+    def test_warn_on_test_item_from_request(self, testdir, request):
         testdir.makepyfile("""
             import pytest
 
             @pytest.fixture
             def fix(request):
                 request.node.warn("T1", "hello")
+
             def test_hello(fix):
                 pass
         """)
@@ -565,7 +566,7 @@ def test_hello(fix):
         result = testdir.runpytest()
         result.stdout.fnmatch_lines("""
             ===*pytest-warning summary*===
-            *WT1*test_warn_on_test_item*:5*hello*
+            *WT1*test_warn_on_test_item*:7 hello*
         """)
 
 class TestRootdir:
@@ -625,6 +626,7 @@ def test_with_specific_inifile(self, tmpdir):
         rootdir, inifile, inicfg = determine_setup(inifile, [tmpdir])
         assert rootdir == tmpdir
 
+
 class TestOverrideIniArgs:
     @pytest.mark.parametrize("name", "setup.cfg tox.ini pytest.ini".split())
     def test_override_ini_names(self, testdir, name):