Skip to content

Commit 190b920

Browse files
RKrahlRKrahl
committed
Merge branch 'doc-debug' into doc
2 parents 291b7bb + 9bf329b commit 190b920

File tree

7 files changed

+711
-0
lines changed

7 files changed

+711
-0
lines changed

‎doc/examples/debugging-logging.out

Lines changed: 325 additions & 0 deletions
Large diffs are not rendered by default.

‎doc/examples/debugging-summary.out

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
$ pytest -rs debugging.py
2+
============================= test session starts ==============================
3+
platform linux -- Python 3.10.1, pytest-6.2.5, py-1.11.0, pluggy-1.0.0
4+
rootdir: /home/user
5+
plugins: dependency-0.6.0
6+
collected 25 items
7+
8+
debugging.py x.s.s.x....sss.s.s.sss..s [100%]
9+
10+
=========================== short test summary info ============================
11+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_c depends on test_a
12+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_e depends on test_c
13+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_multicolored depends on test_colors
14+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_alert depends on test_colors[Color.RED]
15+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_g depends on test_f
16+
SKIPPED [1] debugging.py:15: could not import 'fleet': No module named 'fleet'
17+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_q[0] depends on test_p
18+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_q[1] depends on test_p
19+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_m depends on test_b
20+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_o depends on test_h
21+
SKIPPED [1] /usr/lib/python3.10/site-packages/pytest_dependency.py:101: test_s depends on test_l
22+
================== 12 passed, 11 skipped, 2 xfailed in 0.05s ===================

‎doc/examples/debugging-verbose.out

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
$ pytest --verbose debugging.py
2+
============================= test session starts ==============================
3+
platform linux -- Python 3.10.1, pytest-6.2.5, py-1.11.0, pluggy-1.0.0 -- /usr/bin/python3
4+
cachedir: .pytest_cache
5+
rootdir: /home/user
6+
plugins: dependency-0.6.0
7+
collecting ... collected 25 items
8+
9+
debugging.py::test_a XFAIL (deliberate fail) [ 4%]
10+
debugging.py::test_b PASSED [ 8%]
11+
debugging.py::test_c SKIPPED (test_c depends on test_a) [ 12%]
12+
debugging.py::test_d PASSED [ 16%]
13+
debugging.py::test_e SKIPPED (test_e depends on test_c) [ 20%]
14+
debugging.py::TestClass::test_a PASSED [ 24%]
15+
debugging.py::TestClass::test_b XFAIL (deliberate fail) [ 28%]
16+
debugging.py::TestClass::test_c PASSED [ 32%]
17+
debugging.py::test_colors[RED] PASSED [ 36%]
18+
debugging.py::test_colors[GREEN] PASSED [ 40%]
19+
debugging.py::test_colors[BLUE] PASSED [ 44%]
20+
debugging.py::test_multicolored SKIPPED (test_multicolored depends o...) [ 48%]
21+
debugging.py::test_alert SKIPPED (test_alert depends on test_colors[...) [ 52%]
22+
debugging.py::test_g SKIPPED (test_g depends on test_f) [ 56%]
23+
debugging.py::test_h PASSED [ 60%]
24+
debugging.py::test_k SKIPPED (could not import 'fleet': No module na...) [ 64%]
25+
debugging.py::test_l[0] PASSED [ 68%]
26+
debugging.py::test_q[0] SKIPPED (test_q[0] depends on test_p) [ 72%]
27+
debugging.py::test_l[1] PASSED [ 76%]
28+
debugging.py::test_q[1] SKIPPED (test_q[1] depends on test_p) [ 80%]
29+
debugging.py::test_m SKIPPED (test_m depends on test_b) [ 84%]
30+
debugging.py::test_o SKIPPED (test_o depends on test_h) [ 88%]
31+
debugging.py::test_p PASSED [ 92%]
32+
debugging.py::test_r PASSED [ 96%]
33+
debugging.py::test_s SKIPPED (test_s depends on test_l) [100%]
34+
35+
================== 12 passed, 11 skipped, 2 xfailed in 0.06s ===================

‎doc/examples/debugging.py

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
from enum import Enum
2+
import pytest
3+
4+
5+
class Color(Enum):
6+
RED = 1
7+
GREEN = 2
8+
BLUE = 3
9+
10+
def __str__(self):
11+
return self.name
12+
13+
14+
def get_starship(name):
15+
fleet = pytest.importorskip("fleet")
16+
return fleet.get_ship(name)
17+
18+
19+
@pytest.fixture(scope="module", params=range(2))
20+
def prepenv(request):
21+
pass
22+
23+
@pytest.mark.dependency()
24+
@pytest.mark.xfail(reason="deliberate fail")
25+
def test_a():
26+
assert False
27+
28+
@pytest.mark.dependency()
29+
def test_b():
30+
pass
31+
32+
@pytest.mark.dependency(depends=["test_a"])
33+
def test_c():
34+
pass
35+
36+
@pytest.mark.dependency(depends=["test_b"])
37+
def test_d():
38+
pass
39+
40+
@pytest.mark.dependency(depends=["test_b", "test_c"])
41+
def test_e():
42+
pass
43+
44+
45+
class TestClass(object):
46+
47+
@pytest.mark.dependency()
48+
def test_a(self):
49+
pass
50+
51+
@pytest.mark.dependency()
52+
@pytest.mark.xfail(reason="deliberate fail")
53+
def test_b(self):
54+
assert False
55+
56+
@pytest.mark.dependency(depends=["test_b"])
57+
def test_c(self):
58+
pass
59+
60+
61+
@pytest.mark.dependency()
62+
@pytest.mark.parametrize("c", [ Color.RED, Color.GREEN, Color.BLUE, ])
63+
def test_colors(c):
64+
pass
65+
66+
@pytest.mark.dependency(depends=["test_colors"])
67+
def test_multicolored():
68+
pass
69+
70+
@pytest.mark.dependency(depends=["test_colors[Color.RED]"])
71+
def test_alert():
72+
pass
73+
74+
@pytest.mark.dependency(depends=["test_f"])
75+
def test_g():
76+
pass
77+
78+
@pytest.mark.dependency(name="h")
79+
def test_h():
80+
pass
81+
82+
@pytest.mark.dependency(depends=["test_b"])
83+
def test_k():
84+
s = get_starship("NCC-1701")
85+
86+
@pytest.mark.dependency()
87+
def test_l(prepenv):
88+
pass
89+
90+
@pytest.mark.dependency(depends=["test_b"], scope='session')
91+
def test_m():
92+
pass
93+
94+
@pytest.mark.dependency(depends=["test_h"])
95+
def test_o():
96+
pass
97+
98+
@pytest.mark.dependency()
99+
def test_p():
100+
pass
101+
102+
@pytest.mark.dependency(depends=["test_p"])
103+
def test_q(prepenv):
104+
pass
105+
106+
@pytest.mark.dependency(depends=["test_a"])
107+
@pytest.mark.dependency(name="r")
108+
def test_r():
109+
pass
110+
111+
@pytest.mark.dependency(depends=["test_l"])
112+
def test_s():
113+
pass

‎doc/src/debugging.rst

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
Debugging guide
2+
===============
3+
4+
This section is supposed to provide hints on what to check if
5+
pytest-dependency does not seem to behave as expected.
6+
7+
Example
8+
-------
9+
10+
We consider the following example in this guide:
11+
12+
.. literalinclude:: ../examples/debugging.py
13+
14+
This example contains several cases where the presumably intended
15+
behavior of the code differs from what happens in practice. We will
16+
show below how diagnostic tools in pytest may be used to unravel the
17+
discrepancies. The results that may (or may not) be surprising
18+
include:
19+
20+
+ The test method `test_c` in class `TestClass` depending on `test_b`
21+
is run, although the test method `test_b` fails.
22+
23+
+ All instances of `test_colors` succeed. Yet `test_multicolored`
24+
that only depends on `test_colors` is skipped.
25+
26+
+ Similarly `test_alert` depending only on `test_colors[Color.RED]` is
27+
skipped, although `test_colors` with the parameter value `Color.RED`
28+
succeeds.
29+
30+
+ `test_k` depending only on `test_b` is skipped, although `test_b`
31+
succeeds.
32+
33+
+ Same with `test_m` depending only on `test_b` is skipped.
34+
35+
+ `test_o` depending only on `test_h` is skipped, although `test_h`
36+
succeeds.
37+
38+
+ `test_q` depending only on `test_p` is skipped, although `test_p`
39+
succeeds.
40+
41+
+ `test_r` is run, although `test_a` fails.
42+
43+
+ `test_s` depending only on `test_l` is skipped, although `test_l`
44+
succeeds.
45+
46+
Diagnostic tools
47+
----------------
48+
49+
There are different ways to request diagnostic output from pytest. We
50+
will discuss how they may be used to better understand the behavior of
51+
pytest-dependency.
52+
53+
pytest summary
54+
..............
55+
56+
You can request a short summary from pytest including information on
57+
skipped tests using the ``-rs`` `command line option`__:
58+
59+
.. literalinclude:: ../examples/debugging-summary.out
60+
61+
This summary indicates if a test has been skipped by pytest-dependency
62+
in the first place. In the present example, the summary hints that
63+
`test_k` has been skipped due to another reason, unrelated to
64+
pytest-dependency. If the test has been skipped by pytest-dependency,
65+
the summary displays the name of the missing dependency.
66+
67+
.. __: https://docs.pytest.org/en/stable/usage.html#detailed-summary-report
68+
69+
Verbose pytest output
70+
.....................
71+
72+
A list of all tests with their respective outcome will be displayed if
73+
you call pytest with the ``--verbose`` command line option:
74+
75+
.. literalinclude:: ../examples/debugging-verbose.out
76+
77+
The verbose listing is particular useful, because it shows the pytest
78+
node id for each test, which is not always obvious. As explained in
79+
Section :ref:`names`, this node id is the basis to form the default
80+
test name that need to be used to reference the test in the
81+
dependencies.
82+
83+
From this list we can understand why `test_multicolored` has been
84+
skipped: it depends on `test_colors`. But `test_colors` is
85+
parametrized and thus the parameter value is included in the node id.
86+
As a result, a dependency by the name `test_colors` can not be found.
87+
The same thing happens in the case of `test_s`: it depends on
88+
`test_l`, but the latter uses a parametrized fixture, so it indirectly
89+
takes a parameter value and that value must be included in the
90+
reference for the dependency.
91+
92+
In the case of `test_alert`, the parameter value is included in the
93+
dependency `test_colors[Color.RED]`. But in the node id as displayed
94+
in the verbose list, that test appears as `test_colors[RED]`. Note
95+
that `class Color` overrides the string representation operator and
96+
that affects how the parameter value appears in the node id in this
97+
case.
98+
99+
The verbose list also displays the execution order of the tests. In
100+
the present example, this order differs from the order in the source
101+
code. That is the reason why both instances of `test_q` are skipped:
102+
they are executed before the dependency `test_p`. So the outcome of
103+
the latter is yet unknown at the moment that the dependency is
104+
checked.
105+
106+
Logging
107+
.......
108+
109+
pytest-dependency emits log messages when registering test results and
110+
when checking dependencies for a test. You can request these log
111+
messages to be displayed at runtime using `log command line options`__
112+
in the pytest call. Beware that this may produce a large amount of
113+
output, even for medium size test suites. We will present only a few
114+
fragments of the output here. Consider the start of that output,
115+
covering the first test `test_a`:
116+
117+
.. literalinclude:: ../examples/debugging-logging.out
118+
:end-before: debugging.py::test_b
119+
120+
It is shown how the test outcome for each of the three test phases
121+
(setup, call, and teardown) is registered in pytest-dependency. It is
122+
also shown which name is used to register the test outcome depending
123+
on the scope.
124+
125+
Considering the relevant fragments of the output, we can check why
126+
`TestClass::test_c` is not skipped:
127+
128+
.. literalinclude:: ../examples/debugging-logging.out
129+
:lines: 20-31,86-116
130+
131+
The dependency `test_b` is checked in module scope. If that
132+
dependency was meant to reference the method of the same class, it
133+
would either need to be referenced as `test_b` in class scope or as
134+
`TestClass::test_b` in module scope or as
135+
`debugging.py::TestClass::test_b` in session scope. The way it is
136+
formulated in the example, it actually references the test function
137+
`test_b`, which succeeds.
138+
139+
A similar case is `test_m`:
140+
141+
.. literalinclude:: ../examples/debugging-logging.out
142+
:lines: 20-31,264-274
143+
144+
The dependency `test_b` is checked in session scope. There is no test
145+
that matches this name. If that dependency was mean to reference the
146+
test function `test_b` in the example, it would either need to be
147+
referenced as `debugging.py::test_b` in session scope or as `test_b`
148+
in module scope.
149+
150+
A slightly different situation is given in the case of `test_o`:
151+
152+
.. literalinclude:: ../examples/debugging-logging.out
153+
:lines: 190-201,276-286
154+
155+
In the :func:`pytest.mark.dependency` marker for `test_h` in the
156+
example, the name is overridden as `h`. The outcome of that test is
157+
registered using that name. It can thus not be found by the name
158+
`test_h`.
159+
160+
Considering the case of `test_r`:
161+
162+
.. literalinclude:: ../examples/debugging-logging.out
163+
:lines: 300-310
164+
165+
That test has no dependencies. The error in the example is that the
166+
:func:`pytest.mark.dependency` marker is applied twice to the test.
167+
That doesn't work in pytest, only the last invocation is effective.
168+
As a result, the second invocation setting a name, effectively clears
169+
the dependency list that was set in the first invocation.
170+
171+
.. __: https://docs.pytest.org/en/stable/logging.html#live-logs

‎doc/src/index.rst

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ Content of the documentation
1717
scope
1818
advanced
1919
names
20+
debugging
2021
configuration
2122
changelog
2223
reference

‎tests/test_09_examples_debugging.py

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
"""Test the included examples.
2+
"""
3+
4+
import pytest
5+
from conftest import get_example
6+
7+
8+
def test_debugging(ctestdir):
9+
"""Debugging example
10+
"""
11+
with get_example("debugging.py").open("rt") as f:
12+
ctestdir.makepyfile(f.read())
13+
result = ctestdir.runpytest("--verbose")
14+
try:
15+
result.assert_outcomes(passed=12, skipped=11, failed=0, xfailed=2)
16+
except TypeError:
17+
result.assert_outcomes(passed=12, skipped=11, failed=0)
18+
result.stdout.re_match_lines(r"""
19+
.*::test_a (?:XFAIL(?:\s+\(.*\))?|xfail)
20+
.*::test_b PASSED
21+
.*::test_c SKIPPED(?:\s+\(.*\))?
22+
.*::test_d PASSED
23+
.*::test_e SKIPPED(?:\s+\(.*\))?
24+
.*::TestClass::test_a PASSED
25+
.*::TestClass::test_b (?:XFAIL(?:\s+\(.*\))?|xfail)
26+
.*::TestClass::test_c PASSED
27+
.*::test_colors\[RED\] PASSED
28+
.*::test_colors\[GREEN\] PASSED
29+
.*::test_colors\[BLUE\] PASSED
30+
.*::test_multicolored SKIPPED(?:\s+\(.*\))?
31+
.*::test_alert SKIPPED(?:\s+\(.*\))?
32+
.*::test_g SKIPPED(?:\s+\(.*\))?
33+
.*::test_h PASSED
34+
.*::test_k SKIPPED(?:\s+\(.*\))?
35+
.*::test_l\[0\] PASSED
36+
.*::test_q\[0\] SKIPPED(?:\s+\(.*\))?
37+
.*::test_l\[1\] PASSED
38+
.*::test_q\[1\] SKIPPED(?:\s+\(.*\))?
39+
.*::test_m SKIPPED(?:\s+\(.*\))?
40+
.*::test_o SKIPPED(?:\s+\(.*\))?
41+
.*::test_p PASSED
42+
.*::test_r PASSED
43+
.*::test_s SKIPPED(?:\s+\(.*\))?
44+
""")

0 commit comments

Comments
 (0)