Skip to content

Commit 118ac57

Browse files
RKrahlRKrahl
committed
Add testing of documentation examples
1 parent f6a43ac commit 118ac57

File tree

5 files changed

+340
-0
lines changed

5 files changed

+340
-0
lines changed

‎tests/conftest.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,22 @@
1+
import packaging.version
2+
from pathlib import Path
13
import pytest
24

35
pytest_plugins = "pytester"
46

7+
example_dir = (Path(__file__).parent / "../doc/examples").resolve()
8+
9+
def require_pytest_version(minversion):
10+
pytest_version = packaging.version.parse(pytest.__version__)
11+
if pytest_version < packaging.version.parse(minversion):
12+
pytest.skip("need pytest version %s or newer" % minversion,
13+
allow_module_level=True)
14+
15+
def get_example(fname):
16+
path = example_dir / fname
17+
assert path.is_file()
18+
return path
19+
520

621
@pytest.fixture
722
def ctestdir(testdir):

‎tests/test_09_examples_advanced.py

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
"""Test the included examples.
2+
"""
3+
4+
import pytest
5+
from conftest import get_example, require_pytest_version
6+
7+
require_pytest_version("4.2.0")
8+
9+
10+
def test_dyn_parametrized(ctestdir):
11+
"""Dynamic compilation of marked parameters
12+
"""
13+
with get_example("dyn-parametrized.py").open("rt") as f:
14+
ctestdir.makepyfile(f.read())
15+
result = ctestdir.runpytest("--verbose")
16+
result.assert_outcomes(passed=11, skipped=1, failed=0, xfailed=1)
17+
result.stdout.re_match_lines(r"""
18+
.*::test_child\[c0\] PASSED
19+
.*::test_child\[c1\] PASSED
20+
.*::test_child\[c2\] PASSED
21+
.*::test_child\[c3\] PASSED
22+
.*::test_child\[c4\] PASSED
23+
.*::test_child\[c5\] PASSED
24+
.*::test_child\[c6\] PASSED
25+
.*::test_child\[c7\] XFAIL(?:\s+\(.*\))?
26+
.*::test_child\[c8\] PASSED
27+
.*::test_parent\[p0\] PASSED
28+
.*::test_parent\[p1\] PASSED
29+
.*::test_parent\[p2\] PASSED
30+
.*::test_parent\[p3\] SKIPPED(?:\s+\(.*\))?
31+
""")
32+
33+
34+
def test_group_fixture1(ctestdir):
35+
"""Grouping tests using fixtures 1
36+
"""
37+
with get_example("group-fixture.py").open("rt") as f:
38+
ctestdir.makepyfile(f.read())
39+
result = ctestdir.runpytest("--verbose")
40+
result.assert_outcomes(passed=16, skipped=1, failed=0, xfailed=1)
41+
result.stdout.re_match_lines(r"""
42+
.*::test_a\[1\] PASSED
43+
.*::test_b\[1\] PASSED
44+
.*::test_a\[2\] PASSED
45+
.*::test_b\[2\] PASSED
46+
.*::test_a\[3\] PASSED
47+
.*::test_b\[3\] PASSED
48+
.*::test_a\[4\] PASSED
49+
.*::test_b\[4\] PASSED
50+
.*::test_a\[5\] PASSED
51+
.*::test_b\[5\] PASSED
52+
.*::test_a\[6\] PASSED
53+
.*::test_b\[6\] PASSED
54+
.*::test_a\[7\] XFAIL(?:\s+\(.*\))?
55+
.*::test_b\[7\] SKIPPED(?:\s+\(.*\))?
56+
.*::test_a\[8\] PASSED
57+
.*::test_b\[8\] PASSED
58+
.*::test_a\[9\] PASSED
59+
.*::test_b\[9\] PASSED
60+
""")
61+
62+
63+
def test_group_fixture2(ctestdir):
64+
"""Grouping tests using fixtures 2
65+
"""
66+
with get_example("group-fixture2.py").open("rt") as f:
67+
ctestdir.makepyfile(f.read())
68+
result = ctestdir.runpytest("--verbose")
69+
result.assert_outcomes(passed=24, skipped=2, failed=0, xfailed=1)
70+
result.stdout.re_match_lines(r"""
71+
.*::test_a\[1\] PASSED
72+
.*::test_b\[1\] PASSED
73+
.*::test_c\[1\] PASSED
74+
.*::test_a\[2\] PASSED
75+
.*::test_b\[2\] PASSED
76+
.*::test_c\[2\] PASSED
77+
.*::test_a\[3\] PASSED
78+
.*::test_b\[3\] PASSED
79+
.*::test_c\[3\] PASSED
80+
.*::test_a\[4\] PASSED
81+
.*::test_b\[4\] PASSED
82+
.*::test_c\[4\] PASSED
83+
.*::test_a\[5\] PASSED
84+
.*::test_b\[5\] PASSED
85+
.*::test_c\[5\] PASSED
86+
.*::test_a\[6\] PASSED
87+
.*::test_b\[6\] PASSED
88+
.*::test_c\[6\] PASSED
89+
.*::test_a\[7\] XFAIL(?:\s+\(.*\))?
90+
.*::test_b\[7\] SKIPPED(?:\s+\(.*\))?
91+
.*::test_c\[7\] SKIPPED(?:\s+\(.*\))?
92+
.*::test_a\[8\] PASSED
93+
.*::test_b\[8\] PASSED
94+
.*::test_c\[8\] PASSED
95+
.*::test_a\[9\] PASSED
96+
.*::test_b\[9\] PASSED
97+
.*::test_c\[9\] PASSED
98+
""")
99+
100+
101+
def test_all_params(ctestdir):
102+
"""Depend on all instances of a parametrized test at once
103+
"""
104+
with get_example("all_params.py").open("rt") as f:
105+
ctestdir.makepyfile(f.read())
106+
result = ctestdir.runpytest("--verbose")
107+
result.assert_outcomes(passed=20, skipped=3, failed=0, xfailed=3)
108+
result.stdout.re_match_lines(r"""
109+
.*::test_a\[0\] PASSED
110+
.*::test_a\[1\] PASSED
111+
.*::test_a\[2\] PASSED
112+
.*::test_a\[3\] PASSED
113+
.*::test_a\[4\] PASSED
114+
.*::test_a\[5\] PASSED
115+
.*::test_a\[6\] PASSED
116+
.*::test_a\[7\] PASSED
117+
.*::test_a\[8\] PASSED
118+
.*::test_a\[9\] PASSED
119+
.*::test_a\[10\] PASSED
120+
.*::test_a\[11\] PASSED
121+
.*::test_a\[12\] PASSED
122+
.*::test_a\[13\] XFAIL(?:\s+\(.*\))?
123+
.*::test_a\[14\] PASSED
124+
.*::test_a\[15\] PASSED
125+
.*::test_a\[16\] PASSED
126+
.*::test_b SKIPPED(?:\s+\(.*\))?
127+
.*::test_c\[0-2\] PASSED
128+
.*::test_c\[2-3\] PASSED
129+
.*::test_c\[4-4\] PASSED
130+
.*::test_c\[6-5\] XFAIL(?:\s+\(.*\))?
131+
.*::test_d SKIPPED(?:\s+\(.*\))?
132+
.*::test_e\[abc\] PASSED
133+
.*::test_e\[def\] XFAIL(?:\s+\(.*\))?
134+
.*::test_f SKIPPED(?:\s+\(.*\))?
135+
""")

‎tests/test_09_examples_names.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
"""Test the included examples.
2+
"""
3+
4+
import pytest
5+
from conftest import get_example, require_pytest_version
6+
7+
require_pytest_version("4.2.0")
8+
9+
10+
def test_nodeid(ctestdir):
11+
"""Node ids
12+
"""
13+
with get_example("nodeid.py").open("rt") as f:
14+
ctestdir.makepyfile(f.read())
15+
result = ctestdir.runpytest("--verbose")
16+
result.assert_outcomes(passed=6, skipped=0, failed=0, xfailed=1)
17+
result.stdout.re_match_lines(r"""
18+
.*::test_a PASSED
19+
.*::test_b\[7-True\] PASSED
20+
.*::test_b\[0-False\] PASSED
21+
.*::test_b\[-1-False\] XFAIL(?:\s+\(.*\))?
22+
.*::TestClass::test_c PASSED
23+
.*::TestClass::test_d\[order\] PASSED
24+
.*::TestClass::test_d\[disorder\] PASSED
25+
""")

‎tests/test_09_examples_scope.py

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
"""Test the included examples.
2+
"""
3+
4+
from pathlib import Path
5+
import pytest
6+
from conftest import get_example, require_pytest_version
7+
8+
require_pytest_version("4.2.0")
9+
10+
11+
def test_scope_module(ctestdir):
12+
"""Explicitely specifying the scope
13+
"""
14+
with get_example("scope_module.py").open("rt") as f:
15+
ctestdir.makepyfile(f.read())
16+
result = ctestdir.runpytest("--verbose")
17+
result.assert_outcomes(passed=2, skipped=2, failed=0, xfailed=1)
18+
result.stdout.re_match_lines(r"""
19+
.*::test_a XFAIL(?:\s+\(.*\))?
20+
.*::test_b PASSED
21+
.*::test_c SKIPPED(?:\s+\(.*\))?
22+
.*::test_d PASSED
23+
.*::test_e SKIPPED(?:\s+\(.*\))?
24+
""")
25+
26+
27+
def test_scope_session(ctestdir):
28+
"""Dependencies in session scope
29+
"""
30+
subdir = Path(ctestdir.tmpdir) / "tests"
31+
subdir.mkdir()
32+
with get_example("scope_session_mod_01.py").open("rt") as sf:
33+
with (subdir / "test_mod_01.py").open("wt") as df:
34+
df.write(sf.read())
35+
with get_example("scope_session_mod_02.py").open("rt") as sf:
36+
with (subdir / "test_mod_02.py").open("wt") as df:
37+
df.write(sf.read())
38+
result = ctestdir.runpytest("--verbose")
39+
result.assert_outcomes(passed=5, skipped=1, failed=0, xfailed=2)
40+
result.stdout.re_match_lines(r"""
41+
tests/test_mod_01.py::test_a PASSED
42+
tests/test_mod_01.py::test_b XFAIL(?:\s+\(.*\))?
43+
tests/test_mod_01.py::test_c PASSED
44+
tests/test_mod_01.py::TestClass::test_b PASSED
45+
tests/test_mod_02.py::test_a XFAIL(?:\s+\(.*\))?
46+
tests/test_mod_02.py::test_e PASSED
47+
tests/test_mod_02.py::test_f SKIPPED(?:\s+\(.*\))?
48+
tests/test_mod_02.py::test_g PASSED
49+
""")
50+
51+
52+
def test_scope_class(ctestdir):
53+
"""The class scope
54+
"""
55+
with get_example("scope_class.py").open("rt") as f:
56+
ctestdir.makepyfile(f.read())
57+
result = ctestdir.runpytest("--verbose")
58+
result.assert_outcomes(passed=3, skipped=2, failed=0, xfailed=1)
59+
result.stdout.re_match_lines(r"""
60+
.*::test_a XFAIL(?:\s+\(.*\))?
61+
.*::TestClass1::test_b PASSED
62+
.*::TestClass2::test_a PASSED
63+
.*::TestClass2::test_c SKIPPED(?:\s+\(.*\))?
64+
.*::TestClass2::test_d PASSED
65+
.*::TestClass2::test_e SKIPPED(?:\s+\(.*\))?
66+
""")

‎tests/test_09_examples_usage.py

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
"""Test the included examples.
2+
"""
3+
4+
import pytest
5+
from conftest import get_example, require_pytest_version
6+
7+
require_pytest_version("4.2.0")
8+
9+
10+
def test_basic(ctestdir):
11+
"""Basic usage
12+
"""
13+
with get_example("basic.py").open("rt") as f:
14+
ctestdir.makepyfile(f.read())
15+
result = ctestdir.runpytest("--verbose")
16+
result.assert_outcomes(passed=2, skipped=2, failed=0, xfailed=1)
17+
result.stdout.re_match_lines(r"""
18+
.*::test_a XFAIL(?:\s+\(.*\))?
19+
.*::test_b PASSED
20+
.*::test_c SKIPPED(?:\s+\(.*\))?
21+
.*::test_d PASSED
22+
.*::test_e SKIPPED(?:\s+\(.*\))?
23+
""")
24+
25+
26+
def test_named(ctestdir):
27+
"""Naming tests
28+
"""
29+
with get_example("named.py").open("rt") as f:
30+
ctestdir.makepyfile(f.read())
31+
result = ctestdir.runpytest("--verbose")
32+
result.assert_outcomes(passed=2, skipped=2, failed=0, xfailed=1)
33+
result.stdout.re_match_lines(r"""
34+
.*::test_a XFAIL(?:\s+\(.*\))?
35+
.*::test_b PASSED
36+
.*::test_c SKIPPED(?:\s+\(.*\))?
37+
.*::test_d PASSED
38+
.*::test_e SKIPPED(?:\s+\(.*\))?
39+
""")
40+
41+
42+
def test_testclass(ctestdir):
43+
"""Using test classes
44+
"""
45+
with get_example("testclass.py").open("rt") as f:
46+
ctestdir.makepyfile(f.read())
47+
result = ctestdir.runpytest("--verbose")
48+
result.assert_outcomes(passed=4, skipped=4, failed=0, xfailed=2)
49+
result.stdout.re_match_lines(r"""
50+
.*::TestClass::test_a XFAIL(?:\s+\(.*\))?
51+
.*::TestClass::test_b PASSED
52+
.*::TestClass::test_c SKIPPED(?:\s+\(.*\))?
53+
.*::TestClass::test_d PASSED
54+
.*::TestClass::test_e SKIPPED(?:\s+\(.*\))?
55+
.*::TestClassNamed::test_a XFAIL(?:\s+\(.*\))?
56+
.*::TestClassNamed::test_b PASSED
57+
.*::TestClassNamed::test_c SKIPPED(?:\s+\(.*\))?
58+
.*::TestClassNamed::test_d PASSED
59+
.*::TestClassNamed::test_e SKIPPED(?:\s+\(.*\))?
60+
""")
61+
62+
63+
def test_parametrized(ctestdir):
64+
"""Parametrized tests
65+
"""
66+
with get_example("parametrized.py").open("rt") as f:
67+
ctestdir.makepyfile(f.read())
68+
result = ctestdir.runpytest("--verbose")
69+
result.assert_outcomes(passed=7, skipped=5, failed=0, xfailed=1)
70+
result.stdout.re_match_lines(r"""
71+
.*::test_a\[0-0\] PASSED
72+
.*::test_a\[0-1\] XFAIL(?:\s+\(.*\))?
73+
.*::test_a\[1-0\] PASSED
74+
.*::test_a\[1-1\] PASSED
75+
.*::test_b\[1-2\] SKIPPED(?:\s+\(.*\))?
76+
.*::test_b\[1-3\] PASSED
77+
.*::test_b\[1-4\] PASSED
78+
.*::test_b\[2-3\] SKIPPED(?:\s+\(.*\))?
79+
.*::test_b\[2-4\] SKIPPED(?:\s+\(.*\))?
80+
.*::test_b\[3-4\] PASSED
81+
.*::test_c\[1\] SKIPPED(?:\s+\(.*\))?
82+
.*::test_c\[2\] PASSED
83+
.*::test_c\[3\] SKIPPED(?:\s+\(.*\))?
84+
""")
85+
86+
87+
def test_runtime(ctestdir):
88+
"""Marking dependencies at runtime
89+
"""
90+
with get_example("runtime.py").open("rt") as f:
91+
ctestdir.makepyfile(f.read())
92+
result = ctestdir.runpytest("--verbose")
93+
result.assert_outcomes(passed=1, skipped=2, failed=0, xfailed=1)
94+
result.stdout.re_match_lines(r"""
95+
.*::test_a PASSED
96+
.*::test_b XFAIL(?:\s+\(.*\))?
97+
.*::test_c SKIPPED(?:\s+\(.*\))?
98+
.*::test_d SKIPPED(?:\s+\(.*\))?
99+
""")

0 commit comments

Comments
 (0)