diff --git a/pytest_mpl/plugin.py b/pytest_mpl/plugin.py index 49029d6..969d745 100644 --- a/pytest_mpl/plugin.py +++ b/pytest_mpl/plugin.py @@ -31,6 +31,7 @@ import io import os import json +import uuid import shutil import hashlib import logging @@ -216,6 +217,12 @@ def pytest_addoption(parser): parser.addini(option, help=msg) +class XdistPlugin: + def pytest_configure_node(self, node): + node.workerinput["pytest_mpl_uid"] = node.config.pytest_mpl_uid + node.workerinput["pytest_mpl_results_dir"] = node.config.pytest_mpl_results_dir + + def pytest_configure(config): config.addinivalue_line( @@ -288,12 +295,20 @@ def get_cli_or_ini(name, default=None): if not _hash_library_from_cli: hash_library = os.path.abspath(hash_library) + if not hasattr(config, "workerinput"): + uid = uuid.uuid4().hex + results_dir_path = results_dir or tempfile.mkdtemp() + config.pytest_mpl_uid = uid + config.pytest_mpl_results_dir = results_dir_path + + if config.pluginmanager.hasplugin("xdist"): + config.pluginmanager.register(XdistPlugin(), name="pytest_mpl_xdist_plugin") + plugin = ImageComparison( config, baseline_dir=baseline_dir, baseline_relative_dir=baseline_relative_dir, generate_dir=generate_dir, - results_dir=results_dir, hash_library=hash_library, generate_hash_library=generate_hash_lib, generate_summary=generate_summary, @@ -356,7 +371,6 @@ def __init__( baseline_dir=None, baseline_relative_dir=None, generate_dir=None, - results_dir=None, hash_library=None, generate_hash_library=None, generate_summary=None, @@ -372,7 +386,7 @@ def __init__( self.baseline_dir = baseline_dir self.baseline_relative_dir = path_is_not_none(baseline_relative_dir) self.generate_dir = path_is_not_none(generate_dir) - self.results_dir = path_is_not_none(results_dir) + self.results_dir = None self.hash_library = path_is_not_none(hash_library) self._hash_library_from_cli = _hash_library_from_cli # for backwards compatibility self.generate_hash_library = path_is_not_none(generate_hash_library) @@ -394,11 +408,6 @@ def __init__( self.deterministic = deterministic self.default_backend = default_backend - # Generate the containing dir for all test results - if not self.results_dir: - self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir)) - self.results_dir.mkdir(parents=True, exist_ok=True) - # Decide what to call the downloadable results hash library if self.hash_library is not None: self.results_hash_library_name = self.hash_library.name @@ -411,6 +420,14 @@ def __init__( self._test_stats = None self.return_value = {} + def pytest_sessionstart(self, session): + config = session.config + if hasattr(config, "workerinput"): + config.pytest_mpl_uid = config.workerinput["pytest_mpl_uid"] + config.pytest_mpl_results_dir = config.workerinput["pytest_mpl_results_dir"] + self.results_dir = Path(config.pytest_mpl_results_dir) + self.results_dir.mkdir(parents=True, exist_ok=True) + def get_logger(self): # configure a separate logger for this pluggin which is independent # of the options that are configured for pytest or for the code that @@ -932,27 +949,65 @@ def pytest_runtest_call(self, item): # noqa result._result = None result._excinfo = (type(e), e, e.__traceback__) + def generate_hash_library_json(self): + if hasattr(self.config, "workerinput"): + uid = self.config.pytest_mpl_uid + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + json_file = self.results_dir / f"generated-hashes-xdist-{uid}-{worker_id}.json" + else: + json_file = Path(self.config.rootdir) / self.generate_hash_library + json_file.parent.mkdir(parents=True, exist_ok=True) + with open(json_file, 'w') as f: + json.dump(self._generated_hash_library, f, indent=2) + return json_file + def generate_summary_json(self): - json_file = self.results_dir / 'results.json' + filename = "results.json" + if hasattr(self.config, "workerinput"): + uid = self.config.pytest_mpl_uid + worker_id = os.environ.get("PYTEST_XDIST_WORKER") + filename = f"results-xdist-{uid}-{worker_id}.json" + json_file = self.results_dir / filename with open(json_file, 'w') as f: json.dump(self._test_results, f, indent=2) return json_file - def pytest_unconfigure(self, config): + def pytest_sessionfinish(self, session): """ Save out the hash library at the end of the run. """ + config = session.config + try: + import xdist + is_xdist_controller = xdist.is_xdist_controller(session) + is_xdist_worker = xdist.is_xdist_worker(session) + except ImportError: + is_xdist_controller = False + is_xdist_worker = False + except Exception as e: + if "xdist" not in session.config.option: + is_xdist_controller = False + is_xdist_worker = False + else: + raise e + + if is_xdist_controller: # Merge results from workers + uid = config.pytest_mpl_uid + for worker_hashes in self.results_dir.glob(f"generated-hashes-xdist-{uid}-*.json"): + with worker_hashes.open() as f: + self._generated_hash_library.update(json.load(f)) + for worker_results in self.results_dir.glob(f"results-xdist-{uid}-*.json"): + with worker_results.open() as f: + self._test_results.update(json.load(f)) + result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json") if self.generate_hash_library is not None: - hash_library_path = Path(config.rootdir) / self.generate_hash_library - hash_library_path.parent.mkdir(parents=True, exist_ok=True) - with open(hash_library_path, "w") as fp: - json.dump(self._generated_hash_library, fp, indent=2) - if self.results_always: # Make accessible in results directory + hash_library_path = self.generate_hash_library_json() + if self.results_always and not is_xdist_worker: # Make accessible in results directory # Use same name as generated result_hash_library = self.results_dir / hash_library_path.name shutil.copy(hash_library_path, result_hash_library) - elif self.results_always and self.results_hash_library_name: + elif self.results_always and self.results_hash_library_name and not is_xdist_worker: result_hashes = {k: v['result_hash'] for k, v in self._test_results.items() if v['result_hash']} if len(result_hashes) > 0: # At least one hash comparison test @@ -964,6 +1019,8 @@ def pytest_unconfigure(self, config): if 'json' in self.generate_summary: summary = self.generate_summary_json() print(f"A JSON report can be found at: {summary}") + if is_xdist_worker: + return if result_hash_library.exists(): # link to it in the HTML kwargs["hash_library"] = result_hash_library.name if 'html' in self.generate_summary: diff --git a/tests/conftest.py b/tests/conftest.py index 9c3572d..3982922 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,3 +7,12 @@ @pytest.fixture def pytester(testdir): return testdir + + +def pytest_configure(config): + # Matplotlib versions build with pybind11 3.0.0 or later + # encounter import issues unless run in subprocess mode. + # See: https://github.com/matplotlib/pytest-mpl/issues/248 + import matplotlib + if Version(matplotlib.__version__) > Version("3.10.3"): + config.option.runpytest = "subprocess" diff --git a/tests/subtests/helpers.py b/tests/subtests/helpers.py index 87a2f5b..b48e2e0 100644 --- a/tests/subtests/helpers.py +++ b/tests/subtests/helpers.py @@ -1,4 +1,3 @@ -import os import re import json from pathlib import Path @@ -8,6 +7,8 @@ __all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex', 'remove_specific_hashes', 'transform_hashes', 'transform_images'] +MIN_EXPECTED_ITEMS = 20 # Rough minimum number of items in a summary to be valid + class MatchError(Exception): pass @@ -39,15 +40,26 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra # Load "correct" baseline hashes with open(baseline_hash_library, 'r') as f: baseline_hash_library = json.load(f) + if len(baseline_hash_library.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"baseline_hash_library only has {len(baseline_hash_library.keys())} items") else: baseline_hash_library = {} if result_hash_library and result_hash_library.exists(): # Load "correct" result hashes with open(result_hash_library, 'r') as f: result_hash_library = json.load(f) + if len(result_hash_library.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"result_hash_library only has {len(result_hash_library.keys())} items") else: result_hash_library = {} + b = baseline.get("a", baseline) + if len(b.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"baseline only has {len(b.keys())} items {b}") + r = result.get("a", result) + if len(r.keys()) < MIN_EXPECTED_ITEMS: + raise ValueError(f"result only has {len(r.keys())} items {r}") + # Get test names baseline_tests = set(baseline.keys()) result_tests = set(result.keys()) diff --git a/tests/subtests/test_subtest.py b/tests/subtests/test_subtest.py index 73f7c52..23dd4e1 100644 --- a/tests/subtests/test_subtest.py +++ b/tests/subtests/test_subtest.py @@ -48,7 +48,7 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True, - has_result_hashes=False, generating_hashes=False, testing_hashes=False, + has_result_hashes=False, generating_hashes=False, testing_hashes=False, n_xdist_workers=None, update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY): """ Run pytest (within pytest) and check JSON summary report. @@ -72,6 +72,9 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru both of `--mpl-hash-library` and `hash_library=` were not. testing_hashes : bool, optional, default=False Whether the subtest is comparing hashes and therefore needs baseline hashes generated. + n_xdist_workers : str or int, optional, default=None + Number of xdist workers to use, or "auto" to use all available cores. + None will disable xdist. If pytest-xdist is not installed, this will be ignored. """ if update_baseline and update_summary: raise ValueError("Cannot enable both `update_baseline` and `update_summary`.") @@ -109,6 +112,15 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru shutil.copy(expected_result_hash_library, baseline_hash_library) transform_hashes(baseline_hash_library) + try: + import xdist + if n_xdist_workers is None: + pytest_args += ["-p", "no:xdist"] + else: + pytest_args += ["-n", str(n_xdist_workers)] + except ImportError: + pass + # Run the test and record exit status status = subprocess.call(pytest_args + mpl_args + args) @@ -206,6 +218,21 @@ def test_html(tmp_path): assert (tmp_path / 'results' / 'styles.css').exists() +@pytest.mark.parametrize("num_workers", [None, 0, 1, 2]) +def test_html_xdist(request, tmp_path, num_workers): + if not request.config.pluginmanager.hasplugin("xdist"): + pytest.skip("Skipping: pytest-xdist is not installed") + run_subtest('test_results_always', tmp_path, + [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'], + has_result_hashes=True, n_xdist_workers=num_workers) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + assert (tmp_path / 'results' / 'extra.js').exists() + assert (tmp_path / 'results' / 'styles.css').exists() + if num_workers is not None: + assert len(list((tmp_path / 'results').glob('generated-hashes-xdist-*-*.json'))) == 0 + assert len(list((tmp_path / 'results').glob('results-xdist-*-*.json'))) == num_workers + + def test_html_hashes_only(tmp_path): run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG, *HASH_COMPARISON_MODE], @@ -260,6 +287,24 @@ def test_html_generate(tmp_path): assert (tmp_path / 'results' / 'fig_comparison.html').exists() +@pytest.mark.parametrize("num_workers", [None, 0, 1, 2]) +def test_html_generate_xdist(request, tmp_path, num_workers): + # generating hashes and images; no testing + if not request.config.pluginmanager.hasplugin("xdist"): + pytest.skip("Skipping: pytest-xdist is not installed") + run_subtest('test_html_generate', tmp_path, + [rf'--mpl-generate-path={tmp_path}', + rf'--mpl-generate-hash-library={tmp_path / "test_hashes.json"}'], + summaries=['html'], xfail=False, has_result_hashes="test_hashes.json", + generating_hashes=True, n_xdist_workers=num_workers) + assert (tmp_path / 'results' / 'fig_comparison.html').exists() + assert (tmp_path / 'results' / 'extra.js').exists() + assert (tmp_path / 'results' / 'styles.css').exists() + if num_workers is not None: + assert len(list((tmp_path / 'results').glob('generated-hashes-xdist-*-*.json'))) == num_workers + assert len(list((tmp_path / 'results').glob('results-xdist-*-*.json'))) == num_workers + + def test_html_generate_images_only(tmp_path): # generating images; no testing run_subtest('test_html_generate_images_only', tmp_path, diff --git a/tox.ini b/tox.ini index cff7325..eb6ccfe 100644 --- a/tox.ini +++ b/tox.ini @@ -19,6 +19,7 @@ setenv = changedir = .tmp/{envname} description = run tests deps = + pytest-xdist mpl20: matplotlib==2.0.* mpl21: matplotlib==2.1.* mpl22: matplotlib==2.2.* @@ -58,7 +59,7 @@ commands = # Make sure the tests pass with and without --mpl # Use -m so pytest skips "subtests" which always apply --mpl pytest '{toxinidir}' -m "mpl_image_compare" {posargs} - coverage run --source=pytest_mpl -m pytest '{toxinidir}' --mpl + coverage run --source=pytest_mpl -m pytest '{toxinidir}' -n auto --mpl coverage xml -o '{toxinidir}{/}coverage.xml' [testenv:codestyle]