Add subtests

Author: ConorMacBride

tests/subtests/subtest.py

@@ -0,0 +1,115 @@
+import matplotlib.pyplot as plt
+import pytest
+
+
+def plot(line, **kwargs):
+    fig = plt.figure()
+    ax = fig.add_subplot(1, 1, 1)
+    ax.plot(line, **kwargs)
+    return fig
+
+
+# ### Test all permutations of:
+# baseline hash: match, diff, or missing
+# baseline image: match, diff, or missing
+
+# hash match
+
+@pytest.mark.mpl_image_compare()
+def test_hmatch_imatch():
+    return plot([1, 2, 3, 4])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmatch_idiff():
+    return plot([1, 3, 2, 4])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmatch_imissing():
+    return plot([4, 3, 2, 1])
+
+
+# hash diff
+
+@pytest.mark.mpl_image_compare()
+def test_hdiff_imatch():
+    return plot([1, 4, 2, 3])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hdiff_idiff():
+    return plot([1, 2, 4, 3])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hdiff_imissing():
+    return plot([3, 2, 4, 1])
+
+
+# hash missing
+
+@pytest.mark.mpl_image_compare()
+def test_hmissing_imatch():
+    return plot([1, 3, 4, 2])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmissing_idiff():
+    return plot([1, 4, 3, 2])
+
+
+@pytest.mark.mpl_image_compare()
+def test_hmissing_imissing():
+    return plot([2, 4, 3, 1])
+
+
+# ### Specialized tests
+
+# Tolerance: high to force image match
+@pytest.mark.mpl_image_compare(tolerance=200)
+def test_hdiff_imatch_tolerance():
+    return plot([1, 2, 3, 4], linestyle='--')
+
+
+# Tolerance: non-default to verify option recorded in JSON
+@pytest.mark.mpl_image_compare(tolerance=3)
+def test_hdiff_idiff_tolerance():
+    return plot([1, 2, 3, 4], linestyle='--')
+
+
+# Savefig kwargs
+@pytest.mark.mpl_image_compare(savefig_kwargs={'facecolor': 'r'})
+def test_hdiff_imatch_savefig():
+    return plot([1, 2, 3, 4])
+
+
+# Different baseline directory
+# TODO: Test with a remote `baseline_dir`
+@pytest.mark.mpl_image_compare(baseline_dir='baseline/other')
+def test_hdiff_imatch_baselinedir():
+    return plot([4, 2, 1, 4])
+
+
+# Different filename
+@pytest.mark.mpl_image_compare(filename='test_hdiff_imatch_filename_other.png')
+def test_hdiff_imatch_filename():
+    return plot([4, 2, 1, 4])
+
+
+# Different hash library
+@pytest.mark.mpl_image_compare(hash_library='hashes/other/other.json')
+def test_hdiff_imatch_filename():
+    return plot([4, 2, 1, 4])
+
+
+# Different style
+@pytest.mark.mpl_image_compare(style='fivethirtyeight')
+def test_hdiff_imatch_style():
+    return plot([4, 2, 1, 4])
+
+
+# Remove text
+@pytest.mark.mpl_image_compare(remove_text=True)
+def test_hdiff_imatch_removetext():
+    return plot([4, 2, 1, 4])

tests/subtests/test_subtest.py

@@ -18,6 +18,8 @@
 RESULT_LIBRARY = Path(__file__).parent / 'result_hashes' / (VERSION_ID + ".json")
 HASH_LIBRARY_FLAG = rf'--mpl-hash-library={HASH_LIBRARY}'
 
+BASELINE_IMAGES_FLAG = '--mpl-baseline-path=baseline'
+
 TEST_FILE = Path(__file__).parent / 'subtest.py'
 
 # Global settings to update baselines when running pytest
@@ -101,3 +103,48 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru
 
     # Ensure reported images exist
     assert_existence(result_summary, path=results_path)
+
+
+def test_default(tmp_path):
+    run_subtest('test_default', tmp_path, [])
+
+
+def test_hash(tmp_path):
+    run_subtest('test_hash', tmp_path, [HASH_LIBRARY_FLAG])
+
+
+def test_hybrid(tmp_path):
+    run_subtest('test_hybrid', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG])
+
+
+def test_results_always(tmp_path):
+    run_subtest('test_results_always', tmp_path,
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG, '--mpl-results-always'])
+
+
+def test_html(tmp_path):
+    run_subtest('test_results_always', tmp_path,
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG], summaries=['html'])
+    assert (tmp_path / 'results' / 'fig_comparison.html').exists()
+    assert (tmp_path / 'results' / 'extra.js').exists()
+    assert (tmp_path / 'results' / 'styles.css').exists()
+
+
+def test_html_hashes_only(tmp_path):
+    run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'])
+    assert (tmp_path / 'results' / 'fig_comparison.html').exists()
+    assert (tmp_path / 'results' / 'extra.js').exists()
+    assert (tmp_path / 'results' / 'styles.css').exists()
+
+
+def test_html_images_only(tmp_path):
+    run_subtest('test_html_images_only', tmp_path, [], summaries=['html'])
+    assert (tmp_path / 'results' / 'fig_comparison.html').exists()
+    assert (tmp_path / 'results' / 'extra.js').exists()
+    assert (tmp_path / 'results' / 'styles.css').exists()
+
+
+def test_basic_html(tmp_path):
+    run_subtest('test_results_always', tmp_path,
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG], summaries=['basic-html'])
+    assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists()