Implement hybrid comparisons

Author: Cadair

pytest_mpl/plugin.py

@@ -121,9 +121,9 @@ def pytest_configure(config):
                             "mpl_image_compare: Compares matplotlib figures "
                             "against a baseline image")
 
-    if (config.getoption("--mpl") or  # noqa
-        config.getoption("--mpl-generate-path") is not None or  # noqa
-        config.getoption("--mpl-generate-hash-library") is not None):  # noqa
+    if (config.getoption("--mpl") or
+            config.getoption("--mpl-generate-path") is not None or
+            config.getoption("--mpl-generate-hash-library") is not None):
 
         baseline_dir = config.getoption("--mpl-baseline-path")
         generate_dir = config.getoption("--mpl-generate-path")
@@ -257,6 +257,14 @@ def make_results_dir(self, item):
         """
         return Path(tempfile.mkdtemp(dir=self.results_dir))
 
+    def baseline_directory_specified(self, item):
+        """
+        Returns `True` if a non-default baseline directory is specified.
+        """
+        compare = self.get_compare(item)
+        item_baseline_dir = compare.kwargs.get('baseline_dir', None)
+        return item_baseline_dir or self.baseline_dir or self.baseline_relative_dir
+
     def get_baseline_directory(self, item):
         """
         Return a full path to the baseline directory, either local or remote.
@@ -409,25 +417,32 @@ def compare_image_to_hash_library(self, item, fig, result_dir):
         if test_hash == hash_library[hash_name]:
             return
 
+        error_message = (f"hash {test_hash} doesn't match hash "
+                         f"{hash_library[hash_name]} in library "
+                         f"{hash_library_filename} for test {hash_name}.")
+
+        # If the compare has only been specified with hash and not baseline
+        # dir, don't attempt to find a baseline image at the default path.
+        if not self.baseline_directory_specified(item):
+            return error_message
+
+        baseline_image_path = self.obtain_baseline_image(item, result_dir)
         try:
-            baseline_image = self.obtain_baseline_image(item, result_dir)
+            baseline_image = baseline_image_path
             baseline_image = None if not baseline_image.exists() else baseline_image
         except Exception:
             baseline_image = None
 
-        hash_error = (f"hash {test_hash} doesn't match hash "
-                      f"{hash_library[hash_name]} in library "
-                      f"{hash_library_filename} for test {hash_name}.")
-
         if baseline_image is None:
-            return hash_error
+            error_message += f"\nUnable to find baseline image {baseline_image_path}."
 
-        comparison_error = self.compare_image_to_baseline(item, fig, result_dir)
+        if baseline_image is None:
+            return error_message
 
-        if not comparison_error:
-            return hash_error + "\nHowever, the comparison to the baseline image succeeded."
+        comparison_error = (self.compare_image_to_baseline(item, fig, result_dir) or
+                            "\nHowever, the comparison to the baseline image succeeded.")
 
-        return f"{hash_error}\n{comparison_error}"
+        return f"{error_message}\n{comparison_error}"
 
     def pytest_runtest_setup(self, item):  # noqa
 

setup.cfg

@@ -40,6 +40,7 @@ testpaths = "tests"
 
 [flake8]
 max-line-length = 100
+ignore = W504
 
 [pycodestyle]
 max_line_length = 100

tests/baseline/hybrid/fail/test_hash_fail_hybrid.png

@@ -1,4 +1,5 @@
 {
   "test_pytest_mpl.test_hash_succeeds": "cd01b7a39330033b18d54b507635236214cae5e24f9e09d84b56462a9ac81052",
-  "test.test_hash_fails": "FAIL"
+  "test.test_hash_fails": "FAIL",
+  "test.test_hash_fail_hybrid": "FAIL"
 }

tests/baseline/hybrid/succeed/test_hash_fail_hybrid.png

@@ -30,6 +30,8 @@
                 "hashes" / hash_filename)
 
 fail_hash_library = Path(__file__).parent / "baseline" / "test_hash_lib.json"
+baseline_dir_abs = Path(__file__).parent / "baseline" / baseline_subdir
+hash_baseline_dir_abs = Path(__file__).parent / "baseline" / "hybrid"
 
 
 WIN = sys.platform.startswith('win')
@@ -42,6 +44,15 @@ def call_pytest(args):
     return subprocess.call([sys.executable, '-m', 'pytest', '-s'] + args)
 
 
+def assert_pytest_fails_with(args, output_substring):
+    try:
+        subprocess.check_output([sys.executable, '-m', 'pytest', '-s'] + args)
+    except subprocess.CalledProcessError as exc:
+        output = exc.output.decode()
+        assert output_substring in output, output
+        return output
+
+
 @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
                                tolerance=DEFAULT_TOLERANCE)
 def test_succeeds():
@@ -172,10 +183,7 @@ def test_generate(tmpdir):
     gen_dir = tmpdir.mkdir('spam').mkdir('egg').strpath
 
     # If we don't generate, the test will fail
-    try:
-        subprocess.check_output([sys.executable, '-m', 'pytest', '-s', '--mpl', test_file])
-    except subprocess.CalledProcessError as exc:
-        assert b'Image file not found for comparison test' in exc.output, exc.output.decode()
+    assert_pytest_fails_with(['--mpl', test_file], 'Image file not found for comparison test')
 
     # If we do generate, the test should succeed and a new file will appear
     code = call_pytest([f'--mpl-generate-path={gen_dir}', test_file])
@@ -282,10 +290,50 @@ def test_hash_fails(tmpdir):
         f.write(TEST_FAILING_HASH)
 
     # If we use --mpl, it should detect that the figure is wrong
-    try:
-        subprocess.check_output([sys.executable, '-m', 'pytest', '-s', '--mpl', test_file])
-    except subprocess.CalledProcessError as exc:
-        assert b"doesn't match hash FAIL in library" in exc.output, exc.output.decode()
+    output = assert_pytest_fails_with(['--mpl', test_file], "doesn't match hash FAIL in library")
+    # We didn't specify a baseline dir so we shouldn't attempt to find one
+    assert "Unable to find baseline image" not in output, output
+
+    # If we don't use --mpl option, the test should succeed
+    code = call_pytest([test_file])
+    assert code == 0
+
+
+TEST_FAILING_HYBRID = f"""
+import pytest
+import matplotlib.pyplot as plt
+@pytest.mark.mpl_image_compare(hash_library="{fail_hash_library}")
+def test_hash_fail_hybrid():
+    fig = plt.figure()
+    ax = fig.add_subplot(1,1,1)
+    ax.plot([1,2,3])
+    return fig
+"""
+
+
+def test_hash_fail_hybrid(tmpdir):
+
+    test_file = tmpdir.join('test.py').strpath
+    with open(test_file, 'w', encoding='ascii') as f:
+        f.write(TEST_FAILING_HYBRID)
+
+    # Assert that image comparison runs and fails
+    output = assert_pytest_fails_with(['--mpl', test_file,
+                                       f'--mpl-baseline-path={hash_baseline_dir_abs / "fail"}'],
+                                      "doesn't match hash FAIL in library")
+    assert "Error: Image files did not match." in output, output
+
+    # Assert reports missing baseline image
+    output = assert_pytest_fails_with(['--mpl', test_file,
+                                       '--mpl-baseline-path=/not/a/path'],
+                                      "doesn't match hash FAIL in library")
+    assert "Unable to find baseline image" in output, output
+
+    # Assert reports image comparison succeeds
+    output = assert_pytest_fails_with(['--mpl', test_file,
+                                       f'--mpl-baseline-path={hash_baseline_dir_abs / "succeed"}'],
+                                      "doesn't match hash FAIL in library")
+    assert "However, the comparison to the baseline image succeeded." in output, output
 
     # If we don't use --mpl option, the test should succeed
     code = call_pytest([test_file])
@@ -310,12 +358,13 @@ def test_hash_missing(tmpdir):
     with open(test_file, 'w') as f:
         f.write(TEST_MISSING_HASH)
 
-    # If we use --mpl, it should detect that the figure is wrong
-    try:
-        subprocess.check_output([sys.executable, '-m', 'pytest', '-s', '--mpl', test_file,
-                                 f'--mpl-hash-library={os.path.join(baseline_dir, "test_hash_lib.json")}'])  # noqa
-    except subprocess.CalledProcessError as exc:
-        assert b"Can't find hash library at path" in exc.output, exc.output.decode()
+    # Assert fails if hash library missing
+    assert_pytest_fails_with(['--mpl', test_file, '--mpl-hash-library=/not/a/path'],
+                             "Can't find hash library at path")
+
+    # Assert fails if hash not in library
+    assert_pytest_fails_with(['--mpl', test_file, f'--mpl-hash-library={fail_hash_library}'],
+                             "Hash for test 'test.test_hash_missing' not found in")
 
     # If we don't use --mpl option, the test should succeed
     code = call_pytest([test_file])