Initial options for summary and named results dir

Author: Cadair

pytest_mpl/plugin.py

@@ -109,6 +109,9 @@ def pytest_addoption(parser):
     group.addoption('--mpl-hash-library',
                     help="json library of image hashes, relative to "
                     "location where py.test is run", action='store')
+    group.addoption('--mpl-generate-summary', action='store_true',
+                    help="Generate a summary HTML report of any failed tests"
+                    ", in --mpl-results-path")
 
     results_path_help = "directory for test results, relative to location where py.test is run"
     group.addoption('--mpl-results-path', help=results_path_help, action='store')
@@ -130,6 +133,7 @@ def pytest_configure(config):
         generate_hash_lib = config.getoption("--mpl-generate-hash-library")
         results_dir = config.getoption("--mpl-results-path") or config.getini("mpl-results-path")
         hash_library = config.getoption("--mpl-hash-library")
+        generate_summary = config.getoption("--mpl-generate-summary")
 
         if config.getoption("--mpl-baseline-relative"):
             baseline_relative_dir = config.getoption("--mpl-baseline-path")
@@ -159,7 +163,8 @@ def pytest_configure(config):
                                                       generate_dir=generate_dir,
                                                       results_dir=results_dir,
                                                       hash_library=hash_library,
-                                                      generate_hash_library=generate_hash_lib))
+                                                      generate_hash_library=generate_hash_lib,
+                                                      generate_summary=generate_summary))
 
     else:
 
@@ -215,7 +220,8 @@ def __init__(self,
                  generate_dir=None,
                  results_dir=None,
                  hash_library=None,
-                 generate_hash_library=None
+                 generate_hash_library=None,
+                 generate_summary=False
                  ):
         self.config = config
         self.baseline_dir = baseline_dir
@@ -224,8 +230,12 @@ def __init__(self,
         self.results_dir = path_is_not_none(results_dir)
         self.hash_library = path_is_not_none(hash_library)
         self.generate_hash_library = path_is_not_none(generate_hash_library)
-        if self.results_dir and not self.results_dir.exists():
-            self.results_dir.mkdir()
+        self.generate_summary = bool(generate_summary)
+
+        # Generate the containing dir for all test results
+        if not self.results_dir:
+            self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir))
+        self.results_dir.mkdir(parents=True, exist_ok=True)
 
         # We need global state to store all the hashes generated over the run
         self._generated_hash_library = {}
@@ -251,11 +261,20 @@ def generate_filename(self, item):
 
         return filename
 
-    def make_results_dir(self, item):
+    def generate_test_name(self, item):
+        """
+        Generate a unique name for the hash for this test.
+        """
+        return f"{item.module.__name__}.{item.name}"
+
+    def make_test_results_dir(self, item):
         """
         Generate the directory to put the results in.
         """
-        return Path(tempfile.mkdtemp(dir=self.results_dir))
+        test_name = self.generate_test_name(item)
+        results_dir = self.results_dir / test_name
+        results_dir.mkdir(exist_ok=True, parents=True)
+        return results_dir
 
     def baseline_directory_specified(self, item):
         """
@@ -328,12 +347,6 @@ def generate_baseline_image(self, item, fig):
         close_mpl_figure(fig)
         pytest.skip("Skipping test, since generating image")
 
-    def generate_hash_name(self, item):
-        """
-        Generate a unique name for the hash for this test.
-        """
-        return f"{item.module.__name__}.{item.name}"
-
     def generate_image_hash(self, item, fig):
         """
         For a `matplotlib.figure.Figure`, returns the SHA256 hash as a hexadecimal
@@ -365,7 +378,7 @@ def compare_image_to_baseline(self, item, fig, result_dir):
 
         baseline_image_ref = self.obtain_baseline_image(item, result_dir)
 
-        test_image = (result_dir / self.generate_filename(item)).absolute()
+        test_image = (result_dir / "result.png").absolute()
         fig.savefig(str(test_image), **savefig_kwargs)
 
         if not os.path.exists(baseline_image_ref):
@@ -377,7 +390,7 @@ def compare_image_to_baseline(self, item, fig, result_dir):
 
         # distutils may put the baseline images in non-accessible places,
         # copy to our tmpdir to be sure to keep them in case of failure
-        baseline_image = (result_dir / f"baseline-{self.generate_filename(item)}").absolute()
+        baseline_image = (result_dir / "baseline.png").absolute()
         shutil.copyfile(baseline_image_ref, baseline_image)
 
         # Compare image size ourselves since the Matplotlib
@@ -407,7 +420,7 @@ def compare_image_to_hash_library(self, item, fig, result_dir):
             pytest.fail(f"Can't find hash library at path {hash_library_filename}")
 
         hash_library = self.load_hash_library(hash_library_filename)
-        hash_name = self.generate_hash_name(item)
+        hash_name = self.generate_test_name(item)
 
         if hash_name not in hash_library:
             return f"Hash for test '{hash_name}' not found in {hash_library_filename}."
@@ -489,12 +502,12 @@ def item_function_wrapper(*args, **kwargs):
                     self.generate_baseline_image(item, fig)
 
                 if self.generate_hash_library is not None:
-                    hash_name = self.generate_hash_name(item)
+                    hash_name = self.generate_test_name(item)
                     self._generated_hash_library[hash_name] = self.generate_image_hash(item, fig)
 
                 # Only test figures if we are not generating hashes or images
                 if self.generate_dir is None and self.generate_hash_library is None:
-                    result_dir = self.make_results_dir(item)
+                    result_dir = self.make_test_results_dir(item)
 
                     # Compare to hash library
                     if self.hash_library or compare.kwargs.get('hash_library', None):
@@ -507,7 +520,8 @@ def item_function_wrapper(*args, **kwargs):
                     close_mpl_figure(fig)
 
                     if msg is None:
-                        shutil.rmtree(result_dir)
+                        pass
+                        # shutil.rmtree(result_dir)
                     else:
                         pytest.fail(msg, pytrace=False)
 
@@ -528,6 +542,9 @@ def pytest_unconfigure(self, config):
             with open(hash_library_path, "w") as fp:
                 json.dump(self._generated_hash_library, fp, indent=2)
 
+        if self.generate_summary:
+            breakpoint()
+
 
 class FigureCloser(object):
     """

tests/test_pytest_mpl.py

@@ -148,18 +148,15 @@ def test_output_dir(tmpdir):
     with open(test_file, 'w') as f:
         f.write(TEST_OUTPUT_DIR)
 
-    output_dir = tmpdir.join('test_output_dir').strpath
+    output_dir = tmpdir.join('test_output_dir')
 
     # When we run the test, we should get output images where we specify
     code = call_pytest([f'--mpl-results-path={output_dir}',
                         '--mpl', test_file])
 
     assert code != 0
-    assert os.path.exists(output_dir)
-
-    # Listdir() is to get the random name that the output for the one test is written into
-    assert os.path.exists(os.path.join(output_dir, os.listdir(output_dir)[0],
-                                       'test_output_dir.png'))
+    assert output_dir.exists()
+    assert (output_dir / "test.test_output_dir" / "result.png").exists()
 
 
 TEST_GENERATE = """