Merge pull request matplotlib#19179 from dstansby/test_fig_check

Check that no new figures are created in image comparison tests

Author: jklymak

lib/matplotlib/testing/decorators.py

@@ -408,6 +408,12 @@ def check_figures_equal(*, extensions=("png", "pdf", "svg"), tol=0):
     tol : float
         The RMS threshold above which the test is considered failed.
 
+    Raises
+    ------
+    RuntimeError
+        If any new figures are created (and not subsequently closed) inside
+        the test function.
+
     Examples
     --------
     Check that calling `.Axes.plot` with a single argument plots it against
@@ -421,6 +427,7 @@ def test_plot(fig_test, fig_ref):
     """
     ALLOWED_CHARS = set(string.digits + string.ascii_letters + '_-[]()')
     KEYWORD_ONLY = inspect.Parameter.KEYWORD_ONLY
+
     def decorator(func):
         import pytest
 
@@ -444,7 +451,16 @@ def wrapper(*args, ext, request, **kwargs):
             try:
                 fig_test = plt.figure("test")
                 fig_ref = plt.figure("reference")
+                # Keep track of number of open figures, to make sure test
+                # doesn't create any new ones
+                n_figs = len(plt.get_fignums())
                 func(*args, fig_test=fig_test, fig_ref=fig_ref, **kwargs)
+                if len(plt.get_fignums()) > n_figs:
+                    raise RuntimeError('Number of open figures changed during '
+                                       'test. Make sure you are plotting to '
+                                       'fig_test or fig_ref, or if this is '
+                                       'deliberate explicitly close the '
+                                       'new figure(s) inside the test.')
                 test_image_path = result_dir / (file_name + "." + ext)
                 ref_image_path = result_dir / (file_name + "-expected." + ext)
                 fig_test.savefig(test_image_path)

lib/matplotlib/tests/test_axes.py

@@ -109,10 +109,10 @@ def test_acorr(fig_test, fig_ref):
     x = np.random.normal(0, 1, Nx).cumsum()
     maxlags = Nx-1
 
-    fig_test, ax_test = plt.subplots()
+    ax_test = fig_test.subplots()
     ax_test.acorr(x, maxlags=maxlags)
 
-    fig_ref, ax_ref = plt.subplots()
+    ax_ref = fig_ref.subplots()
     # Normalized autocorrelation
     norm_auto_corr = np.correlate(x, x, mode="full")/np.dot(x, x)
     lags = np.arange(-maxlags, maxlags+1)

lib/matplotlib/tests/test_testing.py

@@ -1,5 +1,8 @@
 import warnings
+
 import pytest
+
+import matplotlib.pyplot as plt
 from matplotlib.testing.decorators import check_figures_equal
 
 
@@ -22,3 +25,17 @@ def test_wrap_failure():
         @check_figures_equal()
         def should_fail(test, ref):
             pass
+
+
+@pytest.mark.xfail(raises=RuntimeError, strict=True,
+                   reason='Test for check_figures_equal test creating '
+                          'new figures')
+@check_figures_equal()
+def test_check_figures_equal_extra_fig(fig_test, fig_ref):
+    plt.figure()
+
+
+@check_figures_equal()
+def test_check_figures_equal_closed_fig(fig_test, fig_ref):
+    fig = plt.figure()
+    plt.close(fig)