Merge pull request #128 from csiro-coasts/pytest-mpl

Add image comparison tests using pytest-mpl

Author: mx-moth

.github/workflows/ci.yaml

@@ -83,6 +83,9 @@ jobs:
         shell: bash -l {0}
         run: |
           pytest -v \
+            --mpl \
+            --mpl-results-path=./mpl-results \
+            --mpl-generate-summary html \
             --junitxml=junit-py${{ matrix.python-version }}.xml \
             --cov --cov-report term \
             --cov-report xml:coverage-${{ matrix.python-version }}.xml
@@ -92,7 +95,16 @@ jobs:
         if: always()
         with:
           report_paths: 'junit-py*.xml'
-          check_name: "JUnit Test Report - Python ${{ matrix.python-version }}"
+          check_name: "JUnit Test Report - python ${{ matrix.python-version }}, ${{ matrix.experimental && 'latest' || 'pinned' }} dependencies"
+
+      - name: MPL image comparison report
+        uses: actions/upload-artifact@v3
+        if: ${{ failure() }}
+        with:
+          name: "MPL image comparison report - python ${{ matrix.python-version }}, ${{ matrix.experimental && 'latest' || 'pinned' }} dependencies"
+          path: './mpl-results'
+          # No guarantee that the test failures were due to image comparisons
+          if-no-files-found: 'ignore'
 
       - uses: actions/upload-artifact@v3
         with:

continuous-integration/requirements.txt

@@ -98,6 +98,7 @@ jinja2==3.1.2
     #   bokeh
     #   dask
     #   distributed
+    #   pytest-mpl
     #   sphinx
 kiwisolver==1.4.4
     # via matplotlib
@@ -115,6 +116,7 @@ matplotlib==3.7.2
     # via
     #   cartopy
     #   emsarray (setup.cfg)
+    #   pytest-mpl
 mccabe==0.7.0
     # via flake8
 msgpack==1.0.5
@@ -156,6 +158,7 @@ packaging==23.1
     #   pydata-sphinx-theme
     #   pyproject-api
     #   pytest
+    #   pytest-mpl
     #   sphinx
     #   tox
     #   xarray
@@ -172,6 +175,7 @@ pillow==10.0.0
     # via
     #   bokeh
     #   matplotlib
+    #   pytest-mpl
 platformdirs==3.10.0
     # via
     #   pooch
@@ -214,8 +218,11 @@ pytest==7.4.0
     # via
     #   emsarray (setup.cfg)
     #   pytest-cov
+    #   pytest-mpl
 pytest-cov==4.1.0
     # via emsarray (setup.cfg)
+pytest-mpl==0.16.1
+    # via emsarray (setup.cfg)
 python-dateutil==2.8.2
     # via
     #   matplotlib

docs/releases/development.rst

@@ -17,3 +17,6 @@ Next release (in development)
 * Use `PEP 585 generic type annotations <https://peps.python.org/pep-0585/>`_
   and stop using `PEP 563 postponed annotation evaluation <https://peps.python.org/pep-0563/>`_
   (:issue:`109`, :pr:`127`).
+* Use `pytest-mpl <https://pypi.org/project/pytest-mpl/>`_ for image comparison tests
+  for testing plotting methods
+  (:pr:`128`).

pyproject.toml

@@ -5,6 +5,12 @@ requires = [
 build-backend = "setuptools.build_meta"
 
 [tool.pytest.ini_options]
+addopts = [
+	# TODO Work around till the mpl-baseline-path ini option is included in the
+	# next pytest-mpl release.
+	"--mpl-baseline-path", "tests/baseline_images",
+]
+
 testpaths = ["tests"]
 filterwarnings = [
 	"error",
@@ -18,6 +24,11 @@ markers = [
 	"tutorial: Tests that involve the tutorial datasets",
 ]
 
+mpl-use-full-test-name = true
+# TODO This option will be included in the next version of pytest-mpl.
+# Till then, it is added using addopts above
+# mpl-baseline-path = "tests/baseline_images"
+
 [tool.mypy]
 python_version = "3.9"
 plugins = ["numpy.typing.mypy_plugin"]

setup.cfg

@@ -63,6 +63,7 @@ testing =
 	%(docs)s
 	pytest
 	pytest-cov
+	pytest-mpl
 	mypy
 	pandas-stubs
 	types-pytz

tests/baseline_images/tests.test_plot.test_plot.png

@@ -52,6 +52,7 @@ def test_add_landmarks():
         assert text.xy == landmark[1].coords.xy
 
 
+@pytest.mark.mpl_image_compare
 @pytest.mark.matplotlib(mock_coast=True)
 @pytest.mark.tutorial
 def test_plot(
@@ -68,8 +69,7 @@ def test_plot(
     axes = figure.axes[0]
     assert axes.get_title() == 'Temperature\n2022-05-11T14:00'
 
-    matplotlib.pyplot.savefig(tmp_path / 'plot.png')
-    logger.info("Saved plot to %r", tmp_path / 'plot.png')
+    return figure
 
 
 @pytest.mark.matplotlib(mock_coast=True)

tests/baseline_images/tests.test_transect.test_plot.png

@@ -10,7 +10,8 @@
 logger = logging.getLogger(__name__)
 
 
-@pytest.mark.matplotlib(mock_coast=True)
+@pytest.mark.mpl_image_compare
+@pytest.mark.matplotlib
 @pytest.mark.tutorial
 def test_plot(
     datasets: pathlib.Path,
@@ -53,10 +54,10 @@ def test_plot(
     # This is the variable units
     assert colorbar.get_ylabel() == 'degrees C'
 
-    matplotlib.pyplot.savefig(tmp_path / 'plot.png')
-    logger.info("Saved plot to %r", tmp_path / 'plot.png')
+    return figure
 
 
+@pytest.mark.mpl_image_compare
 @pytest.mark.matplotlib(mock_coast=True)
 @pytest.mark.tutorial
 def test_plot_no_intersection(
@@ -95,5 +96,4 @@ def test_plot_no_intersection(
     # This is the variable units
     assert colorbar.get_ylabel() == 'degrees C'
 
-    matplotlib.pyplot.savefig(tmp_path / 'plot.png')
-    logger.info("Saved plot to %r", tmp_path / 'plot.png')
+    return figure