Use Pytest for running tests (#726)

* Add pytest and required plugins to dev requirements

* Add settings to .coveragerc

* Update Makefile to use pytest where possible

* Update Jenkins scripts to use Makefile

* Update Jenkinsfile to install new requirements

NOTE: Revert this change once the Jenkins environment is updated

* Remove test run scripts for unit, intregration, and install tests

* Update installation instructions and reference them in CI guide

* Update CHANGELOG.md

* Update doc/guide/install.rst

---------

Co-authored-by: emanuel-schmid <[email protected]>
Co-authored-by: Emanuel Schmid <[email protected]>

Author: peanutfun

.coveragerc

@@ -0,0 +1,25 @@
+# .coveragerc to control coverage.py
+
+[run]
+# Also report branch coverage
+branch = True
+# Set concurrency type for correct coverage of multi-processing code
+concurrency = multiprocessing
+
+[paths]
+source = climada/
+
+[report]
+# Regexes for lines to exclude from consideration
+exclude_also =
+    # Main code is not run
+    if __name__ == .__main__.:
+
+    # Abtract methods are not run
+    @(abc\.)?abstractmethod
+
+# Never fail when reporting
+ignore_errors = True
+
+[html]
+directory = coverage

CHANGELOG.md

@@ -9,6 +9,9 @@ Code freeze date: YYYY-MM-DD
 ### Dependency Updates
 
 Added:
+- `pytest` [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
+- `pytest-cov` [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
+- `pytest-subtests` [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
 
 Changed:
 
@@ -42,6 +45,7 @@ Removed:
 - `Exposures.affected_total_value` now takes a hazard intensity threshold as argument. Affected values are only those for which at least one event exceeds the threshold. (previously, all exposures points with an assigned centroid were considered affected). By default the centroids are reassigned. [#702](https://github.com/CLIMADA-project/climada_python/pull/702) [#730](https://github.com/CLIMADA-project/climada_python/pull/730)
 - Add option to pass region ID to `LitPop.from_shape` [#720](https://github.com/CLIMADA-project/climada_python/pull/720)
 - Slightly improved performance on `LitPop`-internal computations [#720](https://github.com/CLIMADA-project/climada_python/pull/720)
+- Use `pytest` for executing tests [#726](https://github.com/CLIMADA-project/climada_python/pull/726)
 - Users can opt-out of the climada specific logging definitions and freely configure logging to their will, by setting the config value `logging.managed` to `false`. [#724](https://github.com/CLIMADA-project/climada_python/pull/724)
 - Add option to read additional variables from IBTrACS when using `TCTracks.from_ibtracs_netcdf` [#728](https://github.com/CLIMADA-project/climada_python/pull/728)
 

Makefile

@@ -2,6 +2,14 @@
 # test, coverage and lint
 ###
 
+PYTEST_JUNIT_ARGS = --junitxml=tests_xml/tests.xml
+
+PYTEST_COV_ARGS = \
+	--cov --cov-config=.coveragerc --cov-report html --cov-report xml \
+	--cov-report term:skip-covered
+
+PYTEST_ARGS = $(PYTEST_JUNIT_ARGS) $(PYTEST_COV_ARGS)
+
 .PHONY : help
 help:  ## Use one of the following instructions:
 	@fgrep -h "##" $(MAKEFILE_LIST) | fgrep -v fgrep | sed -e 's/\\$$//' | sed -e 's/##//'
@@ -12,13 +20,12 @@ lint : ## Static code analysis with Pylint
 
 .PHONY : unit_test
 unit_test : ## Unit tests execution with coverage and xml reports
-	python -m coverage run tests_runner.py unit
-	python -m coverage xml -o coverage.xml
-	python -m coverage html -d coverage
+	pytest $(PYTEST_ARGS) --ignore=climada/test climada/
 
 .PHONY : install_test
 install_test : ## Test installation was successful
-	python tests_install.py report
+	pytest $(PYTEST_JUNIT_ARGS) climada/engine/test/test_cost_benefit.py \
+	climada/engine/test/test_impact.py
 
 .PHONY : data_test
 data_test : ## Test data APIs
@@ -30,21 +37,14 @@ notebook_test : ## Test notebooks in doc/tutorial
 
 .PHONY : integ_test
 integ_test : ## Integration tests execution with xml reports
-	python -m coverage run --parallel-mode --concurrency=multiprocessing tests_runner.py integ
-	python -m coverage combine
-	python -m coverage xml -o coverage.xml
-	python -m coverage html -d coverage
+	pytest $(PYTEST_ARGS) climada/test/
 
 .PHONY : test
 test : ## Unit and integration tests execution with coverage and xml reports
-	python -m coverage run --parallel-mode --concurrency=multiprocessing tests_runner.py unit
-	python -m coverage run --parallel-mode --concurrency=multiprocessing tests_runner.py integ
-	python -m coverage combine
-	python -m coverage xml -o coverage.xml
-	python -m coverage html -d coverage
+	pytest $(PYTEST_ARGS) climada/
 
 .PHONY : ci-clean
 ci-clean :
 	rm -rf tests_xml
-	rm pylint.log
-
+	rm pylint.log coverage.xml
+	rm -r coverage

doc/guide/Guide_Continuous_Integration_and_Testing.ipynb

@@ -1,13 +1,15 @@
 {
  "cells": [
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "# Testing and Continuous Integration"
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -20,31 +22,43 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
     "## Testing CLIMADA\n",
     "\n",
+    "Executing the entire test suite requires you to install the additional requirements for testing.\n",
+    "See the [installation instructions](install.rst) for [developer dependencies](install-dev) for further information.\n",
+    "\n",
+    "\n",
     "### Installation Test\n",
     "\n",
-    "From the installation directory run\\\n",
-    "`make install_test`\\\n",
+    "From the installation directory run\n",
+    "```\n",
+    "make install_test\n",
+    "```\n",
     "It lasts about 45 seconds. If it succeeds, CLIMADA is properly installed and ready to use.\n",
     "\n",
     "### Unit Tests\n",
     "\n",
-    "From the installation directory run\\\n",
-    "`make unit_test`\\\n",
+    "From the installation directory run\n",
+    "```\n",
+    "make unit_test\n",
+    "```\n",
     "It lasts about 5 minutes and runs unit tests for all modules.\n",
     "\n",
     "### Integration Tests\n",
     "\n",
-    "From the installation directory run\\\n",
-    "`make integ_test`\\\n",
-    "It lasts about 45 minutes and runs extensive integration tests, during which also data from external resources is read. An open internet connection is required for a successful test run."
+    "From the installation directory run\n",
+    "```\n",
+    "make integ_test\n",
+    "```\n",
+    "It lasts about 15 minutes and runs extensive integration tests, during which also data from external resources is read. An open internet connection is required for a successful test run."
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -59,6 +73,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -69,6 +84,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -85,6 +101,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -104,6 +121,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -117,6 +135,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -149,6 +168,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -172,6 +192,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -183,6 +204,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -198,6 +220,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -231,6 +254,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -240,6 +264,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -249,6 +274,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -259,6 +285,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -276,6 +303,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -355,6 +383,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [
@@ -372,6 +401,7 @@
    ]
   },
   {
+   "attachments": {},
    "cell_type": "markdown",
    "metadata": {},
    "source": [

doc/guide/install.rst

@@ -148,6 +148,8 @@ For advanced Python users or developers of CLIMADA, we recommed cloning the CLIM
    If this test passes, great!
    You are good to go.
 
+.. _install-dev:
+
 Install Developer Dependencies (Optional)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -177,6 +179,15 @@ The CLIMADA Python package defines the following `extras <https://peps.python.or
    * - ``dev``
      - combination of ``doc`` and ``test``
 
+For executing the pre-defined test scripts in exactly the same way as they are executed by the automated CI pipeline, you will need ``make`` to be installed.
+On macOS and on Linux it is pre-installed. On Windows, it can easily be installed with conda:
+
+.. code-block:: shell
+
+   conda install -n climada_env make
+
+Instructions for running the test scripts can be found in the :doc:`Testing and CI Guide <Guide_Continuous_Integration_and_Testing>`.
+
 Install CLIMADA Petals (Optional)
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 

script/jenkins/branches/Jenkinsfile

@@ -23,11 +23,10 @@ pipeline {
             sh '''#!/bin/bash
             export PATH=$PATH:$CONDAPATH
             source activate climada_env
+            python -m pip install pytest pytest-cov pytest-subtests
             rm -rf tests_xml/
             rm -rf coverage/
-            python -m coverage run tests_runner.py unit
-            python -m coverage xml -o coverage.xml
-            python -m coverage html -d coverage'''
+            make unit_test'''
           }
         }
 
@@ -41,4 +40,4 @@ pipeline {
       cobertura coberturaReportFile: 'coverage.xml'
     }
   }
-}
+}

script/jenkins/ci_night/Jenkinsfile

@@ -21,11 +21,7 @@ pipeline {
         source activate climada_env
         rm -rf tests_xml/
         rm -rf coverage/
-        python -m coverage run --parallel-mode --concurrency=multiprocessing tests_runner.py unit
-        python -m coverage run --parallel-mode --concurrency=multiprocessing tests_runner.py integ
-        python -m coverage combine
-        python -m coverage xml -o coverage.xml
-        python -m coverage html -d coverage'''
+        make test'''
       }
     }
   }
@@ -36,4 +32,4 @@ pipeline {
       cobertura coberturaReportFile: 'coverage.xml', enableNewApi: false
     }
   }
-}
+}

setup.py

@@ -22,10 +22,12 @@
 
 # Requirements for testing
 DEPS_TEST = [
-    "coverage>=7.2",
     "ipython",
     "mccabe>=0.6",
     "pylint==2.7.1",
+    "pytest",
+    "pytest-cov",
+    "pytest-subtests",
 ]
 
 setup(