Move long-runtime unit tests to integration tests (#709)

Moved tests:
* climada/util/test/test_lines_polys_handler/test_calc_geom_impact_polys (~ 15.5 sec)
* climada/hazard/test/test_storm_europe/test_icon_read (~ 4.5 sec)
* climada/hazard/test/test_storm_europe/test_from_footprints (~ 6 sec)
* climada/hazard/test/test_ibtracks_with_basin (~6 sec)
* climada/hazard/test/test_cutoff_tracks (~5 sec)
* climada/hazard/test/test_base/test_write_read_pass  (4.5 sec)
* climada/hazard/test/test_base/test_raster_to_vector_pass (3.5 sec)
* climada/hazard/test/test_base/test_reproject_raster_pass (3.6 sec)

---------

Co-authored-by: Lukas Riedel <[email protected]>

Author: NicolasColombi

CHANGELOG.md

@@ -30,6 +30,7 @@ Removed:
 - Added test for non-default impact function id in the `lines_polys_handler` [#676](https://github.com/CLIMADA-project/climada_python/pull/676)
 - The sigmoid and step impact functions now require the user to define the hazard type. [#675](https://github.com/CLIMADA-project/climada_python/pull/675)
 - Improved error messages produced by `ImpactCalc.impact()` in case hazard type is not found in exposures/impf_set [#691](https://github.com/CLIMADA-project/climada_python/pull/691)
+- Tests with long runtime were moved to integration tests in `climada/test` [#709](https://github.com/CLIMADA-project/climada_python/pull/709)
 
 ### Fixed
 

climada/hazard/test/test_base.py

@@ -1213,46 +1213,6 @@ def test_hazard_pass(self):
 class TestHDF5(unittest.TestCase):
     """Test reader functionality of the ExposuresExcel class"""
 
-    def test_write_read_pass(self):
-        """Read a hazard mat file correctly."""
-        file_name = str(DATA_DIR.joinpath('test_haz.h5'))
-
-        # Read demo matlab file
-        hazard = Hazard.from_mat(HAZ_TEST_MAT)
-        hazard.event_name = list(map(str, hazard.event_name))
-        for todense_flag in [False, True]:
-            if todense_flag:
-                hazard.write_hdf5(file_name, todense=todense_flag)
-            else:
-                hazard.write_hdf5(file_name)
-
-            haz_read = Hazard.from_hdf5(file_name)
-
-            self.assertEqual(str(hazard.tag.file_name), haz_read.tag.file_name)
-            self.assertIsInstance(haz_read.tag.file_name, str)
-            self.assertEqual(hazard.tag.haz_type, haz_read.tag.haz_type)
-            self.assertIsInstance(haz_read.tag.haz_type, str)
-            self.assertEqual(hazard.tag.description, haz_read.tag.description)
-            self.assertIsInstance(haz_read.tag.description, str)
-            self.assertEqual(hazard.units, haz_read.units)
-            self.assertIsInstance(haz_read.units, str)
-            self.assertTrue(np.array_equal(hazard.centroids.coord, haz_read.centroids.coord))
-            self.assertTrue(u_coord.equal_crs(hazard.centroids.crs, haz_read.centroids.crs))
-            self.assertTrue(np.array_equal(hazard.event_id, haz_read.event_id))
-            self.assertTrue(np.array_equal(hazard.frequency, haz_read.frequency))
-            self.assertEqual(hazard.frequency_unit, haz_read.frequency_unit)
-            self.assertIsInstance(haz_read.frequency_unit, str)
-            self.assertTrue(np.array_equal(hazard.event_name, haz_read.event_name))
-            self.assertIsInstance(haz_read.event_name, list)
-            self.assertIsInstance(haz_read.event_name[0], str)
-            self.assertTrue(np.array_equal(hazard.date, haz_read.date))
-            self.assertTrue(np.array_equal(hazard.orig, haz_read.orig))
-            self.assertTrue(np.array_equal(hazard.intensity.toarray(),
-                                           haz_read.intensity.toarray()))
-            self.assertIsInstance(haz_read.intensity, sparse.csr_matrix)
-            self.assertTrue(np.array_equal(hazard.fraction.toarray(), haz_read.fraction.toarray()))
-            self.assertIsInstance(haz_read.fraction, sparse.csr_matrix)
-
     def test_write_read_unsupported_type(self):
         """Check if the write command correctly handles unsupported types"""
         file_name = str(DATA_DIR.joinpath('test_unsupported.h5'))
@@ -1280,52 +1240,6 @@ class CustomID:
 class TestCentroids(unittest.TestCase):
     """Test return period statistics"""
 
-    def test_reproject_raster_pass(self):
-        """Test reproject_raster reference."""
-        haz_fl = Hazard.from_raster([HAZ_DEMO_FL])
-        haz_fl.check()
-
-        haz_fl.reproject_raster(dst_crs='epsg:2202')
-
-        self.assertEqual(haz_fl.intensity.shape, (1, 1046408))
-        self.assertIsInstance(haz_fl.intensity, sparse.csr_matrix)
-        self.assertIsInstance(haz_fl.fraction, sparse.csr_matrix)
-        self.assertEqual(haz_fl.fraction.shape, (1, 1046408))
-        self.assertTrue(u_coord.equal_crs(haz_fl.centroids.meta['crs'], 'epsg:2202'))
-        self.assertEqual(haz_fl.centroids.meta['width'], 968)
-        self.assertEqual(haz_fl.centroids.meta['height'], 1081)
-        self.assertEqual(haz_fl.fraction.min(), 0)
-        self.assertEqual(haz_fl.fraction.max(), 1)
-        self.assertEqual(haz_fl.intensity.min(), -9999)
-        self.assertTrue(haz_fl.intensity.max() < 4.7)
-
-    def test_raster_to_vector_pass(self):
-        """Test raster_to_vector method"""
-        haz_fl = Hazard.from_raster([HAZ_DEMO_FL], haz_type='FL')
-        haz_fl.check()
-        meta_orig = haz_fl.centroids.meta
-        inten_orig = haz_fl.intensity
-        fract_orig = haz_fl.fraction
-
-        haz_fl.raster_to_vector()
-
-        self.assertEqual(haz_fl.centroids.meta, dict())
-        self.assertAlmostEqual(haz_fl.centroids.lat.min(),
-                               meta_orig['transform'][5]
-                               + meta_orig['height'] * meta_orig['transform'][4]
-                               - meta_orig['transform'][4] / 2)
-        self.assertAlmostEqual(haz_fl.centroids.lat.max(),
-                               meta_orig['transform'][5] + meta_orig['transform'][4] / 2)
-        self.assertAlmostEqual(haz_fl.centroids.lon.max(),
-                               meta_orig['transform'][2]
-                               + meta_orig['width'] * meta_orig['transform'][0]
-                               - meta_orig['transform'][0] / 2)
-        self.assertAlmostEqual(haz_fl.centroids.lon.min(),
-                               meta_orig['transform'][2] + meta_orig['transform'][0] / 2)
-        self.assertTrue(u_coord.equal_crs(haz_fl.centroids.crs, meta_orig['crs']))
-        self.assertTrue(np.allclose(haz_fl.intensity.data, inten_orig.data))
-        self.assertTrue(np.allclose(haz_fl.fraction.data, fract_orig.data))
-
     def test_reproject_vector_pass(self):
         """Test reproject_vector"""
         haz_fl = Hazard('FL',

climada/hazard/test/test_storm_europe.py

@@ -44,55 +44,6 @@ def test_centroids_from_nc(self):
         self.assertTrue(isinstance(cent, Centroids))
         self.assertEqual(cent.size, 9944)
 
-    def test_from_footprints(self):
-        """Test from_footprints constructor, using one small test files"""
-        def _test_first(haz):
-            """Test the expected first entry of the hazard"""
-            self.assertEqual(haz.tag.haz_type, 'WS')
-            self.assertEqual(haz.units, 'm/s')
-            self.assertEqual(haz.event_id.size, 1)
-            self.assertEqual(haz.date.size, 1)
-            self.assertEqual(dt.datetime.fromordinal(haz.date[0]).year, 1999)
-            self.assertEqual(dt.datetime.fromordinal(haz.date[0]).month, 12)
-            self.assertEqual(dt.datetime.fromordinal(haz.date[0]).day, 26)
-            self.assertEqual(haz.event_id[0], 1)
-            self.assertEqual(haz.event_name[0], 'Lothar')
-            self.assertIsInstance(haz.intensity,
-                                  sparse.csr.csr_matrix)
-            self.assertIsInstance(haz.fraction,
-                                  sparse.csr.csr_matrix)
-            self.assertEqual(haz.intensity.shape, (1, 9944))
-            self.assertEqual(haz.fraction.shape, (1, 9944))
-            self.assertEqual(haz.frequency[0], 1.0)
-
-        # Load first entry
-        storms = StormEurope.from_footprints(
-            WS_DEMO_NC[0], description='test_description')
-        _test_first(storms)
-
-        # Omit the second file, should be the same result
-        storms = StormEurope.from_footprints(WS_DEMO_NC, files_omit=str(WS_DEMO_NC[1]))
-        _test_first(storms)
-
-        # Now load both
-        storms = StormEurope.from_footprints(WS_DEMO_NC, description='test_description')
-
-        self.assertEqual(storms.tag.haz_type, 'WS')
-        self.assertEqual(storms.units, 'm/s')
-        self.assertEqual(storms.event_id.size, 2)
-        self.assertEqual(storms.date.size, 2)
-        self.assertEqual(dt.datetime.fromordinal(storms.date[0]).year, 1999)
-        self.assertEqual(dt.datetime.fromordinal(storms.date[0]).month, 12)
-        self.assertEqual(dt.datetime.fromordinal(storms.date[0]).day, 26)
-        self.assertEqual(storms.event_id[0], 1)
-        self.assertEqual(storms.event_name[0], 'Lothar')
-        self.assertIsInstance(storms.intensity,
-                              sparse.csr.csr_matrix)
-        self.assertIsInstance(storms.fraction,
-                              sparse.csr.csr_matrix)
-        self.assertEqual(storms.intensity.shape, (2, 9944))
-        self.assertEqual(storms.fraction.shape, (2, 9944))
-
     def test_read_with_ref(self):
         """Test from_footprints while passing in a reference raster."""
         storms = StormEurope.from_footprints(WS_DEMO_NC, ref_raster=WS_DEMO_NC[1])
@@ -197,47 +148,6 @@ def test_cosmoe_read(self):
         self.assertAlmostEqual(haz.intensity.max(), 36.426735,places=3)
         self.assertEqual(haz.fraction.shape, (21, 25))
 
-    def test_icon_read(self):
-        """test reading from icon grib"""
-        # for this test the forecast file is supposed to be already downloaded from the dwd
-        # another download would fail because the files are available for 24h only
-        # instead, we download it as a test dataset through the climada data api
-        apiclient = Client()
-        ds = apiclient.get_dataset_info(name='test_storm_europe_icon_2021012800', status='test_dataset')
-        dsdir, _ = apiclient.download_dataset(ds)
-        haz = StormEurope.from_icon_grib(
-            dt.datetime(2021, 1, 28),
-            dt.datetime(2021, 1, 28),
-            model_name='test',
-            grib_dir=dsdir,
-            delete_raw_data=False)
-        self.assertEqual(haz.tag.haz_type, 'WS')
-        self.assertEqual(haz.units, 'm/s')
-        self.assertEqual(haz.event_id.size, 40)
-        self.assertEqual(haz.date.size, 40)
-        self.assertEqual(dt.datetime.fromordinal(haz.date[0]).year, 2021)
-        self.assertEqual(dt.datetime.fromordinal(haz.date[0]).month, 1)
-        self.assertEqual(dt.datetime.fromordinal(haz.date[0]).day, 28)
-        self.assertEqual(haz.event_id[-1], 40)
-        self.assertEqual(haz.event_name[-1], '2021-01-28_ens40')
-        self.assertIsInstance(haz.intensity,
-                              sparse.csr.csr_matrix)
-        self.assertIsInstance(haz.fraction,
-                              sparse.csr.csr_matrix)
-        self.assertEqual(haz.intensity.shape, (40, 49))
-        self.assertAlmostEqual(haz.intensity.max(), 17.276321,places=3)
-        self.assertEqual(haz.fraction.shape, (40, 49))
-        with self.assertLogs('climada.hazard.storm_europe', level='WARNING') as cm:
-            with self.assertRaises(ValueError):
-                haz = StormEurope.from_icon_grib(
-                    dt.datetime(2021, 1, 28, 6),
-                    dt.datetime(2021, 1, 28),
-                    model_name='test',
-                    grib_dir=CONFIG.hazard.test_data.str(),
-                    delete_raw_data=False)
-        self.assertEqual(len(cm.output), 1)
-        self.assertIn('event definition is inaccuratly implemented', cm.output[0])
-
     def test_generate_forecast(self):
         """ testing generating a forecast """
         hazard, haz_model, run_datetime, event_date = generate_WS_forecast_hazard(

climada/hazard/test/test_tc_tracks.py

@@ -242,34 +242,6 @@ def test_ibtracs_correct_pass(self):
         self.assertAlmostEqual(tc_try.data[0].central_pressure.values[5], 1008, places=0)
         self.assertAlmostEqual(tc_try.data[0].central_pressure.values[-1], 1012, places=0)
 
-    def test_ibtracs_with_basin(self):
-        """Filter TCs by (genesis) basin."""
-        # South Atlantic (not usually a TC location at all)
-        tc_track = tc.TCTracks.from_ibtracs_netcdf(basin="SA")
-        self.assertEqual(tc_track.size, 3)
-
-        # the basin is not necessarily the genesis basin
-        tc_track = tc.TCTracks.from_ibtracs_netcdf(
-            year_range=(1995, 1995), basin="SP", estimate_missing=True)
-        self.assertEqual(tc_track.size, 6)
-        self.assertEqual(tc_track.data[0].basin[0], 'SP')
-        self.assertEqual(tc_track.data[5].basin[0], 'SI')
-
-        # genesis in NI
-        tc_track = tc.TCTracks.from_ibtracs_netcdf(
-            year_range=(1994, 1994), genesis_basin="NI", estimate_missing=True)
-        self.assertEqual(tc_track.size, 5)
-        for tr in tc_track.data:
-            self.assertEqual(tr.basin[0], "NI")
-
-        # genesis in EP, but crosses WP at some point
-        tc_track = tc.TCTracks.from_ibtracs_netcdf(
-            year_range=(2002, 2003), basin="WP", genesis_basin="EP")
-        self.assertEqual(tc_track.size, 3)
-        for tr in tc_track.data:
-            self.assertEqual(tr.basin[0], "EP")
-            self.assertIn("WP", tr.basin)
-
     def test_ibtracs_discard_single_points(self):
         """Check discard_single_points option"""
         passed = False

climada/hazard/test/test_tc_tracks_synth.py

@@ -598,14 +598,6 @@ def test_random_walk_single_point(self):
                 break
         self.assertTrue(found)
 
-    def test_cutoff_tracks(self):
-        tc_track = tc.TCTracks.from_ibtracs_netcdf(storm_id='1986226N30276')
-        tc_track.equal_timestep()
-        with self.assertLogs('climada.hazard.tc_tracks_synth', level='DEBUG') as cm:
-            tc_track.calc_perturbed_trajectories(nb_synth_tracks=10)
-        self.assertIn('The following generated synthetic tracks moved beyond '
-                      'the range of [-70, 70] degrees latitude', cm.output[1])
-
 # Execute Tests
 if __name__ == "__main__":
     TESTS = unittest.TestLoader().loadTestsFromTestCase(TestDecay)