Feature/coverage nightlight (#659)

* Add coverage test for nightlight

1) Add tests for functions:
       - load_nasa_nl_shape_single_tile
       - load_nightlight_noaa
       - untar_noaa_stable_nighlight
2) Add raises message if wrong layer is selected in function:
     load_nasa_nl_shape_single_tile
3 ) Add logger message in function: load_nasa_nl_shape_single_tile
4)  Correct test unzip_tif_to_py
5)  Apply uniform formatting across all tests of nightlight.py module

* Fix pylint errors

1) line too long
2 )use %s for logging instead of f-strings

* Complete test for gpw_population function

Add small test for function load_gpw_pop_shape() situated in:
climada/entity/exposures/litpop/gpw_population.py

* Add test for litpop functions

Add test for functions:

1) from_nightlight_intensity()
2) from_population()

* Add nighlight test

Add test for function climada/entity/exposures/litpop/nightlight.py
get_required_nl_files()

remove unnecessary brackets from nightlight.py

* Add test for check_nl_local_file_exists

* correct check_nl_local_file_exists

2 BM_FILES in local data folder downloaded with climada
but 5 BM_FILES on the system dir of jenkins

* correct test

Author: NicolasColombi

climada/entity/exposures/litpop/nightlight.py

@@ -200,15 +200,15 @@ def get_required_nl_files(bounds):
 
     # Now latitude. The height of all tiles is the same as the height.
     # Note that for this analysis returns an index which follows from North to South oritentation.
-    first_tile_lat = min(np.floor(-(min_lat - (90)) / tile_width), 1)
+    first_tile_lat = min(np.floor(-(min_lat - 90) / tile_width), 1)
     last_tile_lat = min(np.floor(-(max_lat - 90) / tile_width), 1)
 
     for i_lon in range(0, int(len(req_files) / 2)):
         if first_tile_lon <= i_lon <= last_tile_lon:
             if first_tile_lat == 0 or last_tile_lat == 0:
-                req_files[((i_lon)) * 2] = 1
+                req_files[(i_lon) * 2] = 1
             if first_tile_lat == 1 or last_tile_lat == 1:
-                req_files[((i_lon)) * 2 + 1] = 1
+                req_files[(i_lon) * 2 + 1] = 1
         else:
             continue
     return req_files
@@ -355,6 +355,10 @@ def load_nasa_nl_shape_single_tile(geometry, path, layer=0):
     with rasterio.open(path, 'r') as src:
         # read cropped data from  source file (src) to np.ndarray:
         out_image, transform = rasterio.mask.mask(src, [geometry], crop=True)
+        LOGGER.debug('Read cropped %s as np.ndarray.', path.name)
+        if out_image.shape[0] < layer:
+            raise IndexError(f"{path.name} has only {out_image.shape[0]} layers,"
+                             f" layer {layer} can't be accessed.")
         meta = src.meta
         meta.update({"driver": "GTiff",
                     "height": out_image.shape[1],

climada/entity/exposures/test/test_nightlight.py

@@ -23,6 +23,7 @@
 
 from climada.entity.exposures.litpop import nightlight
 from climada.util.constants import SYSTEM_DIR
+from pathlib import Path
 
 BM_FILENAMES = nightlight.BM_FILENAMES
 
@@ -79,6 +80,75 @@ def test_download_nightlight_files(self):
         # The same length but not the correct length
         self.assertRaises(ValueError, nightlight.download_nl_files, (1, 0, 1), (1, 1, 1))
 
+    def test_get_required_nl_files(self):
+        """ get_required_nl_files return a boolean matrix of 0 and 1
+            indicating which tile of NASA nighlight files are needed giving
+            a bounding box. This test check a few configuration of tiles
+            and check that a value error is raised if the bounding box are 
+            incorrect """
+
+        # incorrect bounds: bounds size =!  4, min lon > max lon, min lat > min lat
+        BOUNDS = [(20, 30, 40), 
+                  (120, -20, 110, 30), 
+                  (-120, 50, 130, 10)]
+        # correct bounds
+        bounds_c1 = (-120, -20, 0, 40)
+        bounds_c2 = (-70, -20, 10, 40)
+        bounds_c3 = (160, 10, 180, 40)
+
+        for bounds in BOUNDS:
+            with self.assertRaises(ValueError) as cm:
+    
+                nightlight.get_required_nl_files(bounds = bounds)
+
+                self.assertEqual('Invalid bounds supplied. `bounds` must be tuple'
+                                ' with (min_lon, min_lat, max_lon, max_lat).',
+                                str(cm.exception))
+        
+        # test first correct bounds configurations
+        req_files = nightlight.get_required_nl_files(bounds = bounds_c1)
+        bool = np.array_equal(np.array([1, 1, 1, 1, 1, 1, 0, 0]), req_files)
+        self.assertTrue(bool)
+        # second correct configuration
+        req_files = nightlight.get_required_nl_files(bounds = bounds_c2)
+        bool = np.array_equal(np.array([0, 0, 1, 1, 1, 1, 0, 0]), req_files)
+        self.assertTrue(bool)
+        # third correct configuration
+        req_files = nightlight.get_required_nl_files(bounds = bounds_c3)
+        bool = np.array_equal(np.array([0, 0, 0, 0, 0, 0, 1, 0]), req_files)
+        self.assertTrue(bool)
+    
+    def test_check_nl_local_file_exists(self):
+        """ Test that an array with the correct number of already existing files
+            is produced, the LOGGER messages logged and the ValueError raised. """
+
+        # check logger messages by giving a to short req_file
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level = 'WARNING') as cm:
+            nightlight.check_nl_local_file_exists(required_files = np.array([0, 0, 1, 1]))
+        self.assertIn('The parameter \'required_files\' was too short and '
+                       'is ignored.', cm.output[0])
+             
+        # check logger message: not all files are available 
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level = 'DEBUG') as cm:
+            nightlight.check_nl_local_file_exists()
+        self.assertIn('Not all satellite files available. '
+                     f'Found 5 out of 8 required files in {Path(SYSTEM_DIR)}', cm.output[0])
+            
+        # check logger message: no files found in checkpath 
+        with self.assertLogs('climada.entity.exposures.litpop.nightlight', level = 'INFO') as cm:
+            # using a random path where no files are stored
+            nightlight.check_nl_local_file_exists(check_path = Path('climada/entity/exposures'))
+        self.assertIn('No satellite files found locally in climada/entity/exposures', cm.output[0])
+      
+        # test raises with wrong path
+        with self.assertRaises(ValueError) as cm:
+            nightlight.check_nl_local_file_exists(check_path = '/random/wrong/path')
+        self.assertEqual('The given path does not exist: /random/wrong/path', str(cm.exception))
+
+        # test that files_exist is correct 
+        files_exist = nightlight.check_nl_local_file_exists()
+        self.assertEqual(int(sum(files_exist)), 5)
+
 # Execute Tests
 if __name__ == "__main__":
     TESTS = unittest.TestLoader().loadTestsFromTestCase(TestNightLight)

climada/test/test_litpop_integr.py

@@ -29,6 +29,13 @@
 from climada.util.constants import SYSTEM_DIR
 from climada import CONFIG
 
+bounds = (8.41, 47.2, 8.70, 47.45) # (min_lon, max_lon, min_lat, max_lat)
+shape = Polygon([
+    (bounds[0], bounds[3]),
+    (bounds[2], bounds[3]),
+    (bounds[2], bounds[1]),
+    (bounds[0], bounds[1])
+    ])
 
 class TestLitPopExposure(unittest.TestCase):
     """Test LitPop exposure data model:"""
@@ -122,14 +129,7 @@ def test_switzerland300_admin1_pc2016_pass(self):
     def test_from_shape_zurich_pass(self):
         """test initiating LitPop for custom shape (square around Zurich City)
         Distributing an imaginary total value of 1000 USD"""
-        bounds = (8.41, 47.2, 8.70, 47.45) # (min_lon, max_lon, min_lat, max_lat)
         total_value=1000
-        shape = Polygon([
-            (bounds[0], bounds[3]),
-            (bounds[2], bounds[3]),
-            (bounds[2], bounds[1]),
-            (bounds[0], bounds[1])
-            ])
         ent = lp.LitPop.from_shape(shape, total_value, res_arcsec=30, reference_year=2016)
         self.assertEqual(ent.gdf.value.sum(), 1000.0)
         self.assertEqual(ent.gdf.value.min(), 0.0)
@@ -144,13 +144,7 @@ def test_from_shape_zurich_pass(self):
     def test_from_shape_and_countries_zurich_pass(self):
         """test initiating LitPop for custom shape (square around Zurich City)
         with from_shape_and_countries()"""
-        bounds = (8.41, 47.2, 8.70, 47.45) # (min_lon, max_lon, min_lat, max_lat)
-        shape = Polygon([
-            (bounds[0], bounds[3]),
-            (bounds[2], bounds[3]),
-            (bounds[2], bounds[1]),
-            (bounds[0], bounds[1])
-            ])
+        
         ent = lp.LitPop.from_shape_and_countries(
             shape, 'Switzerland', res_arcsec=30, reference_year=2016)
         self.assertEqual(ent.gdf.value.min(), 0.0)
@@ -186,6 +180,46 @@ def test_Liechtenstein_30_pop_pass(self):
         self.assertAlmostEqual(ent.gdf.latitude.max(), 47.2541666666666)
         self.assertAlmostEqual(ent.meta['transform'][0], 30/3600)
 
+    def test_from_nightlight_intensity(self):
+        """ Test raises, logger and if methods from_countries and from_shape are 
+            are used."""
+
+        with self.assertRaises(ValueError) as cm:
+            lp.LitPop.from_nightlight_intensity()
+        self.assertEqual('Either `countries` or `shape` required. Aborting.', str(cm.exception))
+
+        with self.assertRaises(ValueError) as cm:
+            lp.LitPop.from_nightlight_intensity(countries = 'Liechtenstein', shape = shape)
+        self.assertEqual('Not allowed to set both `countries` and `shape`. Aborting.', str(cm.exception))
+        
+        exp = lp.LitPop.from_nightlight_intensity(countries = 'Liechtenstein') 
+        self.assertEqual(exp.fin_mode, 'none')
+
+        exp = lp.LitPop.from_nightlight_intensity(shape = shape) 
+        self.assertEqual(exp.value_unit, '')
+        
+        with self.assertLogs('climada.entity.exposures.litpop.litpop', level = 'WARNING') as cm:
+            lp.LitPop.from_nightlight_intensity(shape = shape)
+        self.assertIn('Note: set_nightlight_intensity sets values to raw nightlight intensity,', cm.output[0])
+
+    def test_from_population(self):
+        """ Test raises, logger and if methods from_countries and from_shape are 
+            are used."""
+
+        with self.assertRaises(ValueError) as cm:
+            lp.LitPop.from_population()
+        self.assertEqual('Either `countries` or `shape` required. Aborting.', str(cm.exception))
+
+        exp = lp.LitPop.from_population(countries = 'Liechtenstein') 
+        self.assertEqual(exp.fin_mode, 'pop')
+
+        exp = lp.LitPop.from_population(shape = shape) 
+        self.assertEqual(exp.value_unit, 'people')
+
+        with self.assertRaises(ValueError) as cm:
+            lp.LitPop.from_population(countries = 'Liechtenstein', shape = shape)
+        self.assertEqual('Not allowed to set both `countries` and `shape`. Aborting.', str(cm.exception))        
+
 class TestAdmin1(unittest.TestCase):
     """Test the admin1 functionalities within the LitPop module"""
 
@@ -265,20 +299,17 @@ def test_get_gpw_file_path_pass(self):
     def test_load_gpw_pop_shape_pass(self):
         """test method gpw_population.load_gpw_pop_shape"""
         gpw_version = CONFIG.exposures.litpop.gpw_population.gpw_version.int()
-        bounds = (8.41, 47.2, 8.70, 47.45) # (min_lon, max_lon, min_lat, max_lat)
-        shape = Polygon([
-            (bounds[0], bounds[3]),
-            (bounds[2], bounds[3]),
-            (bounds[2], bounds[1]),
-            (bounds[0], bounds[1])
-            ])
         try:
             data, meta, glb_transform = \
                 gpw_population.load_gpw_pop_shape(shape, 2020, gpw_version, verbose=False)
             self.assertEqual(data.shape, (31, 36))
             self.assertAlmostEqual(meta['transform'][0], 0.00833333333333333)
             self.assertAlmostEqual(meta['transform'][0], glb_transform[0])
-
+            self.assertEqual(meta['driver'], 'GTiff')
+            self.assertEqual(meta['height'], data.shape[0])
+            self.assertEqual(meta['width'], data.shape[1])
+            self.assertIsInstance(data, np.ndarray)
+            self.assertEqual(len(data.shape), 2) 
         except FileExistsError as err:
             self.assertIn('lease download', err.args[0])
             self.skipTest('GPW input data for GPW v4.%i not found.' %(gpw_version))