Merge branch 'develop' into calibrate-impact-functions

Author: peanutfun

.jenkins_install_env.sh

@@ -4,5 +4,6 @@ mamba remove --name climada_env --all
 mamba env create -f requirements/env_climada.yml --name climada_env
 
 source activate climada_env
+python -m pip install -e "./[test]"
 make install_test
 conda deactivate

.pylintrc

@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call,W0201
+disable=raw-checker-failed,bad-inline-option,locally-disabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,W0201
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -388,4 +388,4 @@ min-public-methods=2
 
 # Exceptions that will emit a warning when being caught. Defaults to
 # "Exception"
-overgeneral-exceptions=Exception
+overgeneral-exceptions=builtins.Exception

.readthedocs.yml

@@ -8,5 +8,12 @@ build:
 conda:
   environment: requirements/env_docs.yml
 
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - doc
+
 formats:
   - pdf

AUTHORS.md

@@ -28,3 +28,4 @@
 * Lukas Riedel
 * Raphael Portmann
 * Nicolas Colombi
+* Leonie Villiger

CHANGELOG.md

@@ -13,12 +13,17 @@ Added:
 Changed:
 
 Removed:
+- `nbsphinx` [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
+- `pandoc` [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
 
 ### Added
 
 - `Impact.impact_at_reg` method for aggregating impacts per country or custom region [#642](https://github.com/CLIMADA-project/climada_python/pull/642)
 - `Impact.match_centroids` convenience method for matching (hazard) centroids to impact objects [#602](https://github.com/CLIMADA-project/climada_python/pull/602)
 - `climada.util.coordinates.match_centroids` method for matching (hazard) centroids to GeoDataFrames [#602](https://github.com/CLIMADA-project/climada_python/pull/602)
+- 'Extra' requirements `doc`, `test`, and `dev` for Python package [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
+- Added method `Exposures.centroids_total_value` to replace the functionality of `Exposures.affected_total_value`. This method is temporary and deprecated. [#702](https://github.com/CLIMADA-project/climada_python/pull/702)
+
 
 ### Changed
 
@@ -29,15 +34,32 @@ Removed:
 - Modified the method to disaggregate lines in the `lines_polys_handler` utility module in order to better conserve the total length of all lines on average [#679](https://github.com/CLIMADA-project/climada_python/pull/679).
 - Added test for non-default impact function id in the `lines_polys_handler` [#676](https://github.com/CLIMADA-project/climada_python/pull/676)
 - The sigmoid and step impact functions now require the user to define the hazard type. [#675](https://github.com/CLIMADA-project/climada_python/pull/675)
+- Improved error messages produced by `ImpactCalc.impact()` in case hazard type is not found in exposures/impf_set [#691](https://github.com/CLIMADA-project/climada_python/pull/691)
+- Tests with long runtime were moved to integration tests in `climada/test` [#709](https://github.com/CLIMADA-project/climada_python/pull/709)
+- Use `myst-nb` for parsing Jupyter Notebooks for the documentation instead of `nbsphinx` [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
+- Installation guide now recommends installing CLIMADA directly via `conda install` [#714](https://github.com/CLIMADA-project/climada_python/pull/714)
+- `Exposures.affected_total_value` now takes a hazard intensity threshold as argument. Affected values are only those for which at least one event exceeds the threshold. (previously, all exposures points with an assigned centroid were considered affected). By default the centroids are reassigned. [#702](https://github.com/CLIMADA-project/climada_python/pull/702) [#730](https://github.com/CLIMADA-project/climada_python/pull/730)
+- Add option to pass region ID to `LitPop.from_shape` [#720](https://github.com/CLIMADA-project/climada_python/pull/720)
+- Slightly improved performance on `LitPop`-internal computations [#720](https://github.com/CLIMADA-project/climada_python/pull/720)
+- Users can opt-out of the climada specific logging definitions and freely configure logging to their will, by setting the config value `logging.managed` to `false`. [#724](https://github.com/CLIMADA-project/climada_python/pull/724)
+- Add option to read additional variables from IBTrACS when using `TCTracks.from_ibtracs_netcdf` [#728](https://github.com/CLIMADA-project/climada_python/pull/728)
 
 ### Fixed
 
 - `util.lines_polys_handler` solve polygon disaggregation issue in metre-based projection [#666](https://github.com/CLIMADA-project/climada_python/pull/666)
+- Problem with `pyproj.CRS` as `Impact` attribute, [#706](https://github.com/CLIMADA-project/climada_python/issues/706). Now CRS is always stored as `str` in WKT format.
 
 ### Deprecated
 
+- `Centroids.from_geodataframe` and `Centroids.from_pix_bounds` [#721](https://github.com/CLIMADA-project/climada_python/pull/721)
+- `Impact.tot_value`: Use `Exposures.affected_total_value` to compute the total value affected by a hazard intensity above a custom threshold [#702](https://github.com/CLIMADA-project/climada_python/pull/702)
+
 ### Removed
 
+- `Centroids.set_raster_from_pix_bounds` [#721](https://github.com/CLIMADA-project/climada_python/pull/721)
+
+- `requirements/env_developer.yml` environment specs. Use 'extra' requirements when installing the Python package instead [#712](https://github.com/CLIMADA-project/climada_python/pull/712)
+
 ## v3.3.2
 
 Release date: 2023-03-02

README.md

@@ -19,7 +19,13 @@ This is the Python (3.8+) version of CLIMADA - please see https://github.com/dav
 
 ## Getting started
 
-CLIMADA runs on Windows, macOS and Linux. It can be installed from sources or - in case of climada_python - directly with pip. See the [installation guide](https://climada-python.readthedocs.io/en/latest/guide/install.html) for instructions.
+CLIMADA runs on Windows, macOS and Linux.
+The released versions of the CLIMADA core can be installed directly through Anaconda:
+```shell
+conda install -c conda-forge climada
+```
+It is **highly recommended** to install CLIMADA into a **separate** Anaconda environment.
+See the [installation guide](https://climada-python.readthedocs.io/en/latest/guide/install.html) for further information.
 
 Follow the [tutorial](https://climada-python.readthedocs.io/en/latest/tutorial/1_main_climada.html) `climada_python-x.y.z/doc/tutorial/1_main_climada.ipynb` in a Jupyter Notebook to see what can be done with CLIMADA and how.
 

climada/conf/climada.conf

@@ -57,6 +57,9 @@
         }
     },
     "log_level": "WARNING",
+    "logging": {
+        "managed": true
+    },
     "max_matrix_size": 1000000000,
     "data_api": {
         "url": "https://climada.ethz.ch/data-api/v2/",

climada/engine/calibration_opt.py

@@ -31,7 +31,7 @@
 
 from climada.engine import ImpactCalc
 from climada.entity import ImpactFuncSet, ImpfTropCyclone, impact_funcs
-from climada.engine.impact_data import emdat_impact_yearlysum, emdat_impact_event
+from climada.engine.impact_data import emdat_impact_yearlysum  #, emdat_impact_event
 
 LOGGER = logging.getLogger(__name__)
 
@@ -261,7 +261,7 @@ def init_impact_data(hazard_type,
                                              reference_year=reference_year)
         else:
             raise ValueError('init_impact_data not yet implemented for yearly_impact = False.')
-            em_data = emdat_impact_event(source_file)
+            #em_data = emdat_impact_event(source_file)
     else:
         raise ValueError('init_impact_data not yet implemented for other impact_data_sources '
                          'than emdat.')

climada/engine/impact.py

@@ -41,6 +41,8 @@
 import xlsxwriter
 from tqdm import tqdm
 import h5py
+from pyproj import CRS as pyprojCRS
+from rasterio.crs import CRS as rasterioCRS  # pylint: disable=no-name-in-module
 
 from climada.entity import Exposures, Tag
 from climada.hazard import Tag as TagHaz
@@ -72,6 +74,8 @@ class Impact():
         ordinal 1 (ordinal format of datetime library)
     coord_exp : np.array
         exposures coordinates [lat, lon] (in degrees)
+    crs : str
+        WKT string of the impact's crs
     eai_exp : np.array
         expected impact for each exposure within a period of 1/frequency_unit
     at_event : np.array
@@ -80,8 +84,6 @@ class Impact():
         frequency of event
     frequency_unit : str
         frequency unit used (given by hazard), default is '1/year'
-    tot_value : float
-        total exposure value affected
     aai_agg : float
         average impact within a period of 1/frequency_unit (aggregated)
     unit : str
@@ -126,7 +128,8 @@ def __init__(self,
         coord_exp : np.array, optional
             exposures coordinates [lat, lon] (in degrees)
         crs : Any, optional
-            coordinate reference system
+            Coordinate reference system. CRS instances from ``pyproj`` and ``rasterio``
+            will be transformed into WKT. Other types are not handled explicitly.
         eai_exp : np.array, optional
             expected impact for each exposure within a period of 1/frequency_unit
         at_event : np.array, optional
@@ -149,12 +152,12 @@ def __init__(self,
         self.event_name = [] if event_name is None else event_name
         self.date = np.array([], int) if date is None else date
         self.coord_exp = np.array([], float) if coord_exp is None else coord_exp
-        self.crs = crs
+        self.crs = crs.to_wkt() if isinstance(crs, (pyprojCRS, rasterioCRS)) else crs
         self.eai_exp = np.array([], float) if eai_exp is None else eai_exp
         self.at_event = np.array([], float) if at_event is None else at_event
         self.frequency = np.array([],float) if frequency is None else frequency
         self.frequency_unit = frequency_unit
-        self.tot_value = tot_value
+        self._tot_value = tot_value
         self.aai_agg = aai_agg
         self.unit = unit
 
@@ -246,7 +249,7 @@ def from_eih(cls, exposures, impfset, hazard,
                                  axis=1),
             crs = exposures.crs,
             unit = exposures.value_unit,
-            tot_value = exposures.affected_total_value(hazard),
+            tot_value = exposures.centroids_total_value(hazard),
             eai_exp = eai_exp,
             at_event = at_event,
             aai_agg = aai_agg,
@@ -257,6 +260,29 @@ def from_eih(cls, exposures, impfset, hazard,
                    }
             )
 
+    @property
+    def tot_value(self):
+        """Return the total exposure value close to a hazard
+
+        .. deprecated:: 3.3
+           Use :py:meth:`climada.entity.exposures.base.Exposures.affected_total_value`
+           instead.
+        """
+        LOGGER.warning("The Impact.tot_value attribute is deprecated."
+                       "Use Exposures.affected_total_value to calculate the affected "
+                       "total exposure value based on a specific hazard intensity "
+                       "threshold")
+        return self._tot_value
+
+    @tot_value.setter
+    def tot_value(self, value):
+        """Set the total exposure value close to a hazard"""
+        LOGGER.warning("The Impact.tot_value attribute is deprecated."
+                       "Use Exposures.affected_total_value to calculate the affected "
+                       "total exposure value based on a specific hazard intensity "
+                       "threshold")
+        self._tot_value = value
+
     def transfer_risk(self, attachment, cover):
         """Compute the risk transfer for the full portfolio. This is the risk
         of the full portfolio summed over all events. For each
@@ -860,7 +886,7 @@ def write_csv(self, file_name):
                          [self.tag['haz'].description]],
                         [[self.tag['exp'].file_name], [self.tag['exp'].description]],
                         [[self.tag['impf_set'].file_name], [self.tag['impf_set'].description]],
-                        [self.unit], [self.tot_value], [self.aai_agg],
+                        [self.unit], [self._tot_value], [self.aai_agg],
                         self.event_id, self.event_name, self.date,
                         self.frequency, [self.frequency_unit], self.at_event,
                         self.eai_exp, self.coord_exp[:, 0], self.coord_exp[:, 1],
@@ -901,7 +927,7 @@ def write_col(i_col, imp_ws, xls_data):
         data = [str(self.tag['impf_set'].file_name), str(self.tag['impf_set'].description)]
         write_col(2, imp_ws, data)
         write_col(3, imp_ws, [self.unit])
-        write_col(4, imp_ws, [self.tot_value])
+        write_col(4, imp_ws, [self._tot_value])
         write_col(5, imp_ws, [self.aai_agg])
         write_col(6, imp_ws, self.event_id)
         write_col(7, imp_ws, self.event_name)
@@ -1030,8 +1056,9 @@ def write_csr(group, name, value):
         with h5py.File(file_path, "w") as file:
 
             # Now write all attributes
+            # NOTE: Remove leading underscore to write '_tot_value' as regular attribute
             for name, value in self.__dict__.items():
-                write(file, name, value)
+                write(file, name.lstrip("_"), value)
 
     def write_sparse_csr(self, file_name):
         """Write imp_mat matrix in numpy's npz format."""

climada/engine/impact_calc.py

@@ -112,6 +112,22 @@ def impact(self, save_mat=True, assign_centroids=True,
         apply_deductible_to_mat : apply deductible to impact matrix
         apply_cover_to_mat : apply cover to impact matrix
         """
+        # check for compability of exposures and hazard type
+        if all(name not in self.exposures.gdf.columns for
+               name in ['if_', f'if_{self.hazard.haz_type}',
+                        'impf_', f'impf_{self.hazard.haz_type}']):
+            raise AttributeError(
+                "Impact calculation not possible. No impact functions found "
+                f"for hazard type {self.hazard.haz_type} in exposures."
+                )
+
+        # check for compability of impact function and hazard type
+        if not self.impfset.get_func(haz_type=self.hazard.haz_type):
+            raise AttributeError(
+                "Impact calculation not possible. No impact functions found "
+                f"for hazard type {self.hazard.haz_type} in impf_set."
+                )
+
         impf_col = self.exposures.get_impf_column(self.hazard.haz_type)
         exp_gdf = self.minimal_exp_gdf(impf_col, assign_centroids, ignore_cover, ignore_deductible)
         if exp_gdf.size == 0: