Merge branch 'reduce_etas' of https://github.com/ClimateImpactLab/dscim into reduce_etas

Author: JMGilbert

requirements.txt

@@ -1,17 +1,18 @@
 # This is used to build the CI test environment and so Github can track dependencies.
-xarray==2024.3.0
+xarray==2025.1.2
 pandas==2.2.3
 numpy==1.26.4
 matplotlib==3.10.1
 dask[array, distributed]==2025.2.0
 requests==2.32.3
 statsmodels==0.14.4
 pytest==8.3.5
-pytest-cov==6.0.0
+pytest-cov==6.2.1
 zarr==2.18.4
 ruff==0.8.6
 netcdf4==1.7.2
 h5netcdf==1.6.1
 impactlab-tools==0.6.0
 p_tqdm==1.4.2
 pyarrow==19.0.1
+numcodecs==0.15.1

src/dscim/preprocessing/input_damages.py

@@ -14,7 +14,6 @@
 from functools import partial
 from p_tqdm import p_map, p_umap
 from dscim.menu.simple_storage import EconVars
-from zarr.errors import GroupNotFoundError
 
 logger = logging.getLogger(__name__)
 
@@ -275,13 +274,21 @@ def concatenate_labor_damages(
     return concat_ds
 
 
-def calculate_labor_batch_damages(batch, ec, input_path, save_path):
+def calculate_labor_batch_damages(
+    batch,
+    ec,
+    input_path,
+    save_path,
+    variable="rebased",
+    file_prefix="uninteracted_main_model",
+):
     print(f"Processing batch={batch} damages in {os.getpid()}")
     concatenate_labor_damages(
         input_path=input_path,
         save_path=save_path,
         ec_cls=ec,
-        variable="rebased",
+        variable=variable,
+        file_prefix=file_prefix,
         val_type="wage-levels",
         format_file="zarr",
         query=f"exists==True&batch=='batch{batch}'",
@@ -293,6 +300,8 @@ def calculate_labor_damages(
     path_econ,
     input_path,
     save_path,
+    variable="rebased",
+    file_prefix="uninteracted_main_model",
 ):
     ec = EconVars(path_econ)
     # process in 3 rounds to limit memory usage
@@ -302,6 +311,8 @@ def calculate_labor_damages(
             input_path=input_path,
             save_path=save_path,
             ec=ec,
+            variable=variable,
+            file_prefix=file_prefix,
         )
         print("Processing batches:")
         print(list(range(i * 5, i * 5 + 5)))
@@ -716,6 +727,7 @@ def prep_mortality_damages(
 
     # longest-string gcm has to be processed first so the coordinate is the right str length
     gcms = sorted(gcms, key=len, reverse=True)
+    max_gcm_len = len(gcms[0])
 
     if mortality_version == 0:
         scaling_deaths = "epa_scaled"
@@ -826,7 +838,7 @@ def prep(
             if damages[v].dtype == object:
                 damages[v] = damages[v].astype("unicode")
 
-        damages.coords["gcm"] = damages.coords["gcm"].astype("object")
+        damages["gcm"] = damages["gcm"].astype("U" + str(max_gcm_len))
 
         if i == 0:
             damages.to_zarr(
@@ -852,11 +864,7 @@ def coastal_inputs(
     adapt_type,
     vsl_valuation=None,
 ):
-    try:
-        d = xr.open_zarr(f"{path}/coastal_damages_{version}.zarr")
-    except GroupNotFoundError:
-        print(f"Zarr not found: {path}/coastal_damages_{version}.zarr")
-        exit()
+    d = xr.open_zarr(f"{path}/coastal_damages_{version}.zarr")
 
     if "vsl_valuation" in d.coords:
         if vsl_valuation is None:

tests/test_main_recipe.py

@@ -258,11 +258,11 @@ def test_weitzman_min(menu_instance):
     diff = censored_values_slower[0:index] - censored_values[0:index]
     assert all(x > 0 for x in diff)
     as_share = menu_instance.weitzman_min(
-        [20, 50, 100], [5, 30, 60], as_share
+        np.array([20, 50, 100]), np.array([5, 30, 60]), as_share
     )  # the first value only should be censored
     assert as_share[0] != 5 and [as_share[1], as_share[2]] == [30, 60]
     all_cons = menu_instance.weitzman_min(
-        [20, 50, 100], [5, 30, 60], all_cons
+        np.array([20, 50, 100]), np.array([5, 30, 60]), all_cons
     )  # all values should be changed
     assert all([x != 0 for x in all_cons - [5, 30, 60]])