Merge pull request #48 from NSAPH-Data-Processing/audiracmichelle/issue47

Audiracmichelle/issue47

Author: gilmourj

README.md

@@ -82,7 +82,7 @@ mamba activate <env_name>
 Run
 
 ```bash
-python utils/create_dir_paths.py 
+python src/create_dir_paths.py 
 ```
 
 ### Pipeline

Snakefile

@@ -1,26 +1,37 @@
-import yaml
+import hydra
+from omegaconf import OmegaConf
+import json
 
 conda: "requirements.yaml"
-configfile: "conf/config.yaml"
 
-# == Load configuration ==
+# the workflow configuration file is orchestrated by hydra
+# read config with hydra
+with hydra.initialize(config_path="conf", version_base=None):
+    cfg = hydra.compose(config_name="config", overrides=[])
+    #print(OmegaConf.to_yaml(cfg))
 
-# dynamic config files
-defaults_dict = {key: value for d in config['defaults'] if isinstance(d, dict) for key, value in d.items()}
-shapefiles_cfg = yaml.safe_load(open(f"conf/shapefiles/{defaults_dict['shapefiles']}.yaml", 'r'))
-# == Define variables ==
-shapefile_list = shapefiles_cfg.keys()
-print(shapefile_list)
+# convert to dict of single shapefile dicts 
+shapefiles_cfg = OmegaConf.to_container(cfg.shapefiles, resolve=True) 
+#print(shapefiles_cfg)
+shapefiles_cfg_dict = {shapefile["name"]: "[" + json.dumps(shapefile).replace('"', '') + "]" for shapefile in shapefiles_cfg}
+#print(shapefiles_cfg_dict)
+shapefiles_list = list(shapefiles_cfg_dict.keys())
+#print(shapefiles_list)
+# print(f"""
+#     python src/aggregate_climate_types.py "+shapefiles={shapefiles_cfg_dict[shapefiles_list[0]]}"
+#     """)
+
+#raise ValueError("stop here")
 
 rule all:
     input:
         expand(f"data/output/climate_types_raster2polygon/climate_types_{{shapefile_name}}.parquet", 
-            shapefile_name=shapefile_list
+            shapefile_name=shapefiles_list
         )
 
 rule download_climate_types:
     output:
-        f"data/input/climate_types/{config['climate_types_file']}"
+        f"data/input/climate_types/{cfg.climate_types_file}"
     shell:
         "python src/download_climate_types.py"
 
@@ -33,11 +44,17 @@ rule download_climate_types:
 
 rule aggregate_climate_types:
     input:
-        f"data/input/climate_types/{config['climate_types_file']}", 
+        f"data/input/climate_types/{cfg.climate_types_file}", 
         f"data/input/shapefiles/{{shapefile_name}}/{{shapefile_name}}.shp"
     output:
         f"data/output/climate_types_raster2polygon/climate_types_{{shapefile_name}}.parquet",
         f"data/intermediate/climate_pcts/climate_pcts_{{shapefile_name}}.json",
         f"data/intermediate/climate_pcts/climate_types_{{shapefile_name}}.csv"
+    params:
+        shapefile_name = lambda wildcards: shapefiles_cfg_dict[wildcards.shapefile_name]
     shell:
-        f"python src/aggregate_climate_types.py"
+        (f"""
+        echo {{wildcards.shapefile_name}}
+        python src/aggregate_climate_types.py "+shapefiles={{params.shapefile_name}}"
+        """)
+#python src/aggregate_climate_types.py "+shapefiles=[{name: CAN_ADM2, url: null, idvar: shapeID, output_idvar: id}]"

conf/config.yaml

@@ -1,9 +1,11 @@
 defaults:
   - _self_
-  - datapaths: datapaths
-  - shapefiles: county
+  - datapaths: example
+  - geoboundaries: example
   - climate_keys.yaml
 
+shapefiles: ${geoboundaries}
+
 # Climate types raw raster information
 climate_types_url:  https://figshare.com/ndownloader/files/12407516
 climate_types_file: Beck_KG_V1_present_0p0083.tif  # it's the one at highest resolution

conf/example_config.yaml

@@ -1,5 +1,5 @@
 #County shapefiles (cartographic boudaries) https://www.census.gov/programs-surveys/geography/guidance/tiger-data-products-guide.html
-county_2015:
+- name: county_2015
   url: https://www2.census.gov/geo/tiger/GENZ2015/shp/cb_2015_us_county_20m.zip
   idvar: GEOID
   output_idvar: county

conf/shapefiles/county.yaml conf/geoboundaries/county.yamlconf/shapefiles/county.yaml renamed to conf/geoboundaries/county.yaml

@@ -0,0 +1,12 @@
+- name: CAN_ADM2
+  url: null
+  idvar: shapeID
+  output_idvar: id
+- name: MEX_ADM2
+  url: null
+  idvar: shapeID
+  output_idvar: id
+- name: USA_ADM2
+  url: null
+  idvar: shapeID
+  output_idvar: id