Merge pull request #42 from NSAPH-Data-Processing/audiracmichelle/issue40

Audiracmichelle/issue40

Author: audiracmichelle

Dockerfile

@@ -18,6 +18,6 @@ RUN apt-get install -y libxrender-dev
 # Create paths to data placeholders
 RUN python utils/create_dir_paths.py datapaths=datapaths.yaml
 
-# snakemake --configfile conf/config.yaml --cores 4 -C shapefile_polygon_name=zcta
+# snakemake --configfile conf/config.yaml --cores 4
 ENTRYPOINT ["snakemake"]
-CMD ["--cores", "1", "-C", "shapefile_tag=2015", "shapefile_polygon_name=county"]
+CMD ["--cores", "1", "--configfile", "conf/config.yaml"]

README.md

@@ -101,10 +101,7 @@ python src/aggregate_climate_types.py
 or run the pipeline:
 
 ```bash
-# to generate county aggregations
 snakemake --cores 1 
-# to generate zcta aggregations
-snakemake --cores 1 -C shapefile_polygon_name=zcta
 ```
 
 ## Dockerized Pipeline
@@ -122,7 +119,7 @@ docker pull nsaph/climate_types_raster2polygon
 # to generate county aggregations
 docker run -v <test_path>:/app/data/output/climate_types_raster2polygon -t nsaph/climate_types_raster2polygon
 # to generate zcta aggregations
-docker run -v <test_path>:/app/data/output/climate_types_raster2polygon -t nsaph/climate_types_raster2polygon --cores 1 -C shapefile_polygon_name=zcta
+docker run -v <test_path>:/app/data/output/climate_types_raster2polygon -t nsaph/climate_types_raster2polygon --cores 1 
 ```
 
 ### If you want to build your own image use

Snakefile

@@ -3,12 +3,20 @@ import yaml
 conda: "requirements.yaml"
 configfile: "conf/config.yaml"
 
-tag=config["shapefile_tag"]
-polygon_name=config["shapefile_polygon_name"]
+# == Load configuration ==
+
+# dynamic config files
+defaults_dict = {key: value for d in config['defaults'] if isinstance(d, dict) for key, value in d.items()}
+shapefiles_cfg = yaml.safe_load(open(f"conf/shapefiles/{defaults_dict['shapefiles']}.yaml", 'r'))
+# == Define variables ==
+shapefile_list = shapefiles_cfg.keys()
+print(shapefile_list)
 
 rule all:
     input:
-        f"data/output/climate_types_raster2polygon/climate_types_{polygon_name}_{tag}.parquet"
+        expand(f"data/output/climate_types_raster2polygon/climate_types_{{shapefile_name}}.parquet", 
+            shapefile_name=shapefile_list
+        )
 
 rule download_climate_types:
     output:
@@ -19,17 +27,17 @@ rule download_climate_types:
 # temporarily removing download step in snakemake
 # rule download_shapefiles:
 #     output:
-#         f"data/input/shapefiles/shapefile_{polygon_name}_{tag}/shapefile.shp" #ext = ["shp", "shx", "dbf", "prj", "cpg", "xml"]
+#         f"data/input/shapefiles/{{shapefile_name}}/{{shapefile_name}}.shp" #ext = ["shp", "shx", "dbf", "prj", "cpg", "xml"]
 #     shell:
-#         f"python src/download_shapefile.py shapefile_tag={tag} shapefile_polygon_name={polygon_name}"
+#         f"python src/download_us_shapefile.py"
 
 rule aggregate_climate_types:
     input:
         f"data/input/climate_types/{config['climate_types_file']}", 
-        f"data/input/shapefiles/shapefile_{polygon_name}_{tag}/shapefile.shp"
+        f"data/input/shapefiles/{{shapefile_name}}/{{shapefile_name}}.shp"
     output:
-        f"data/output/climate_types_raster2polygon/climate_types_{polygon_name}_{tag}.parquet",
-        f"data/intermediate/climate_pcts/climate_pcts_{polygon_name}_{tag}.json",
-        f"data/intermediate/climate_pcts/climate_types_{polygon_name}_{tag}.csv"
+        f"data/output/climate_types_raster2polygon/climate_types_{{shapefile_name}}.parquet",
+        f"data/intermediate/climate_pcts/climate_pcts_{{shapefile_name}}.json",
+        f"data/intermediate/climate_pcts/climate_types_{{shapefile_name}}.csv"
     shell:
-        f"python src/aggregate_climate_types.py shapefile_tag={tag} shapefile_polygon_name={polygon_name}"
+        f"python src/aggregate_climate_types.py"

conf/climate_keys/climate_keys.yaml conf/climate_keys.yamlconf/climate_keys/climate_keys.yaml renamed to conf/climate_keys.yaml

@@ -1,3 +1,4 @@
+#@package climate_keys
 1:  ["Af", "Tropical, rainforest"]
 2:  ["Am", "Tropical, monsoon"]
 3:  ["Aw", "Tropical, savannah"]

conf/config.yaml

@@ -1,8 +1,8 @@
 defaults:
   - _self_
   - datapaths: datapaths
-  - shapefiles: global_shapefiles
-  - climate_keys: climate_keys
+  - shapefiles: county
+  - climate_keys.yaml
 
 # Climate types raw raster information
 climate_types_url:  https://figshare.com/ndownloader/files/12407516
@@ -13,10 +13,6 @@ climate_types_file: Beck_KG_V1_present_0p0083.tif  # it's the one at highest res
   # - Beck_KG_V1_present_conf_0p083.tif
   # - Beck_KG_V1_present_conf_0p5.tif
 
-# Shapefiles information
-shapefile_tag: ADM2all #to be matched with cfg.shapefiles
-shapefile_polygon_name: MEX #to be matched with cfg.shapefiles, column name of id in output file
-
 hydra:
   run:
     dir: logs/${now:%Y-%m-%d}/${now:%H-%M-%S}

conf/datapaths/example.yaml

@@ -0,0 +1,10 @@
+# if files are stored within the local copy of the repository, then use null:
+input:
+  climate_types: null
+  shapefiles: data/example/
+
+intermediate:
+  climate_pcts: null
+
+output:
+  climate_types_raster2polygon: null

conf/example_config.yaml

@@ -0,0 +1,18 @@
+defaults:
+  - _self_
+  - datapaths: example
+  - shapefiles: example
+  - climate_keys.yaml
+
+# Climate types raw raster information
+climate_types_url:  https://figshare.com/ndownloader/files/12407516
+climate_types_file: Beck_KG_V1_present_0p0083.tif  # it's the one at highest resolution
+  # - Beck_KG_V1_present_0p083.tif
+  # - Beck_KG_V1_present_0p5.tif
+  # - Beck_KG_V1_present_conf_0p0083.tif  # this is uncertainty
+  # - Beck_KG_V1_present_conf_0p083.tif
+  # - Beck_KG_V1_present_conf_0p5.tif
+
+hydra:
+  run:
+    dir: logs/${now:%Y-%m-%d}/${now:%H-%M-%S}

conf/shapefiles/county.yaml

@@ -0,0 +1,5 @@
+#County shapefiles (cartographic boudaries) https://www.census.gov/programs-surveys/geography/guidance/tiger-data-products-guide.html
+county_2015:
+  url: https://www2.census.gov/geo/tiger/GENZ2015/shp/cb_2015_us_county_20m.zip
+  idvar: GEOID
+  output_idvar: county

conf/shapefiles/example.yaml

@@ -0,0 +1,12 @@
+CAN_ADM2:
+    url: null
+    idvar: shapeID
+    output_idvar: id
+MEX_ADM2:
+    url: null
+    idvar: shapeID
+    output_idvar: id
+USA_ADM2:
+    url: null
+    idvar: shapeID
+    output_idvar: id

conf/shapefiles/global_shapefiles.yaml

@@ -1,15 +1,14 @@
-# Catalog of shapefiles
-# cfg.shapefile_tag and cfg.shapefile_polygon_name are used to match the shapefile to use
+#TODO: is it possible to have a single shapefile for all countries?
 
-CAN: 
-  ADM3all:
+global_ADM1:
     url: null
-    idvar: shapeID_3
-MEX:
-  ADM2all:
+    idvar: shapeID
+    output_idvar: id
+global_ADM2:
     url: null
-    idvar: shapeID_2
-USA:
-  ADM2all:
+    idvar: shapeID
+    output_idvar: id
+global_ADM3:
     url: null
-    idvar: shapeID_2
+    idvar: shapeID
+    output_idvar: id