Postprocessing, DVC pipeline and Cml workflow

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/config

@@ -0,0 +1,7 @@
+[core]
+    remote = gcbm_logs
+['remote "gcbm_logs"']
+    url = gdrive://17Ef_mOvbWtY7wdp1o9WwzsGbQH_nSkza
+['remote "processed_output"']
+    url = gdrive://1xXQGus0NJB6bRTasUPACxQmoAPd1QeET
+

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.github/workflows/cml-container.yaml

@@ -0,0 +1,99 @@
+name: gcbm-Carpathians-ubuntu-CML
+on: 
+  push:
+    branches:
+      - main
+    paths:
+      - Standalone_GCBM/input_database/gcbm_input.db
+  pull_request:
+    branches:
+      - main
+    paths:
+      - Standalone_GCBM/input_database/gcbm_input.db
+
+jobs:
+  run-on-container:
+    runs-on: [ubuntu-latest]
+    container: 
+      image: ghcr.io/moja-global/rest_api_gcbm:master
+      ports:
+        - "8080:8080"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Run GCBM
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: 
+          cd Standalone_GCBM/gcbm_project
+
+          /opt/gcbm/moja.cli --config_file gcbm_config.cfg --config_provider provider_config.json
+
+      - name: Compile Results
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cd Standalone_GCBM/tools/CompileGCBMResults
+
+          python3 compileresults.py sqlite:///../../gcbm_project/output/gcbm_output.db --output_db sqlite:///../../processed_output/compiled_gcbm_output.db
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: compiled-gcbm-output
+          path: Standalone_GCBM/processed_output/compiled_gcbm_output.db
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: compiled-results-log
+          path: Standalone_GCBM/logs/compile_results.log
+
+  run-on-vmhost:
+    runs-on: [ubuntu-latest]
+    needs: run-on-container
+    steps:
+      - uses: actions/checkout@v3
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: '4.1.3'
+      - uses: r-lib/actions/setup-renv@v2
+      - uses: iterative/setup-cml@v1
+      - uses: iterative/setup-dvc@v1
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: compiled-gcbm-output
+          path: Standalone_GCBM/processed_output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: compiled-results-log
+          path: Standalone_GCBM/logs/
+
+      - name: Installations and renv restore
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: | 
+          pip install pandas
+          cd Postprocessing
+          Rscript -e "renv::activate()"
+          Rscript -e "renv::restore(lockfile = 'renv.lock')"
+
+      - name: dvc repro - postprocessing
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          dvc repro -s postprocessing
+
+      - name: Display plots for DOM stocks
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          dvc metrics diff --show-md > report.md
+          echo "## Figures created" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_BorealWet.png)" >> report.md 
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateMoist.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateSteppe.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateWet.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_PolarRainTundra.png)" >> report.md
+          cml comment create --pr=false report.md
+
+

.github/workflows/cml-windows.yaml

@@ -0,0 +1,107 @@
+name: gcbm-Carpathians-windows-CML
+on: 
+  push:
+    branches:
+      - main
+    paths:
+      - Standalone_GCBM/input_database/gcbm_input.db
+  pull_request:
+    branches:
+      - main
+    paths:
+      - Standalone_GCBM/input_database/gcbm_input.db
+
+jobs:
+  run-on-windows:
+    runs-on: [windows-latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.7.9'
+
+      - name: Required Installations
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: cmd
+        run: |
+          pip install installation\GDAL-2.4.1-cp37-cp37m-win_amd64.whl
+          del /q "C:\hostedtoolcache\windows\Python\3.7.9\x64\lib\site-packages\osgeo\gdalplugins\*.*"
+
+          pip install -r installation\requirements-py37.txt
+
+      - name: Run GCBM
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: cmd
+        run: |
+          cd Standalone_GCBM/gcbm_project
+          run_gcbm.bat pythonLocation
+
+      - name: Compile results
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: cmd
+        run: |
+          cd Standalone_GCBM/tools/CompileGCBMResults
+          python compileresults.py sqlite:///../../gcbm_project/output/gcbm_output.db --output_db sqlite:///../../processed_output/compiled_gcbm_output.db
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: compiled-gcbm-output
+          path: Standalone_GCBM/processed_output/compiled_gcbm_output.db
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: compiled-results-log
+          path: Standalone_GCBM/logs/compile_results.log
+
+  run-on-ubuntu:
+    runs-on: [ubuntu-latest]
+    needs: run-on-windows
+    steps:
+      - uses: actions/checkout@v3
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: '4.1.3'
+      - uses: r-lib/actions/setup-renv@v2
+      - uses: iterative/setup-cml@v1
+      - uses: iterative/setup-dvc@v1
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: compiled-gcbm-output
+          path: Standalone_GCBM/processed_output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: compiled-results-log
+          path: Standalone_GCBM/logs/
+
+      - name: Installations and renv restore
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: | 
+          pip install pandas
+          cd Postprocessing
+          Rscript -e "renv::activate()"
+          Rscript -e "renv::restore(lockfile = 'renv.lock')"
+
+      - name: dvc repro - postprocessing
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          dvc repro -s postprocessing
+
+      - name: Display plots for DOM stocks
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          dvc metrics diff --show-md > report.md
+          echo "## Figures created" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_BorealWet.png)" >> report.md 
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateMoist.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateSteppe.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateWet.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_PolarRainTundra.png)" >> report.md
+          cml comment create --pr=false report.md
+

.github/workflows/cml_compiled_db.yaml

@@ -0,0 +1,55 @@
+name: compiled-db-CML
+on: 
+  push:
+    branches:
+      - main
+    paths:
+      - Standalone_GCBM/logs/compile_results.log
+      - Standalone_GCBM/processed_output/compiled_gcbm_output.db
+  pull_request:
+    branches:
+      - main
+    paths:
+      - Standalone_GCBM/logs/compile_results.log
+      - Standalone_GCBM/processed_output/compiled_gcbm_output.db
+
+jobs:
+  run:
+    runs-on: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v3
+      - uses: r-lib/actions/setup-r@v2
+        with:
+          r-version: '4.1.3'
+      - uses: r-lib/actions/setup-renv@v2
+      - uses: iterative/setup-cml@v1
+      - uses: iterative/setup-dvc@v1
+
+      - name: Installations and renv restore
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: | 
+          pip install pandas
+          cd Postprocessing
+          Rscript -e "renv::activate()"
+          Rscript -e "renv::restore(lockfile = 'renv.lock')"
+
+      - name: dvc repro - postprocessing
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          dvc repro -s postprocessing
+
+      - name: Display plots for DOM stocks
+        env:
+          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          dvc metrics diff --show-md > report.md
+          echo "## Figures created" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_BorealWet.png)" >> report.md 
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateMoist.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateSteppe.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_CoolTemperateWet.png)" >> report.md
+          echo "![](Postprocessing/Figures/Carpathians_Sensitivity_PolarRainTundra.png)" >> report.md
+          cml comment create --pr=false report.md       
+

Installation/requirements-py37.txt

@@ -0,0 +1,24 @@
+cycler==0.10.0
+ftfy==5.6
+future==0.18.2
+geographiclib==1.50
+geopy==1.20.0
+imageio==2.6.1
+imageio-ffmpeg==0.3.0
+kiwisolver==1.1.0
+matplotlib==3.1.2
+pandas==0.25.3
+Pillow==7.0.0
+psutil==5.6.5
+pyparsing==2.4.6
+PySAL==1.14.4.post2
+python-dateutil==2.8.1
+pytz==2019.3
+scipy==1.4.1
+seaborn==0.10.0
+simplejson==3.16.0
+six==1.14.0
+SQLAlchemy==1.3.11
+utm==0.5.0
+wcwidth==0.1.8
+mojadata

Postprocessing/.Rprofile

@@ -0,0 +1 @@
+source("renv/activate.R")

Postprocessing/GCBM_Carpathians_Sensitivity.Rproj

@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX

Postprocessing/Metrics/Boreal_wet-Deadwood.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":61.3656,"pool_tc_per_ha_std":7.1105,"pool_tc_sum_mean":318510842.662800014,"area_sum_mean":5190377.0729}

Postprocessing/Metrics/Boreal_wet-Litter.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":17.5166,"pool_tc_per_ha_std":1.3261,"pool_tc_sum_mean":90917501.5828000009,"area_sum_mean":5190377.0729}

Postprocessing/Metrics/Boreal_wet-Soil_Carbon.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":65.9538,"pool_tc_per_ha_std":0.245,"pool_tc_sum_mean":342325029.793299973,"area_sum_mean":5190377.0729}

Postprocessing/Metrics/Boreal_wet-Total_Biomass.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":21.956,"pool_tc_per_ha_std":7.7473,"pool_tc_sum_mean":113959785.5372000039,"area_sum_mean":5190377.0729}

Postprocessing/Metrics/Cool_temperate_moist-Deadwood.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":59.8821,"pool_tc_per_ha_std":7.3325,"pool_tc_sum_mean":400746076.3432000279,"area_sum_mean":6692252.2209999999}

Postprocessing/Metrics/Cool_temperate_moist-Litter.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":16.7454,"pool_tc_per_ha_std":1.1646,"pool_tc_sum_mean":112064294.4546000063,"area_sum_mean":6692252.2209999999}

Postprocessing/Metrics/Cool_temperate_moist-Soil_Carbon.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":64.6953,"pool_tc_per_ha_std":0.2129,"pool_tc_sum_mean":432956982.3614000082,"area_sum_mean":6692252.2209999999}

Postprocessing/Metrics/Cool_temperate_moist-Total_Biomass.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":21.956,"pool_tc_per_ha_std":7.7473,"pool_tc_sum_mean":146934917.6662999988,"area_sum_mean":6692252.2209999999}

Postprocessing/Metrics/Cool_temperate_steppe-Deadwood.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":58.9054,"pool_tc_per_ha_std":7.4939,"pool_tc_sum_mean":2453711.0189,"area_sum_mean":41655.1375}

Postprocessing/Metrics/Cool_temperate_steppe-Litter.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":16.2071,"pool_tc_per_ha_std":1.0556,"pool_tc_sum_mean":675107.9201,"area_sum_mean":41655.1375}

Postprocessing/Metrics/Cool_temperate_steppe-Soil_Carbon.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":63.7701,"pool_tc_per_ha_std":0.1911,"pool_tc_sum_mean":2656352.6812,"area_sum_mean":41655.1375}

Postprocessing/Metrics/Cool_temperate_steppe-Total_Biomass.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":21.956,"pool_tc_per_ha_std":7.7473,"pool_tc_sum_mean":914579.1274,"area_sum_mean":41655.1375}

Postprocessing/Metrics/Cool_temperate_wet-Deadwood.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":59.6229,"pool_tc_per_ha_std":7.3642,"pool_tc_sum_mean":10998396.6458999999,"area_sum_mean":184465.8729}

Postprocessing/Metrics/Cool_temperate_wet-Litter.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":16.6263,"pool_tc_per_ha_std":1.1384,"pool_tc_sum_mean":3066986.6368,"area_sum_mean":184465.8729}

Postprocessing/Metrics/Cool_temperate_wet-Soil_Carbon.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":64.5214,"pool_tc_per_ha_std":0.2079,"pool_tc_sum_mean":11901988.0252,"area_sum_mean":184465.8729}

Postprocessing/Metrics/Cool_temperate_wet-Total_Biomass.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":21.956,"pool_tc_per_ha_std":7.7473,"pool_tc_sum_mean":4050127.9615000002,"area_sum_mean":184465.8729}

Postprocessing/Metrics/Polar_rain_tundra-Deadwood.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":63.2617,"pool_tc_per_ha_std":6.8518,"pool_tc_sum_mean":26225047.0208999999,"area_sum_mean":414548.8501}

Postprocessing/Metrics/Polar_rain_tundra-Litter.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":18.4499,"pool_tc_per_ha_std":1.5272,"pool_tc_sum_mean":7648375.8392000003,"area_sum_mean":414548.8501}

Postprocessing/Metrics/Polar_rain_tundra-Soil_Carbon.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":67.4019,"pool_tc_per_ha_std":0.2848,"pool_tc_sum_mean":27941363.8114,"area_sum_mean":414548.8501}

Postprocessing/Metrics/Polar_rain_tundra-Total_Biomass.json

@@ -0,0 +1 @@
+{"pool_tc_per_ha_mean":21.956,"pool_tc_per_ha_std":7.7473,"pool_tc_sum_mean":9101823.8918999992,"area_sum_mean":414548.8501}