Runner: use metadata file as input, remove benchmark_config.yaml (#126)

This PR fixes #117:
- Updates `runner/run_benchmarks.py` to use a metadata file (e.g.
`results/metadata.yaml`) as input instead of the special
`benchmarks_config.yaml`
- Deletes `benchmarks_config.yaml`
- Adds the info (e.g. URL) from `benchmarks_config.yaml` into the
individual metadata files
- Adds a CI check that `results/metadata.yaml` contains the merged
metadata of all metadata files under `benchmarks/`

Author: siddharth-krishna

.github/workflows/ci.yaml

@@ -33,6 +33,17 @@ jobs:
           pre-commit install
           pre-commit run --all-files
 
+      - name: Check benchmark metadata
+        run: |
+          python benchmarks/merge_metadata.py
+          if git diff --quiet; then
+            echo "git diff is empty, metadata must be up to date"
+          else
+            git diff
+            printf "ERROR: git diff non empty, metadata needs updating, please run the following command:\npython benchmarks/merge_metadata.py"
+            exit 1
+          fi
+
       - name: Run Streamlit App Action
         uses: streamlit/streamlit-app-action@v0.0.3
         with:
@@ -48,7 +59,7 @@ jobs:
       - name: Run sample benchmarks
         shell: bash -el {0}
         run: |
-          ./runner/benchmark_all.sh -y "tests" benchmarks/tests.yaml
+          ./runner/benchmark_all.sh -y "tests" tests/sample_benchmarks_metadata.yaml
           echo ""
           cat results/benchmark_results.csv
           if awk -F',' 'NR>1 {if ($6 != "ok") exit 1}' results/benchmark_results.csv; then

benchmarks/benchmark_config.yaml

@@ -8,13 +8,15 @@ genx-15-zones_elec_trex:
   Time horizon: Single period (1 year)
   MILP features:
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx-15-zones_elec_trex.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 11832658
     N. of continuous variables: 13777226
     N. of integer variables:
     N. of binary variables:
-
 genx-15-zones_elec_trex_co2:
   Short description: This is a one-year example which contains 15 zones. This example includes transmission expansion, a CO2 emissions constraint.
   Model name: GenX
@@ -25,13 +27,15 @@ genx-15-zones_elec_trex_co2:
   Time horizon: Single period (1 year)
   MILP features:
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx-15-zones_elec_trex_co2.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 5051253
     N. of continuous variables: 4573584
     N. of integer variables:
     N. of binary variables:
-
 genx-15-zones_elec_trex_co2_uc:
   Short description: This is a one-year example which contains 15 zones. This example includes transmission expansion, a CO2 emissions constraint and unit commitment for conventional generators.
   Model name: GenX
@@ -42,13 +46,15 @@ genx-15-zones_elec_trex_co2_uc:
   Time horizon: Single period (1 year)
   MILP features: Unit commitment
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx-15-zones_elec_trex_co2_uc.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 20128499
     N. of continuous variables: 13897939
     N. of integer variables: 3417368
     N. of binary variables: 120960
-
 genx-15-zones_elec_trex_uc:
   Short description: This is a one-year example which contains 15 zones. This example includes transmission expansion and unit commitment for conventional generators.
   Model name: GenX
@@ -59,13 +65,15 @@ genx-15-zones_elec_trex_uc:
   Time horizon: Single period (1 year)
   MILP features: Unit commitment
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx-15-zones_elec_trex_uc.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 20128499
     N. of continuous variables: 13897939
     N. of integer variables: 3417368
     N. of binary variables: 120960
-
 genx-15-zones_elec_co2:
   Short description: This is a one-year example which contains 15 zones. This example includes a CO2 emissions constraint.
   Model name: GenX
@@ -76,13 +84,15 @@ genx-15-zones_elec_co2:
   Time horizon: Single period (1 year)
   MILP features: Unit commitment
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx_15-zones_elec_co2.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 11832633
     N. of continuous variables: 13777201
     N. of integer variables:
     N. of binary variables:
-
 genx-15-zones_elec_co2_uc:
   Short description: This is a one-year example which contains 15 zones. This example includes a CO2 emissions constraint and unit commitment for conventional generators.
   Model name: GenX
@@ -93,13 +103,15 @@ genx-15-zones_elec_co2_uc:
   Time horizon: Single period (1 year)
   MILP features: Unit commitment
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx-15-zones_elec_co2_uc.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 20128473
     N. of continuous variables: 13897913
     N. of integer variables: 3417368
     N. of binary variables: 120960
-
 genx-15-zones_elec_uc:
   Short description: This is a one-year example which contains 15 zones. This example includes unit commitment for conventional generators.
   Model name: GenX
@@ -110,7 +122,10 @@ genx-15-zones_elec_uc:
   Time horizon: Single period (1 year)
   MILP features: Unit commitment
   Sizes:
-  - Temporal resolution: 168
+  - Name: 15-168h
+    Size: R
+    URL: https://storage.googleapis.com/solver-benchmarks/genx-15-zones_elec_uc.lp
+    Temporal resolution: 168
     Spatial resolution: 15
     N. of constraints: 20128472
     N. of continuous variables: 13897912