Skip to content

Commit acdc157

Browse files
KristijanFaust-OETKristijanFaust-OET
committed
Implement reference benchmark
1 parent 8b1664c commit acdc157

File tree

5 files changed

+163
-8
lines changed

5 files changed

+163
-8
lines changed

‎infrastructure/README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ Each YAML file defines a benchmark with specific configuration:
128128
| `enable_gcs_upload` | Enable/disable results upload to GCS bucket | true |
129129
| `gcs_bucket_name` | Name of the GCS bucket to upload the results | solver-benchmarks |
130130
| `auto_destroy_vm` | Enable/disable auto deletion of VM on benchmark completion | true |
131+
| `reference_benchmark_interval` | Time interval in seconds to run the reference benchmark | 3600 |
131132
| `ssh_user` | SSH username | "" |
132133
| `ssh_key_path` | Path to SSH public key | "" |
133134

‎infrastructure/main.tf‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,12 @@ variable "auto_destroy_vm" {
7373
default = true
7474
}
7575

76+
variable "reference_benchmark_interval" {
77+
description = "Time interval in seconds for running reference benchmarks (0 disables reference benchmarks)"
78+
type = number
79+
default = 3600
80+
}
81+
7682
locals {
7783
benchmark_files = fileset("${path.module}/benchmarks", "*.yaml*")
7884

@@ -124,6 +130,7 @@ resource "google_compute_instance" "benchmark_instances" {
124130
auto_destroy_vm = tostring(var.auto_destroy_vm)
125131
project_id = var.project_id
126132
zone = var.zone
133+
reference_benchmark_interval = tostring(var.reference_benchmark_interval)
127134
}
128135

129136
# Add the startup script from external file

‎infrastructure/startup-script.sh‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,17 @@ apt-get install -y tmux git time curl jq
1212
echo "Cloning repository..."
1313
git clone https://github.com/open-energy-transition/solver-benchmark.git
1414

15+
# Install a global highs binary for reference runs
16+
echo "Installing Highs..."
17+
mkdir -p /opt/highs/bin
18+
curl -L "https://github.com/JuliaBinaryWrappers/HiGHSstatic_jll.jl/releases/download/HiGHSstatic-v1.10.0%2B0/HiGHSstatic.v1.10.0.x86_64-linux-gnu-cxx11.tar.gz" -o HiGHSstatic.tar.gz
19+
tar -xzf HiGHSstatic.tar.gz -C /opt/highs/
20+
chmod +x /opt/highs/bin/highs
21+
/opt/highs/bin/highs --version
22+
23+
# Downloading benchmark reference model
24+
curl -L "https://storage.googleapis.com/solver-benchmarks/benchmark-test-model.lp" -o benchmark-test-model.lp
25+
1526
# Install Miniconda
1627
echo "Installing Miniconda..."
1728
mkdir -p ~/miniconda3
@@ -36,6 +47,10 @@ echo "Parsed benchmark years: ${BENCHMARK_YEARS_STR}"
3647
BENCHMARK_FILE=$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/benchmark_file")
3748
echo "Using benchmark file: ${BENCHMARK_FILE}"
3849

50+
# Get reference benchmark interval from instance metadata
51+
REFERENCE_BENCHMARK_INTERVAL=$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/reference_benchmark_interval")
52+
echo "Reference benchmark interval: ${REFERENCE_BENCHMARK_INTERVAL} seconds"
53+
3954
# Get benchmark content
4055
BENCHMARK_CONTENT=$(curl -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/benchmark_content")
4156

@@ -49,7 +64,7 @@ chmod +x ./runner/benchmark_all.sh
4964
# Run the benchmark_all.sh script with our years
5065
echo "Starting benchmarks for years: ${BENCHMARK_YEARS_STR}"
5166
source ~/miniconda3/bin/activate
52-
./runner/benchmark_all.sh -y "${BENCHMARK_YEARS_STR}" ./benchmarks/${BENCHMARK_FILE}
67+
./runner/benchmark_all.sh -y "${BENCHMARK_YEARS_STR}" -r "${REFERENCE_BENCHMARK_INTERVAL}" ./benchmarks/"${BENCHMARK_FILE}"
5368
BENCHMARK_EXIT_CODE=$?
5469

5570
if [ $BENCHMARK_EXIT_CODE -ne 0 ]; then

‎runner/benchmark_all.sh‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,17 @@ set -euo pipefail
44

55
# Parse command line arguments
66
usage() {
7-
echo "Usage: $0 [-a] [-y \"<space separated years>\"] <benchmarks yaml file>"
7+
echo "Usage: $0 [-a] [-y \"<space separated years>\"] [-r <seconds>] <benchmarks yaml file>"
88
echo "Runs the solvers from the specified years (default all) on the benchmarks in the given file"
99
echo "Options:"
1010
echo " -a Append to the results CSV file instead of overwriting. Default: overwrite"
1111
echo " -y A space separated string of years to run. Default: 2020 2021 2022 2023 2024"
12+
echo " -r Reference benchmark interval in seconds. Default: 0 (disabled)"
1213
}
1314
overwrite_results="true"
1415
years=(2020 2021 2022 2023 2024)
15-
while getopts "hay:" flag
16+
reference_interval=0 # Default: disabled
17+
while getopts "hay:r:" flag
1618
do
1719
case ${flag} in
1820
h) usage
@@ -23,6 +25,9 @@ do
2325
;;
2426
y) IFS=', ' read -r -a years <<< "$OPTARG"
2527
;;
28+
r) reference_interval="$OPTARG"
29+
echo "Reference benchmark will run every $reference_interval seconds"
30+
;;
2631
esac
2732
done
2833
shift $(($OPTIND - 1))
@@ -52,9 +57,9 @@ for year in "${years[@]}"; do
5257
echo "Running benchmarks for the year: $year"
5358
conda activate "$env_name"
5459
if [ "$idx" -eq 0 ]; then
55-
python "$BENCHMARK_SCRIPT" "$BENCHMARKS_FILE" "$year" "$overwrite_results"
60+
python "$BENCHMARK_SCRIPT" "$BENCHMARKS_FILE" "$year" "$overwrite_results" "$reference_interval"
5661
else
57-
python "$BENCHMARK_SCRIPT" "$BENCHMARKS_FILE" "$year" false
62+
python "$BENCHMARK_SCRIPT" "$BENCHMARKS_FILE" "$year" false "$reference_interval"
5863
fi
5964
conda deactivate
6065

‎runner/run_benchmarks.py‎

Lines changed: 130 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,12 @@
22
import gzip
33
import json
44
import os
5+
import re
56
import shutil
67
import statistics
78
import subprocess
89
import sys
10+
import time
911
from pathlib import Path
1012

1113
import requests
@@ -229,17 +231,101 @@ def benchmark_solver(input_file, solver_name, timeout):
229231
return metrics
230232

231233

234+
def get_highs_binary_version():
235+
"""Get the version of the HiGHS binary from the --version command"""
236+
highs_binary = "/opt/highs/bin/highs"
237+
238+
try:
239+
result = subprocess.run(
240+
[highs_binary, "--version"],
241+
capture_output=True,
242+
text=True,
243+
check=True,
244+
encoding="utf-8",
245+
)
246+
247+
version_match = re.search(r"HiGHS version (\d+\.\d+\.\d+)", result.stdout)
248+
if version_match:
249+
return version_match.group(1)
250+
251+
return "unknown"
252+
except Exception as e:
253+
print(f"Error getting HiGHS binary version: {str(e)}")
254+
return "unknown"
255+
256+
257+
def benchmark_highs_binary():
258+
"""
259+
Run a reference benchmark using the pre-installed HiGHS binary
260+
"""
261+
reference_model = "/benchmark-test-model.lp"
262+
highs_binary = "/opt/highs/bin/highs"
263+
264+
command = [
265+
highs_binary,
266+
reference_model,
267+
]
268+
269+
# Run the command and capture the output
270+
start_time = time.time()
271+
result = subprocess.run(
272+
command,
273+
capture_output=True,
274+
text=True,
275+
check=False,
276+
encoding="utf-8",
277+
)
278+
runtime = time.time() - start_time
279+
if result.returncode != 0:
280+
print(
281+
f"ERROR running reference benchmark. Captured output:\n{result.stdout}\n{result.stderr}"
282+
)
283+
metrics = {
284+
"status": "ER",
285+
"condition": "Error",
286+
"objective": None,
287+
"runtime": runtime,
288+
"duality_gap": None,
289+
"max_integrality_violation": None,
290+
}
291+
else:
292+
# Parse HiGHS output to extract objective value
293+
objective = None
294+
for line in result.stdout.splitlines():
295+
if "Objective value" in line:
296+
try:
297+
objective = float(line.split(":")[-1].strip())
298+
except (ValueError, IndexError):
299+
pass
300+
301+
metrics = {
302+
"status": "OK",
303+
"condition": "Optimal",
304+
"objective": objective,
305+
"runtime": runtime,
306+
"memory": "N/A",
307+
"duality_gap": None, # Not available from command line output
308+
"max_integrality_violation": None, # Not available from command line output
309+
}
310+
311+
return metrics
312+
313+
232314
def main(
233315
benchmark_yaml_path,
234316
solvers,
235317
year=None,
236318
iterations=1,
237319
timeout=10 * 60,
320+
reference_interval=0, # Default: disabled
238321
override=True,
239322
):
240323
size_categories = None # TODO add this to CLI args
241324
results = {}
242325

326+
# Track the last time we ran the reference benchmark
327+
last_reference_run = 0
328+
243329
# Load benchmarks from YAML file
244330
with open(benchmark_yaml_path, "r") as file:
245331
yaml_content = yaml.safe_load(file)
@@ -360,24 +446,65 @@ def main(
360446
results[(benchmark["name"], benchmark["size"], solver, solver_version)] = (
361447
metrics
362448
)
449+
450+
# Check if we should run the reference benchmark based on the interval
451+
if reference_interval > 0:
452+
current_time = time.time()
453+
time_since_last_run = current_time - last_reference_run
454+
455+
if last_reference_run == 0 or time_since_last_run >= int(
456+
reference_interval
457+
):
458+
print(
459+
f"Running reference benchmark with HiGHS binary (interval: {reference_interval}s)...",
460+
flush=True,
461+
)
462+
reference_metrics = benchmark_highs_binary()
463+
464+
# Add required fields to reference metrics
465+
reference_metrics["size"] = "reference"
466+
reference_metrics["solver"] = "highs-binary"
467+
reference_metrics["solver_version"] = get_highs_binary_version()
468+
reference_metrics["solver_release_year"] = "N/A"
469+
470+
# Record reference benchmark results
471+
write_csv_row(results_csv, "reference-benchmark", reference_metrics)
472+
473+
# Update the last reference run time
474+
last_reference_run = current_time
475+
else:
476+
print(
477+
f"Skipping reference benchmark (last run {time_since_last_run:.1f}s ago, interval: {reference_interval}s)",
478+
flush=True,
479+
)
480+
363481
return results
364482

365483

366484
if __name__ == "__main__":
367-
# Check for benchmark file argument and optional year and override arguments
485+
# Check for benchmark file argument and optional year, override, and reference interval arguments
368486
if len(sys.argv) < 3:
369487
raise ValueError(
370-
"Usage: python run_benchmarks.py <path_to_benchmarks.yaml> [<year>] [<override>]"
488+
"Usage: python run_benchmarks.py <path_to_benchmarks.yaml> [<year>] [<override>] [<reference_interval>]"
371489
)
372490
sys.exit(1)
373491

374492
benchmark_yaml_path = sys.argv[1]
375493
year = sys.argv[2] if len(sys.argv) > 2 else None
376494
override = sys.argv[3].lower() == "true" if len(sys.argv) > 3 else True
495+
reference_interval = (
496+
int(sys.argv[4]) if len(sys.argv) > 4 else 0
497+
) # Default: disabled
377498

378499
# solvers = ["highs", "glpk"] # For dev and testing
379500
solvers = ["highs", "scip", "cbc", "glpk"] # For production
380501

381-
main(benchmark_yaml_path, solvers, year, override=override)
502+
main(
503+
benchmark_yaml_path,
504+
solvers,
505+
year,
506+
reference_interval=reference_interval,
507+
override=override,
508+
)
382509
# Print a message indicating completion
383510
print("Benchmarking complete.")

0 commit comments

Comments
 (0)