diff --git a/devops/scripts/benchmarks/.gitignore b/devops/scripts/benchmarks/.gitignore new file mode 100644 index 0000000000000..a5f518f408db6 --- /dev/null +++ b/devops/scripts/benchmarks/.gitignore @@ -0,0 +1 @@ +/html/data.js diff --git a/devops/scripts/benchmarks/benches/base.py b/devops/scripts/benchmarks/benches/base.py index b34c7ec64d1f8..e9c0301fb7d15 100644 --- a/devops/scripts/benchmarks/benches/base.py +++ b/devops/scripts/benchmarks/benches/base.py @@ -6,6 +6,7 @@ from dataclasses import dataclass import os import shutil +import subprocess from pathlib import Path from utils.result import BenchmarkMetadata, BenchmarkTag, Result from options import options @@ -68,7 +69,9 @@ def get_adapter_full_path(): False ), f"could not find adapter file {adapter_path} (and in similar lib paths)" - def run_bench(self, command, env_vars, ld_library=[], add_sycl=True): + def run_bench( + self, command, env_vars, ld_library=[], add_sycl=True, use_stdout=True + ): env_vars = env_vars.copy() if options.ur is not None: env_vars.update( @@ -80,13 +83,18 @@ def run_bench(self, command, env_vars, ld_library=[], add_sycl=True): ld_libraries = options.extra_ld_libraries.copy() ld_libraries.extend(ld_library) - return run( + result = run( command=command, env_vars=env_vars, add_sycl=add_sycl, cwd=options.benchmark_cwd, ld_library=ld_libraries, - ).stdout.decode() + ) + + if use_stdout: + return result.stdout.decode() + else: + return result.stderr.decode() def create_data_path(self, name, skip_data_dir=False): if skip_data_dir: diff --git a/devops/scripts/benchmarks/benches/gromacs.py b/devops/scripts/benchmarks/benches/gromacs.py new file mode 100644 index 0000000000000..1d994d1a8cbfe --- /dev/null +++ b/devops/scripts/benchmarks/benches/gromacs.py @@ -0,0 +1,268 @@ +# Copyright (C) 2025 Intel Corporation +# Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM Exceptions. +# See LICENSE.TXT +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +import os +import subprocess +from pathlib import Path +from .base import Suite, Benchmark +from options import options +from utils.utils import git_clone, download, run, create_build_path +from utils.result import Result +import re + + +class GromacsBench(Suite): + + def git_url(self): + return "https://gitlab.com/gromacs/gromacs.git" + + def git_tag(self): + return "v2025.1" + + def grappa_url(self): + return "https://zenodo.org/record/11234002/files/grappa-1.5k-6.1M_rc0.9.tar.gz" + + def grappa_file(self): + return Path(os.path.basename(self.grappa_url())) + + def __init__(self, directory): + self.directory = Path(directory).resolve() + model_path = str(self.directory / self.grappa_file()).replace(".tar.gz", "") + self.grappa_dir = Path(model_path) + build_path = create_build_path(self.directory, "gromacs-build") + self.gromacs_build_path = Path(build_path) + self.gromacs_src = self.directory / "gromacs-repo" + + def name(self): + return "Gromacs Bench" + + def benchmarks(self) -> list[Benchmark]: + return [ + GromacsBenchmark(self, "0006", "pme", "graphs"), + GromacsBenchmark(self, "0006", "pme", "eager"), + GromacsBenchmark(self, "0006", "rf", "graphs"), + GromacsBenchmark(self, "0006", "rf", "eager"), + # some people may need it + # GromacsBenchmark(self, "0192", "pme", "eager"), + # GromacsBenchmark(self, "0192", "rf", "eager"), + ] + + def setup(self): + self.gromacs_src = git_clone( + self.directory, + "gromacs-repo", + self.git_url(), + self.git_tag(), + ) + + # TODO: Detect the GPU architecture and set the appropriate flags + + # Build GROMACS + run( + [ + "cmake", + f"-S {str(self.directory)}/gromacs-repo", + f"-B {self.gromacs_build_path}", + f"-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_CXX_COMPILER=clang++", + f"-DCMAKE_C_COMPILER=clang", + f"-DGMX_GPU=SYCL", + f"-DGMX_SYCL_ENABLE_GRAPHS=ON", + f"-DGMX_SYCL_ENABLE_EXPERIMENTAL_SUBMIT_API=ON", + f"-DGMX_FFT_LIBRARY=MKL", + f"-DGMX_GPU_FFT_LIBRARY=MKL", + f"-DGMX_GPU_NB_CLUSTER_SIZE=8", + f"-DGMX_GPU_NB_NUM_CLUSTER_PER_CELL_X=1", + f"-DGMX_OPENMP=OFF", + ], + add_sycl=True, + ) + run( + f"cmake --build {self.gromacs_build_path} -j {options.build_jobs}", + add_sycl=True, + ) + download( + self.directory, + self.grappa_url(), + self.directory / self.grappa_file(), + checksum="cc02be35ba85c8b044e47d097661dffa8bea57cdb3db8b5da5d01cdbc94fe6c8902652cfe05fb9da7f2af0698be283a2", + untar=True, + ) + + def teardown(self): + pass + + +class GromacsBenchmark(Benchmark): + def __init__(self, suite, model, type, option): + self.suite = suite + self.model = model # The model name (e.g., "0001.5") + self.type = type # The type of benchmark ("pme" or "rf") + self.option = option # "graphs" or "eager" + + self.gromacs_src = suite.gromacs_src + self.grappa_dir = suite.grappa_dir + self.gmx_path = suite.gromacs_build_path / "bin" / "gmx" + + if self.type == "pme": + self.extra_args = [ + "-pme", + "gpu", + "-pmefft", + "gpu", + "-notunepme", + ] + else: + self.extra_args = [] + + def name(self): + return f"gromacs-{self.model}-{self.type}-{self.option}" + + def setup(self): + if self.type != "rf" and self.type != "pme": + raise ValueError(f"Unknown benchmark type: {self.type}") + + if self.option != "graphs" and self.option != "eager": + raise ValueError(f"Unknown option: {self.option}") + + if not self.gmx_path.exists(): + raise FileNotFoundError(f"gmx executable not found at {self.gmx_path}") + + model_dir = self.grappa_dir / self.model + + if not model_dir.exists(): + raise FileNotFoundError(f"Model directory not found: {model_dir}") + + cmd_list = [ + str(self.gmx_path), + "grompp", + "-f", + f"{str(self.grappa_dir)}/{self.type}.mdp", + "-c", + str(model_dir / "conf.gro"), + "-p", + str(model_dir / "topol.top"), + "-o", + f"{str(model_dir)}/{self.type}.tpr", + ] + + # Generate configuration files + self.conf_result = run( + cmd_list, + add_sycl=True, + ) + + def run(self, env_vars): + model_dir = self.grappa_dir / self.model + + env_vars.update({"SYCL_CACHE_PERSISTENT": "1"}) + + if self.option == "graphs": + env_vars.update({"GMX_CUDA_GRAPH": "1"}) + + # Run benchmark + command = [ + str(self.gmx_path), + "mdrun", + "-s", + f"{str(model_dir)}/{self.type}.tpr", + "-nb", + "gpu", + "-update", + "gpu", + "-bonded", + "gpu", + "-ntmpi", + "1", + "-ntomp", + "1", + "-nobackup", + "-noconfout", + "-nstlist", + "100", + "-pin", + "on", + "-resethway", + ] + self.extra_args + + mdrun_output = self.run_bench( + command, + env_vars, + add_sycl=True, + use_stdout=False, + ) + + if self.type == "pme" and not self._validate_correctness( + options.benchmark_cwd + "/md.log" + ): + raise ValueError( + f"Validation failed: Conserved energy drift exceeds threshold in {model_dir / 'md.log'}" + ) + + time = self._extract_execution_time(mdrun_output) + + if options.verbose: + print(f"[{self.name()}] Time: {time:.3f} seconds") + + return [ + Result( + label=f"{self.name()}", + value=time, + unit="s", + command=command, + env=env_vars, + stdout=mdrun_output, + git_url=self.suite.git_url(), + git_hash=self.suite.git_tag(), + ) + ] + + def _extract_execution_time(self, log_content): + # Look for the line containing "Time:" + # and extract the first numeric value after it + time_lines = [line for line in log_content.splitlines() if "Time:" in line] + + if len(time_lines) != 1: + raise ValueError( + f"Expected exactly 1 line containing 'Time:' in the log content, " + f"but found {len(time_lines)}." + ) + + for part in time_lines[0].split(): + if part.replace(".", "", 1).isdigit(): + return float(part) + + raise ValueError(f"No numeric value found in the 'Time:' line.") + + def _validate_correctness(self, log_file): + threshold = 1e-3 # Define an acceptable energy drift threshold + + log_file = Path(log_file) + if not log_file.exists(): + raise FileNotFoundError(f"Log file not found: {log_file}") + + sci_pattern = r"([-+]?\d*\.\d+(?:e[-+]?\d+)?)" + with open(log_file, "r") as file: + for line in file: + if "Conserved energy drift:" in line: + match = re.search(sci_pattern, line, re.IGNORECASE) + if match: + try: + drift_value = float(match.group(1)) + return abs(drift_value) <= threshold + except ValueError: + print( + f"Parsed drift value: {drift_value} exceeds threshold" + ) + return False + else: + raise ValueError( + f"No valid numerical value found in line: {line}" + ) + + raise ValueError(f"Conserved Energy Drift not found in log file: {log_file}") + + def teardown(self): + pass diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index 397632e138978..d3d3eda3f5628 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -6,6 +6,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception from benches.compute import * +from benches.gromacs import GromacsBench from benches.velocity import VelocityBench from benches.syclbench import * from benches.llamacpp import * @@ -166,6 +167,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): SyclBench(directory), LlamaCppBench(directory), UMFSuite(directory), + GromacsBench(directory), TestSuite(), ] @@ -198,6 +200,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): except Exception as e: failures[s.name()] = f"Suite setup failure: {e}" print(f"{type(s).__name__} setup failed. Benchmarks won't be added.") + print(f"failed: {e}") else: print(f"{type(s).__name__} setup complete.") benchmarks += suite_benchmarks @@ -250,7 +253,7 @@ def main(directory, additional_env_vars, save_name, compare_names, filter): print(f"tearing down {benchmark.name()}... ", flush=True) benchmark.teardown() if options.verbose: - print("{benchmark.name()} teardown complete.") + print(f"{benchmark.name()} teardown complete.") this_name = options.current_run_name chart_data = {} diff --git a/devops/scripts/benchmarks/presets.py b/devops/scripts/benchmarks/presets.py index fc7e1ffb59f3d..0fb45cc4c943f 100644 --- a/devops/scripts/benchmarks/presets.py +++ b/devops/scripts/benchmarks/presets.py @@ -8,6 +8,7 @@ presets: dict[str, list[str]] = { "Full": [ "Compute Benchmarks", + "Gromacs Bench", "llama.cpp bench", "SYCL-Bench", "Velocity Bench", @@ -24,12 +25,16 @@ ], "Normal": [ "Compute Benchmarks", + "Gromacs Bench", "llama.cpp bench", "Velocity Bench", ], "Test": [ "Test Suite", ], + "Gromacs": [ + "Gromacs Bench", + ], }