diff --git a/devops/actions/run-tests/benchmark/action.yml b/devops/actions/run-tests/benchmark/action.yml index 6c070f83470c2..46df2313430c2 100644 --- a/devops/actions/run-tests/benchmark/action.yml +++ b/devops/actions/run-tests/benchmark/action.yml @@ -192,8 +192,10 @@ runs: sycl-ls echo "-----" - taskset -c "$CORES" ./devops/scripts/benchmarks/main.py \ - "$(realpath ./llvm_test_workdir)" \ + WORKDIR="$(realpath ./llvm_test_workdir)" + if [ -n "$WORKDIR" ] && [ -d "$WORKDIR" ] && [[ "$WORKDIR" == *llvm_test_workdir* ]]; then rm -rf "$WORKDIR" ; fi + + taskset -c "$CORES" ./devops/scripts/benchmarks/main.py "$WORKDIR" \ --sycl "$(realpath ./toolchain)" \ --ur "$(realpath ./ur/install)" \ --adapter "$FORCELOAD_ADAPTER" \ diff --git a/devops/scripts/benchmarks/README.md b/devops/scripts/benchmarks/README.md index 4f7341e3b1e8a..e7a2a1e743bb2 100644 --- a/devops/scripts/benchmarks/README.md +++ b/devops/scripts/benchmarks/README.md @@ -36,6 +36,10 @@ using the built compiler located in `~/llvm/build/` and installed Unified Runtime in directory `~/ur_install`, and then **run** the benchmarks for `adapter_name` adapter. +The scripts will try to reuse the files stored in `~/benchmarks_workdir/`. +If any dependant projects binaries are already built, they will not be rebuilt +again if their tags match tags specified by benchmarks source code. + >NOTE: By default `level_zero` adapter is used. >NOTE: Pay attention to the `--ur` parameter. It points directly to the directory where UR is installed. @@ -48,10 +52,6 @@ $ cmake --build ~/ur_build -j $(nproc) $ cmake --install ~/ur_build ``` -### Rebuild -The scripts will try to reuse the files stored in `~/benchmarks_workdir/`, but the benchmarks will be rebuilt every time. -To avoid that, use `--no-rebuild` option. - ## Results By default, the benchmark results are not stored. diff --git a/devops/scripts/benchmarks/benches/benchdnn.py b/devops/scripts/benchmarks/benches/benchdnn.py index d06f473a3697c..d7b7e8193a5b7 100644 --- a/devops/scripts/benchmarks/benches/benchdnn.py +++ b/devops/scripts/benchmarks/benches/benchdnn.py @@ -66,9 +66,13 @@ def setup(self) -> None: self.git_tag(), Path(options.workdir), "onednn", - force_rebuild=True, + use_installdir=False, ) + if not self.project.needs_rebuild(): + log.info(f"Rebuilding {self.project.name()} skipped") + return + extra_cmake_args = [ f"-DCMAKE_PREFIX_PATH={options.sycl}", "-DCMAKE_CXX_COMPILER=clang++", @@ -80,7 +84,6 @@ def setup(self) -> None: ] self.project.configure( extra_cmake_args, - install_prefix=False, add_sycl=True, ) self.project.build( diff --git a/devops/scripts/benchmarks/benches/compute.py b/devops/scripts/benchmarks/benches/compute.py index 6a2aea460ddb9..a8ecfbc10d207 100644 --- a/devops/scripts/benchmarks/benches/compute.py +++ b/devops/scripts/benchmarks/benches/compute.py @@ -74,9 +74,13 @@ def setup(self) -> None: self.git_hash(), Path(options.workdir), "compute-benchmarks", - force_rebuild=True, + use_installdir=False, ) + if not self.project.needs_rebuild(): + log.info(f"Rebuilding {self.project.name()} skipped") + return + extra_args = [ f"-DBUILD_SYCL=ON", f"-DSYCL_COMPILER_ROOT={options.sycl}", @@ -96,7 +100,7 @@ def setup(self) -> None: f"-Dunified-runtime_DIR={options.ur}/lib/cmake/unified-runtime", ] - self.project.configure(extra_args, install_prefix=False, add_sycl=True) + self.project.configure(extra_args, add_sycl=True) self.project.build(add_sycl=True) def additional_metadata(self) -> dict[str, BenchmarkMetadata]: diff --git a/devops/scripts/benchmarks/benches/gromacs.py b/devops/scripts/benchmarks/benches/gromacs.py index 2a8ce37e338e7..cd3acdccd51f8 100644 --- a/devops/scripts/benchmarks/benches/gromacs.py +++ b/devops/scripts/benchmarks/benches/gromacs.py @@ -57,7 +57,7 @@ def setup(self) -> None: self.git_tag(), Path(options.workdir), "gromacs", - force_rebuild=True, + use_installdir=False, ) # TODO: Detect the GPU architecture and set the appropriate flags @@ -83,8 +83,12 @@ def setup(self) -> None: if options.unitrace: extra_args.append("-DGMX_USE_ITT=ON") - self.project.configure(extra_args, install_prefix=False, add_sycl=True) - self.project.build(add_sycl=True, ld_library=self.oneapi.ld_libraries()) + if self.project.needs_rebuild(): + self.project.configure(extra_args, add_sycl=True) + self.project.build(add_sycl=True, ld_library=self.oneapi.ld_libraries()) + else: + log.info(f"Rebuilding {self.project.name()} skipped") + download( options.workdir, self.grappa_url(), diff --git a/devops/scripts/benchmarks/benches/llamacpp.py b/devops/scripts/benchmarks/benches/llamacpp.py index f93d040c246e0..86c84dae49186 100644 --- a/devops/scripts/benchmarks/benches/llamacpp.py +++ b/devops/scripts/benchmarks/benches/llamacpp.py @@ -39,7 +39,6 @@ def setup(self) -> None: self.git_hash(), Path(options.workdir), "llamacpp", - force_rebuild=True, ) models_dir = Path(options.workdir, "llamacpp-models") @@ -54,6 +53,10 @@ def setup(self) -> None: self.oneapi = get_oneapi() + if not self.project.needs_rebuild(): + log.info(f"Rebuilding {self.project.name()} skipped") + return + extra_args = [ f"-DGGML_SYCL=ON", f"-DCMAKE_C_COMPILER=clang", diff --git a/devops/scripts/benchmarks/benches/syclbench.py b/devops/scripts/benchmarks/benches/syclbench.py index 8707fc9ca704d..d4895c683cdb3 100644 --- a/devops/scripts/benchmarks/benches/syclbench.py +++ b/devops/scripts/benchmarks/benches/syclbench.py @@ -36,9 +36,13 @@ def setup(self) -> None: self.git_hash(), Path(options.workdir), "sycl-bench", - force_rebuild=True, + use_installdir=False, ) + if not self.project.needs_rebuild(): + log.info(f"Rebuilding {self.project.name()} skipped") + return + extra_args = [ f"-DCMAKE_CXX_COMPILER={options.sycl}/bin/clang++", f"-DCMAKE_C_COMPILER={options.sycl}/bin/clang", @@ -53,7 +57,7 @@ def setup(self) -> None: f"-DCMAKE_CXX_FLAGS=-fsycl -fsycl-targets=amdgcn-amd-amdhsa -Xsycl-target-backend --offload-arch={options.hip_arch}" ] - self.project.configure(extra_args, install_prefix=False, add_sycl=True) + self.project.configure(extra_args, add_sycl=True) self.project.build(add_sycl=True) def benchmarks(self) -> list[Benchmark]: diff --git a/devops/scripts/benchmarks/benches/velocity.py b/devops/scripts/benchmarks/benches/velocity.py index ec76df8a9b3b6..d7eb57611043b 100644 --- a/devops/scripts/benchmarks/benches/velocity.py +++ b/devops/scripts/benchmarks/benches/velocity.py @@ -106,12 +106,13 @@ def ld_libraries(self) -> list[str]: def setup(self): self.download_deps() - self.configure() - self.build() + if self.needs_rebuild(): + self.configure() + self.build() + else: + log.info(f"Skipping {self.bench_name} rebuild") def configure(self) -> None: - if options.rebuild and self.build_dir.exists(): - shutil.rmtree(self.build_dir) self.build_dir.mkdir(parents=True, exist_ok=True) cmd = [ diff --git a/devops/scripts/benchmarks/git_project.py b/devops/scripts/benchmarks/git_project.py index 32abfeecedb43..b28bc61c204b8 100644 --- a/devops/scripts/benchmarks/git_project.py +++ b/devops/scripts/benchmarks/git_project.py @@ -18,7 +18,7 @@ def __init__( ref: str, directory: Path, name: str, - force_rebuild: bool = False, + use_installdir: bool = True, no_suffix_src: bool = False, shallow_clone: bool = True, ) -> None: @@ -26,7 +26,7 @@ def __init__( self._ref = ref self._directory = directory self._name = name - self._force_rebuild = force_rebuild + self._use_installdir = use_installdir self._no_suffix_src = no_suffix_src self._shallow_clone = shallow_clone self._rebuild_needed = self._setup_repo() @@ -44,56 +44,29 @@ def build_dir(self) -> Path: def install_dir(self) -> Path: return self._directory / f"{self._name}-install" - def needs_rebuild(self, check_build=False, check_install=False) -> bool: - """Checks if the project needs to be rebuilt. + def needs_rebuild(self) -> bool: + if self._rebuild_needed: + log.debug( + f"Rebuild needed because new sources were detected for project {self._name}." + ) + return True - Args: - check_build (bool): If True, checks if the build directory exists and has some files. - check_install (bool): If True, checks if the install directory exists and has some files. + dir_to_check = self.install_dir if self._use_installdir else self.build_dir - Returns: - bool: True if the project needs to be rebuilt, False otherwise. - """ - log.debug(f"Checking if project {self._name} needs rebuild.") - if self._force_rebuild: + if not ( + dir_to_check.exists() + and any(path.is_file() for path in dir_to_check.glob("**/*")) + ): log.debug( - f"Force rebuild is enabled for project {self._name}, rebuild needed." + f"{dir_to_check} does not exist or does not contain any file, rebuild needed." ) - if Path(self.build_dir).exists(): - shutil.rmtree(self.build_dir) return True - elif self._rebuild_needed: - return True - if check_build: - if self.build_dir.exists() and any( - path.is_file() for path in self.build_dir.glob("**/*") - ): - log.debug( - f"Build directory {self.build_dir} exists and is not empty, no rebuild needed." - ) - else: - log.debug( - f"Build directory {self.build_dir} does not exist or does not contain any file, rebuild needed." - ) - return True - if check_install: - if self.install_dir.exists() and any( - path.is_file() for path in self.install_dir.glob("**/*") - ): - log.debug( - f"Install directory {self.install_dir} exists and is not empty, no rebuild needed." - ) - else: - log.debug( - f"Install directory {self.install_dir} does not exist or does not contain any file, rebuild needed." - ) - return True + log.debug(f"{dir_to_check} exists and is not empty, no rebuild needed.") return False def configure( self, extra_args: list | None = None, - install_prefix: bool = True, add_sycl: bool = False, ) -> None: """Configures the project.""" @@ -103,7 +76,7 @@ def configure( f"-B {self.build_dir}", f"-DCMAKE_BUILD_TYPE=Release", ] - if install_prefix: + if self._use_installdir: cmd.append(f"-DCMAKE_INSTALL_PREFIX={self.install_dir}") if extra_args: cmd.extend(extra_args) diff --git a/devops/scripts/benchmarks/main.py b/devops/scripts/benchmarks/main.py index ae3427d73ad10..28ab464e732ca 100755 --- a/devops/scripts/benchmarks/main.py +++ b/devops/scripts/benchmarks/main.py @@ -434,11 +434,6 @@ def validate_and_parse_env_args(env_args): help="Unified Runtime adapter to use.", default="level_zero", ) - parser.add_argument( - "--no-rebuild", - help="Do not rebuild the benchmarks from scratch.", - action="store_true", - ) parser.add_argument( "--redownload", help="Always download benchmark data dependencies, even if they already exist.", @@ -694,7 +689,6 @@ def validate_and_parse_env_args(env_args): additional_env_vars = validate_and_parse_env_args(args.env) options.workdir = args.benchmark_directory - options.rebuild = not args.no_rebuild options.redownload = args.redownload options.sycl = args.sycl options.iterations = args.iterations diff --git a/devops/scripts/benchmarks/options.py b/devops/scripts/benchmarks/options.py index d888774710a8c..1168affaf2575 100644 --- a/devops/scripts/benchmarks/options.py +++ b/devops/scripts/benchmarks/options.py @@ -54,7 +54,6 @@ class Options: ur: str = None ur_adapter: str = None umf: str = None - rebuild: bool = True redownload: bool = False benchmark_cwd: str = "INVALID" timeout: float = 600 diff --git a/devops/scripts/benchmarks/utils/compute_runtime.py b/devops/scripts/benchmarks/utils/compute_runtime.py index b3ef45aac859b..42a9b291e6a76 100644 --- a/devops/scripts/benchmarks/utils/compute_runtime.py +++ b/devops/scripts/benchmarks/utils/compute_runtime.py @@ -54,8 +54,8 @@ def build_gmmlib(self, repo, commit) -> tuple[Path, bool]: log.info("Building GMMLib...") project = GitProject(repo, commit, Path(options.workdir), "gmmlib") rebuilt = False - if project.needs_rebuild(check_install=True): - project.configure(install_prefix=True) + if project.needs_rebuild(): + project.configure() project.build() project.install() rebuilt = True @@ -74,8 +74,8 @@ def build_level_zero(self, repo, commit) -> tuple[Path, bool]: ) rebuilt = False - if project.needs_rebuild(check_install=True): - project.configure(install_prefix=True) + if project.needs_rebuild(): + project.configure() project.build() project.install() rebuilt = True @@ -88,7 +88,7 @@ def build_igc(self, repo, commit) -> tuple[Path, bool]: log.info("Building IGC...") igc_project = GitProject(repo, commit, Path(options.workdir), "igc") rebuilt = False - if igc_project.needs_rebuild(check_install=True): + if igc_project.needs_rebuild(): # Clone igc dependencies by creating a GitProject instance for each dependency. # Repos with commit hashes as refs can't be cloned shallowly. GitProject( @@ -142,7 +142,7 @@ def build_igc(self, repo, commit) -> tuple[Path, bool]: "-DCMAKE_C_FLAGS=-Wno-error", "-DCMAKE_CXX_FLAGS=-Wno-error", ] - igc_project.configure(extra_args=configure_args, install_prefix=True) + igc_project.configure(extra_args=configure_args) # set timeout to 2h. IGC takes A LONG time to build if building from scratch. igc_project.build(timeout=60 * 60 * 2) # cmake --install doesn't work... @@ -172,6 +172,7 @@ def build_compute_runtime(self): options.compute_runtime_tag, Path(options.workdir), "compute-runtime", + use_installdir=False, ) manifest_path = project.src_dir / "manifests" / "manifest.yml" @@ -190,7 +191,7 @@ def build_compute_runtime(self): self.igc, self.igc_rebuilt = self.build_igc(igc_repo, igc_commit) if ( - project.needs_rebuild(check_build=True) + project.needs_rebuild() or self.level_zero_rebuilt or self.gmmlib_rebuilt or (options.build_igc and self.igc_rebuilt)