[GR-65027] Run 'graalos' suite in one command

PullRequest: graal/20937

Author: Andrija Kolic

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.55.6",
+  "mx_version": "7.55.7",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {

sdk/mx.sdk/mx_sdk_benchmark.py

@@ -4003,7 +4003,7 @@ def run_stage(self, vm, stage: Stage, command, out, err, cwd, nonZeroIsFatal):
         # produced in the corresponding run stages.
         if not stage.is_image() and self.stages_info.should_produce_datapoints(stage.stage_name):
             final_command = self.apply_command_mapper_hooks(command, vm)
-        return mx.run(final_command, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal)
+        return mx.run(final_command, out=out, err=err, cwd=cwd, nonZeroIsFatal=nonZeroIsFatal, env=self.get_stage_env())
 
     def is_native_mode(self, bm_suite_args: List[str]):
         """Checks whether the given arguments request a Native Image benchmark"""
@@ -4139,6 +4139,10 @@ def build_assertions(self, benchmark, is_gate):
     def checkSamplesInPgo(self):
         return True
 
+    def get_stage_env(self) -> Optional[dict]:
+        """Return the environment to be used when executing a stage."""
+        return None
+
 
 def measureTimeToFirstResponse(bmSuite):
     protocolHost = bmSuite.serviceHost()

substratevm/mx.substratevm/mx_substratevm_benchmark.py

@@ -34,6 +34,7 @@
 from glob import glob
 from pathlib import Path
 from typing import List, Optional
+from contextlib import contextmanager
 
 import mx
 import mx_benchmark
@@ -561,18 +562,16 @@ class GraalOSNativeImageBenchmarkSuite(mx_benchmark.CustomHarnessBenchmarkSuite,
 
     This benchmark suite utilizes the `graalos-load-tester` harness to execute scenarios that run workloads against
     images of apps located in the `vm-benchmarks/graalos` repository.
-
-    Running a benchmark from this suite requires the following prerequisites:
-     * the `graalos-load-tester` repository is cloned, installed, and all of it's dependencies are installed
-     * the GOS_SCENARIO_HOME environment variable is set to point to the `graalos-load-tester` cloned repo
-     * the NIB (Native Image Bundle) has been generated for the application that comprises the benchmark
     """
     def __init__(self, custom_harness_command: mx_benchmark.CustomHarnessCommand = None):
         if custom_harness_command is None:
             custom_harness_command = GraalOSNativeImageBenchmarkSuite.GraalOSLoadTesterCommand()
         super().__init__(custom_harness_command)
         self._version = None
+        self._gos_scenario_home: Optional[Path] = None
         self._deployment = None
+        self._bundle_paths: dict[str, str] = {}
+        self._stage_env = os.environ.copy()
 
     def name(self):
         return "graalos"
@@ -592,21 +591,26 @@ def version(self):
             mx.log(f"Running GraalOS Load Tester version '{self._version}'")
         return self._version
 
-    def _gos_scenario_home(self) -> Path:
-        """Verifies that the GOS_SCENARIO_HOME env var points to a directory and then returns the path to it."""
+    @property
+    def gos_scenario_home(self) -> Path:
+        if self._gos_scenario_home is None:
+            self._gos_scenario_home = self._load_gos_scenario_home()
+        return self._gos_scenario_home
+
+    def _load_gos_scenario_home(self) -> Path:
+        """
+        Returns the path to the 'graalos-load-tester' directory sibling to the root 'graal' directory,
+        cloning the remote repository if it is missing.
+
+        :raises StopIteration: If 'graalos-load-tester' is not found in the list of ignored suites.
+        """
         try:
-            gos_scenario_home_env_var = mx.get_env("GOS_SCENARIO_HOME")
-            if gos_scenario_home_env_var is None:
-                raise ValueError("GOS_SCENARIO_HOME is not set!")
-            gos_scenario_home = Path(gos_scenario_home_env_var)
-            if not gos_scenario_home.is_dir():
-                raise ValueError("GOS_SCENARIO_HOME does not point to an existing directory!")
-            return gos_scenario_home
-        except ValueError as e:
-            mx.abort(
-                str(e) + "\nPlease set the GOS_SCENARIO_HOME environment variable to point"
-                " to a copy of the 'graalos-load-tester' repository."
-            )
+            glt_suite = mx.suite("graalos-load-tester", fatalIfMissing=False)
+            if glt_suite is None:
+                glt_suite = mx.primary_suite().clone_foreign_suite("graalos-load-tester", clone_binary_first=False)
+            return Path(glt_suite.dir)
+        except StopIteration:
+            mx.abort("Cloning of 'graalos-load-tester' as a sibling of the current suite has failed!")
 
     def _read_gos_scenario_version(self):
         """
@@ -615,7 +619,7 @@ def _read_gos_scenario_version(self):
         """
         # Revisit this method once we rework versioning for graalos-load-tester (GR-59986)
         try:
-            return mx.GitConfig().git_command(self._gos_scenario_home(), ["describe", "--tags", "--abbrev=0"]).strip()
+            return mx.GitConfig().git_command(self.gos_scenario_home, ["describe", "--tags", "--abbrev=0"]).strip()
         except:
             return self.defaultSuiteVersion()
 
@@ -625,7 +629,7 @@ def _gos_scenario_command(self) -> str:
 
     def _gos_scenarios_dir(self) -> Path:
         """Verifies that the root scenarios directory exists and returns the path to it."""
-        scenarios_dir = self._gos_scenario_home() / "scenarios"
+        scenarios_dir = self.gos_scenario_home / "scenarios"
         if not scenarios_dir.is_dir():
             raise ValueError(f"Directory '{scenarios_dir}' is supposed to contain load-testing scenarios"
                              f" but instead it does not exist!")
@@ -639,12 +643,21 @@ def _app_source_dir(self, app: str) -> Path:
         """Returns the path to the source code directory of the application."""
         return self._vm_benchmarks_graalos_dir() / app / "app"
 
-    def _check_if_gos_scenario_command_is_installed(self):
-        """Verifies that the command that executes the `graalos-load-tester` benchmarking harness is installed."""
+    @contextmanager
+    def catch_all_errors(self, propagate):
+        """
+        Context manager that catches any error raised by the managed code and either suppresses or propagates it.
+
+        The error is either propagated or suppressed based on the value of the `propagate` parameter:
+          * If `propagate` is `True` then any error raised by the managed code is propagated - the behaviour is as if
+            the manager was absent.
+          * If `propagate` is `False` then any error raised by the managed code is suppressed.
+        """
         try:
-            mx.run([self._gos_scenario_command(), "--help"], out=mx.OutputCapture())
+            yield
         except:
-            mx.abort("Please install the 'gos-scenario' command from the 'graalos-load-tester' repository!")
+            if propagate:
+                raise
 
     def completeBenchmarkList(self, bmSuiteArgs):
         return _graalosConfig["benchmarks"].keys()
@@ -661,11 +674,42 @@ def benchmarkList(self, bmSuiteArgs):
         # Exclude any benchmarks unsupported on the current platform, JDK version, VM
         return self.completeBenchmarkList(bmSuiteArgs)
 
+    def _install_graalos_load_tester(self):
+        """
+        Installs the 'graalos-load-tester' project and its dependencies and modifies the PATH
+        environment variable inside `_stage_env` to include paths emitted by the installation script.
+        """
+        install_script_path = self.gos_scenario_home / "mx-devenv" / "local-install.py"
+        install_cmd = [str(install_script_path)]
+
+        mx.log(f"Installing `graalos-load-tester' with: {install_cmd}")
+        out = mx.OutputCapture()
+        err = mx.OutputCapture()
+        try:
+            mx.run(install_cmd, out=out, err=err)
+        except BaseException as e:
+            for line in out.data.split("\n"):
+                mx.log(line)
+            for line in err.data.split("\n"):
+                mx.log_error(line)
+            if isinstance(e, SystemExit):
+                mx.abort(f"Installing 'graalos-load-tester' failed with exit code {e}!")
+            else:
+                mx.abort(f"{e}\nInstalling 'graalos-load-tester' failed!")
+
+        for line in out.data.split("\n"):
+            mx.log(line)
+        path_env_var_pattern = r"^.*export PATH=(.*):\$PATH$"
+        for line in out.data.split("\n"):
+            path_entry_match = re.match(path_env_var_pattern, line)
+            if path_entry_match:
+                new_path_entry = path_entry_match.group(1)
+                mx.log(f"Prepending '{new_path_entry}' to the PATH environment variable for the duration of the benchmark.")
+                self._stage_env["PATH"] = f"{new_path_entry}:{self._stage_env['PATH']}"
+
     def validateEnvironment(self):
-        # Verify GOS_SCENARIO_HOME env var is set
-        self._gos_scenario_home()
-        # Verify 'gos-scenario' command is installed
-        self._check_if_gos_scenario_command_is_installed()
+        # Make sure 'graalos-load-tester' repo is present and dependencies are installed
+        self._install_graalos_load_tester()
 
     def new_execution_context(self, vm: Vm, benchmarks: List[str], bmSuiteArgs: List[str]) -> SingleBenchmarkExecutionContext:
         return SingleBenchmarkExecutionContext(self, vm, benchmarks, bmSuiteArgs)
@@ -687,19 +731,52 @@ def _get_benchmark_config(self, benchmark):
         return _graalosConfig["benchmarks"][benchmark]
 
     def applicationDist(self):
+        app_name = self._get_benchmark_config(self.benchmarkName())["app"]
+        if app_name not in self._bundle_paths:
+            self._bundle_paths[app_name] = self.generate_or_lookup_bundle()
+            mx.log(f"Using bundle at '{self._bundle_paths[app_name]}' for app '{app_name}'.")
+        return self._bundle_paths[app_name].parent
+
+    def generate_or_lookup_bundle(self) -> Path:
+        """
+        Looks up the path to the NIB file for the app asociated with the current benchmark,
+        generating it first if it does not exist.
+        """
+        # Initial NIB lookup
         app_name = self._get_benchmark_config(self.benchmarkName())["app"]
         app_dir = self._app_source_dir(app_name)
         nib_candidates = list(app_dir.glob("**/*.nib"))
+
+        # Generate a NIB file for the app if none exists
+        if len(nib_candidates) == 0:
+            nib_generation_cmd = ["./graalos-gate.py", "build", "--build-profile", "nib", app_name]
+            working_dir = self._vm_benchmarks_graalos_dir()
+            mx.log(f"Generating the NIB file by running {nib_generation_cmd} in working dir {working_dir}")
+            out = mx.OutputCapture()
+            err = mx.OutputCapture()
+            try:
+                mx.run(nib_generation_cmd, cwd=working_dir, out=out, err=err, env=self.get_nib_generation_env())
+            except BaseException as e:
+                for line in out.data.split("\n"):
+                    mx.log(line)
+                for line in err.data.split("\n"):
+                    mx.log_error(line)
+                if isinstance(e, SystemExit):
+                    mx.abort(f"Generating the NIB file failed with exit code {e}!")
+                else:
+                    mx.abort(f"{e}\nGenerating the NIB file failed!")
+            for line in out.data.split("\n"):
+                mx.logvv(line)
+            # Repeat the lookup
+            nib_candidates = list(app_dir.glob("**/*.nib"))
+
+        # Final check
         if len(nib_candidates) == 0:
-            build_cmd_to_run = [f"{self._vm_benchmarks_graalos_dir()}/graalos-gate.py",
-                                "build", "--build-profile", "nib", app_name]
-            build_cmd_to_run = " ".join(build_cmd_to_run)
-            mx.log_error(f"Did you forget to run: '{build_cmd_to_run}'")
             mx.abort(f"Expected to find exactly one '.nib' file in the '{app_dir}' app directory, instead found none!")
         if len(nib_candidates) > 1:
             mx.abort(f"Expected to find exactly one '.nib' file in the '{app_dir}' app directory, instead found "
                      + "multiple: [" + ", ".join(str(path) for path in nib_candidates) + "]")
-        return nib_candidates[0].parent
+        return nib_candidates[0]
 
     def uses_bundles(self):
         return True
@@ -808,6 +885,16 @@ def _get_runnable_app_image(self):
             return accessible_app_image_path
         return original_app_image_path
 
+    def get_stage_env(self) -> Optional[dict]:
+        return self._stage_env
+
+    def get_nib_generation_env(self):
+        env = self.get_stage_env().copy()
+        graalvm_home = self.execution_context.virtual_machine.home()
+        env["GRAALVM_HOME"] = graalvm_home
+        env["GRADLE_CLIENT_JAVA_HOME"] = graalvm_home
+        return env
+
     def run(self, benchmarks, bmSuiteArgs) -> DataPoints:
         return self.intercept_run(super(), benchmarks, bmSuiteArgs)
 
@@ -842,7 +929,7 @@ def produceHarnessCommand(self, cmd: List[str], suite: BenchmarkSuite) -> List[s
                 # Add explicit run stage args
                 app_cmd += parse_prefixed_args("-Dnative-image.benchmark.extra-run-arg=", bmSuiteArgs)
 
-            gos_cmd = ["gos-scenario", f"{scenario}", "--local-load-testers", "--skip-upload"]
+            gos_cmd = [suite._gos_scenario_command(), f"{scenario}", "--local-load-testers", "--skip-upload"]
             timestamp = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
             gos_log_file_name = f"{timestamp}-gos-out.log"
             gos_cmd += ["--log-to", f"stdout,file:{gos_log_file_name}"]

substratevm/mx.substratevm/suite.py

@@ -1,6 +1,6 @@
 # pylint: disable=line-too-long
 suite = {
-    "mxversion": "7.55.2",
+    "mxversion": "7.55.7",
     "name": "substratevm",
     "version" : "26.0.0",
     "release" : False,