Pipeline polybench jobs

Author: DSouzaM

vm/ci/ci_common/common-bench.jsonnet

@@ -62,13 +62,9 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
   },
 
 
-  polybench_vm_common(env='${VM_ENV}', fail_fast=false, skip_machine=false): (if skip_machine then self.vm_bench_base(machine_name=null) else self.vm_bench_common) + common.deps.svm + vm.custom_vm {
-    local is_enterprise = (vm.edition == 'ee'),
-    setup+: [
-      ['mx', '--env', env, 'sforceimports'],
-      ['mx', '--env', env, 'build'],
-      ['set-export', 'POLYBENCH_JVM', ['mx', '--env', env, 'graalvm-home']],
-    ] + if is_enterprise then [['mx', '--dy', '/truffle-enterprise', 'build']] else [],
+  polybench_vm_common(os, arch, fail_fast=false, skip_machine=false): (if skip_machine then self.vm_bench_base(machine_name=null) else self.vm_bench_common) + vm_common.pipelined_graalvm(repo_config.graalvm_edition, os, arch, environment_variable='POLYBENCH_JVM') + {
+    local is_enterprise = (repo_config.graalvm_edition == 'ee'),
+    setup+: if is_enterprise then [['mx', '--dy', '/truffle-enterprise', 'build']] else [],
 
     # Extends the provided polybench command with common arguments used in CI. We want the command at the call site
     # to be simple (i.e., a flat array of string literals) so it can be easily copied and run locally; using this
@@ -82,7 +78,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
     notify_groups:: ['polybench'],
   },
 
-  polybench_vm_hpc_common: self.polybench_vm_common(skip_machine=true) + self.polybench_hpc_linux_common(shape='e4_8_64') + {
+  polybench_vm_hpc_common: self.polybench_vm_common('linux', 'amd64', skip_machine=true) + self.polybench_hpc_linux_common(shape='e4_8_64') + {
     polybench_wrap(command)::
       super.polybench_wrap(command) + [
         '--mx-benchmark-args',
@@ -91,12 +87,12 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet';
     teardown: [self.upload_and_wait_for_indexing + ['||', 'echo', 'Result upload failed!']],
   },
 
-  polybench_vm_gate(os, arch, language, name = null): vm_common.vm_base(os, arch, 'tier3') + self.polybench_vm_common(fail_fast=true, skip_machine=true) + vm.vm_java_Latest + {
+  polybench_vm_gate(os, arch, language, name = null): vm_common.vm_base(os, arch, 'tier3') + self.polybench_vm_common(os, arch, fail_fast=true, skip_machine=true) + vm.vm_java_Latest + {
     name: utils.hyphenize(['gate-vm', vm.vm_setup.short_name, 'polybench', language, name, utils.jdk_and_hardware(self)]),
     timelimit: '1:00:00',
   },
 
-  polybench_vm_daily(os, arch, language, name = null): vm_common.vm_base(os, arch, 'daily', bench=true) + self.polybench_vm_common() + vm.vm_java_Latest + {
+  polybench_vm_daily(os, arch, language, name = null): vm_common.vm_base(os, arch, 'daily', bench=true) + self.polybench_vm_common(os, arch) + vm.vm_java_Latest + {
     name: utils.hyphenize(['daily-bench-vm', vm.vm_setup.short_name, 'polybench', language, name, utils.jdk_and_hardware(self)]),
     teardown: [self.upload],
     timelimit: '4:00:00',

vm/ci/ci_common/common.jsonnet

@@ -2,7 +2,7 @@ local vm = import '../ci_includes/vm.jsonnet';
 local graal_common = import '../../../ci/ci_common/common.jsonnet';
 local galahad = import '../../../ci/ci_common/galahad-common.libsonnet';
 local utils = import '../../../ci/ci_common/common-utils.libsonnet';
-local repo_config = import '../../../repo-configuration.libsonnet';
+local repo_config = import '../../../ci/repo-configuration.libsonnet';
 local devkits = graal_common.devkits;
 
 {
@@ -572,6 +572,54 @@ local devkits = graal_common.devkits;
      name: 'gate-vm-native-sulong-' + self.jdk_name + '-linux-amd64',
   },
 
+  # loading large artifacts is not fast on those machines, so avoid them
+  big_artifacts: {
+    capabilities+: ["!x82", "!x82_16_367"]
+  },
+
+  # Builds a GraalVM and publishes a pipeline artifact. Use pipelined_graalvm to declare a dependency on this build.
+  # Be careful to only declare one job per set of arguments, otherwise there will be duplicate build jobs and enumeration may fail.
+  build_graalvm(vm, os, arch)::
+    graal_common[os + "_" + arch] + graal_common["labsjdk-" + vm + "-latest"] + self.big_artifacts + graal_common.deps.proguard + graal_common.deps.windows_devkit
+  {
+    local name = "graalvm-jdklatest-" + vm + "-" + os + "-" + arch,
+    local env = vm,
+    local mx = ["mx", "--strip-jars", "--env", env],
+    name: "build-" + name,
+    run+: [
+      ["cd", repo_config.vm.suite_dir],
+      mx + ["build", "--dependencies=GRAALVM"],
+      ["mkdir", "$BUILD_DIR/artifacts"],
+      ["mv", mx + ["--quiet", "--no-warning", "graalvm-home"], "$BUILD_DIR/artifacts/graalvm"],
+    ],
+    python_version: "3",
+    publishArtifacts+: [
+      {
+        name: name,
+        dir: "../artifacts",
+        patterns: ["*"],
+      },
+    ],
+    logs+: [
+      "*/mxbuild/dists/stripped/*.map",
+    ],
+    targets: ["ondemand"],
+    timelimit: "0:30:00",
+  },
+
+  # Use this helper to request a pipelined GraalVM in the current job. The job definition will export the pipelined VM
+  # to the given environment variable. There should be a corresponding build_graalvm job somewhere in the CI config.
+  pipelined_graalvm(vm, os, arch, environment_variable = "GRAALVM_HOME"): {
+    requireArtifacts+: [{
+      name: "graalvm-jdklatest-%s-%s-%s" % [vm, os, arch],
+      dir: "../artifacts",
+      autoExtract: true,
+    }],
+    setup+: [
+      ["set-export", environment_variable, "$BUILD_DIR/artifacts/graalvm"],
+    ],
+  },
+
   local builds = [
     #
     # Gates

vm/ci/ci_includes/vm.jsonnet

@@ -164,6 +164,9 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet';
     vm_common.deploy_vm_standalones_javaLatest_windows_amd64,
     # Trigger the releaser service and notify the indexer
     self.vm_notifier_daily,
+
+    # Pipelined builds
+    vm_common.build_graalvm("ce", "linux", "amd64"),
   ],
 
   builds: [vm_common.verify_name(b) for b in vm_common.builds + vm_common_runspec.builds + vm_common_bench.builds + vm_bench.builds + vm_native.builds + utils.add_defined_in(builds, std.thisFile)],