Merge in master

Author: zapster

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.58.6",
+  "mx_version": "7.58.9",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {

compiler/ci/ci_common/benchmark-suites.libsonnet

@@ -129,7 +129,7 @@
 
   barista_template(suite_version=null, suite_name="barista", max_jdk_version=null, cmd_app_prefix=["hwloc-bind --cpubind node:0.core:0-3.pu:0 --membind node:0"], non_prefix_barista_args=[]):: cc.compiler_benchmark + {
     suite:: suite_name,
-    local barista_version = "v0.4.4",
+    local barista_version = "v0.4.6",
     local suite_version_args = if suite_version != null then ["--bench-suite-version=" + suite_version] else [],
     local prefix_barista_arg = if std.length(cmd_app_prefix) > 0 then [std.format("--cmd-app-prefix=%s", std.join(" ", cmd_app_prefix))] else [],
     local all_barista_args = prefix_barista_arg + non_prefix_barista_args,

compiler/ci/ci_common/gate.jsonnet

@@ -66,6 +66,18 @@
     python_version: "3"
   },
 
+  tier1:: {
+    targets: ["tier1"],
+  },
+
+  tier2:: {
+    targets: ["tier2"],
+  },
+
+  tier3:: {
+    targets: ["tier3"],
+  },
+
   daily:: {
     # Re-use existing mailing list for weeklies
     notify_groups: ["compiler_weekly"],
@@ -87,7 +99,11 @@
   },
 
   test:: s.base(no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"),
+  unittest_compiler:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"),
+  unittest_truffle:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-Djdk.graal.DetailedAsserts=true"),
   test_zgc:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
+  unittest_compiler_zgc:: s.base(tags="build,unittest-compiler", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
+  unittest_truffle_zgc:: s.base(tags="build,unittest-truffle", no_warning_as_error=true, extra_vm_args="-XX:+UseZGC"),
   test_serialgc:: s.base(no_warning_as_error=true, extra_vm_args="-XX:+UseSerialGC"),
 
   jacoco_gate_args:: ["--jacoco-omit-excluded", "--jacoco-relativize-paths", "--jacoco-omit-src-gen", "--jacocout", "coverage", "--jacoco-format", "lcov"],
@@ -175,10 +191,6 @@
     }
   },
 
-  many_cores:: {
-    capabilities+: ["manycores"]
-  },
-
   # Returns the value of the `name` field if it exists in `obj` otherwise `default`.
   get(obj, name, default=null)::
       if obj == null then default else
@@ -195,16 +207,6 @@
 
   manifest_match(manifest, name):: [key for key in std.objectFields(manifest) if key_matches_value(key, name)] != [],
 
-  # Request nodes with at least 16GB of RAM
-  ram16gb:: {
-    capabilities+: ["ram16gb"],
-  },
-
-  # Avoid using xgene3 nodes
-  avoid_xgene3:: {
-    capabilities+: ["!xgene3"],
-  },
-
   jdk_latest:: "Latest",
 
   # Filters out the non-style gate jobs if in CE
@@ -213,102 +215,140 @@
     for name in std.objectFields(gate_jobs) if utils.contains(name, "style")
   } else gate_jobs,
 
-  # Converts the non-style gate jobs to dailies if in CE
-  as_dailies(gate_jobs):: if config.graalvm_edition == "ce" then {
+  # Converts the non-style jobs to dailies if in CE
+  as_dailies(jobs):: if config.graalvm_edition == "ce" then {
     # Force daily timelimit (assumes it is greater than any specified gate timelimit)
-    [std.strReplace(name, "gate", "daily")]: gate_jobs[name] + {timelimit: $.daily.timelimit}
-    for name in std.objectFields(gate_jobs) if !utils.contains(name, "style")
+    [name]: jobs[name] + {timelimit: $.daily.timelimit}
+    for name in std.objectFields(jobs) if !utils.contains(name, "style")
   } else {},
 
-  # Candidates for gate jobs. In CE, these will be dailies instead of gates.
-  local gate_jobs = {
-    "gate-compiler-test-labsjdk-latest-linux-amd64": t("1:00:00"),
-    "gate-compiler-test-labsjdk-latest-linux-aarch64": t("1:50:00") + s.avoid_xgene3,
-    "gate-compiler-test-labsjdk-latest-darwin-amd64": t("1:00:00") + s.ram16gb,
-    "gate-compiler-test-labsjdk-latest-darwin-aarch64": t("1:00:00"),
-    "gate-compiler-test-labsjdk-latest-windows-amd64": t("1:30:00"),
-    "gate-compiler-test_zgc-labsjdk-latest-linux-amd64": t("1:00:00"),
-    "gate-compiler-test_zgc-labsjdk-latest-linux-aarch64": t("1:50:00") + s.avoid_xgene3,
-    "gate-compiler-test_zgc-labsjdk-latest-darwin-amd64": t("1:00:00") + s.ram16gb,
-    "gate-compiler-test_zgc-labsjdk-latest-darwin-aarch64": t("1:00:00"),
-
+  # Candidates for Tier1 jobs. In CE, these will be dailies.
+  local tier1_jobs = {
     # Style jobs need to stay on a JDK compatible with all the style
     # checking tools (SpotBugs, Checkstyle, Eclipse formatter etc).
-    "gate-compiler-style-labsjdk-latest-linux-amd64": t("45:00"),
+    "compiler-style-labsjdk-latest-linux-amd64": {},
+  },
 
-    "gate-compiler-ctw-labsjdk-latest-linux-amd64": {},
-    "gate-compiler-ctw-labsjdk-latest-windows-amd64": t("1:50:00"),
-    "gate-compiler-ctw_zgc-labsjdk-latest-linux-amd64": {},
+  # Candidates for Tier2 jobs. In CE, these will be dailies.
+  local tier2_jobs = {
+    "compiler-unittest_compiler-labsjdk-latest-linux-amd64": {},
+    "compiler-unittest_truffle-labsjdk-latest-linux-amd64": {},
+    "compiler-ctw-labsjdk-latest-linux-amd64": {},
+    "compiler-ctw_economy-labsjdk-latest-linux-amd64": {},
+    "compiler-benchmarktest-labsjdk-latest-linux-amd64": {},
+  },
 
-    "gate-compiler-ctw_economy-labsjdk-latest-linux-amd64": {},
-    "gate-compiler-ctw_economy-labsjdk-latest-windows-amd64": t("1:50:00"),
+  # Candidates for Tier3 jobs. In CE, these will be dailies.
+  local tier3_jobs = {
+    "compiler-unittest_compiler-labsjdk-latest-darwin-aarch64": {},
+    "compiler-unittest_truffle-labsjdk-latest-darwin-aarch64": {},
+    "compiler-unittest_compiler-labsjdk-latest-linux-aarch64": {},
+    "compiler-unittest_truffle-labsjdk-latest-linux-aarch64": {},
 
-    "gate-compiler-benchmarktest-labsjdk-latest-linux-amd64": {},
-    "gate-compiler-benchmarktest_zgc-labsjdk-latest-linux-amd64": {},
+    "compiler-unittest_compiler_zgc-labsjdk-latest-linux-amd64": {},
 
-    "gate-compiler-truffle_xcomp-labsjdk-latest-linux-amd64": t("1:30:00"),
-    "gate-compiler-truffle_xcomp_zgc-labsjdk-latest-linux-amd64": t("1:30:00")
+    "compiler-truffle_xcomp-labsjdk-latest-linux-amd64": {},
   },
 
+  # Candidates for gate jobs. In CE, these will be dailies instead of gates.
+  local gate_jobs = {},
+
   # This map defines the builders that run as gates. Each key in this map
   # must correspond to the name of a build created by `make_build`.
   # Each value in this map is an object that overrides or extends the
   # fields of the denoted build.
   local gates = $.as_gates(gate_jobs),
 
+  # This map defines the builders that run as tier1. Each key in this map
+  # must correspond to the name of a build created by `make_build`.
+  # Each value in this map is an object that overrides or extends the
+  # fields of the denoted build.
+  local tier1s = $.as_gates(tier1_jobs),
+
+  # This map defines the builders that run as tier2. Each key in this map
+  # must correspond to the name of a build created by `make_build`.
+  # Each value in this map is an object that overrides or extends the
+  # fields of the denoted build.
+  local tier2s = $.as_gates(tier2_jobs),
+
+  # This map defines the builders that run as tier3. Each key in this map
+  # must correspond to the name of a build created by `make_build`.
+  # Each value in this map is an object that overrides or extends the
+  # fields of the denoted build.
+  local tier3s = $.as_gates(tier3_jobs),
+
   # This map defines the builders that run daily. Each key in this map
   # must be the name of a build created by `make_build` (or be the prefix
   # of a build name if the key ends with "*").
   # Each value in this map is an object that overrides or extends the
   # fields of the denoted build.
   local dailies = {
-    "daily-compiler-ctw-labsjdk-latest-linux-aarch64": {},
-    "daily-compiler-ctw-labsjdk-latest-darwin-amd64": {},
-    "daily-compiler-ctw-labsjdk-latest-darwin-aarch64": {},
+    "compiler-test-labsjdk-latest-darwin-amd64": {},
+    "compiler-test-labsjdk-latest-windows-amd64": {},
+
+    "compiler-test_zgc-labsjdk-latest-darwin-aarch64": {},
+    "compiler-test_zgc-labsjdk-latest-darwin-amd64": {},
+    "compiler-test_zgc-labsjdk-latest-linux-aarch64": {},
+    "compiler-test_zgc-labsjdk-latest-linux-amd64": {},
+
+    "compiler-ctw-labsjdk-latest-darwin-aarch64": {},
+    "compiler-ctw-labsjdk-latest-darwin-amd64": {},
+    "compiler-ctw-labsjdk-latest-linux-aarch64": {},
+    "compiler-ctw-labsjdk-latest-windows-amd64": {},
+
+    "compiler-ctw_zgc-labsjdk-latest-linux-amd64": {},
+
+    "compiler-ctw_economy-labsjdk-latest-darwin-aarch64": {},
+    "compiler-ctw_economy-labsjdk-latest-darwin-amd64": {},
+    "compiler-ctw_economy-labsjdk-latest-linux-aarch64": {},
+    "compiler-ctw_economy-labsjdk-latest-windows-amd64": {},
 
-    "daily-compiler-ctw_economy-labsjdk-latest-linux-aarch64": {},
-    "daily-compiler-ctw_economy-labsjdk-latest-darwin-amd64": {},
-    "daily-compiler-ctw_economy-labsjdk-latest-darwin-aarch64": {},
+    "compiler-bootstrap_lite-labsjdk-latest-darwin-amd64": {},
 
-    "daily-compiler-bootstrap_lite-labsjdk-latest-darwin-amd64": {},
+    "compiler-bootstrap_full-labsjdk-latest-linux-amd64": {},
+    "compiler-bootstrap_full_zgc-labsjdk-latest-linux-amd64": {},
 
-    "daily-compiler-bootstrap_full-labsjdk-latest-linux-amd64": s.many_cores,
-    "daily-compiler-bootstrap_full_zgc-labsjdk-latest-linux-amd64": s.many_cores
-  } + $.as_dailies(gate_jobs),
+    "compiler-benchmarktest_zgc-labsjdk-latest-linux-amd64": {},
+    "compiler-truffle_xcomp_zgc-labsjdk-latest-linux-amd64": {},
+  }
+  + $.as_dailies(gate_jobs)
+  + $.as_dailies(tier1_jobs)
+  + $.as_dailies(tier2_jobs)
+  + $.as_dailies(tier3_jobs),
 
   # This map defines the builders that run weekly. Each key in this map
   # must be the name of a build created by `make_build` (or be the prefix
   # of a build name if the key ends with "*").
   # Each value in this map is an object that overrides or extends the
   # fields of the denoted build.
   local weeklies = {
-    "weekly-compiler-ctw_phaseplan_fuzzing-labsjdk-latest-linux-amd64": {
+    "compiler-ctw_phaseplan_fuzzing-labsjdk-latest-linux-amd64": {
       notify_groups: [],
       notify_emails: ["[email protected]"],
       logs+: ["*/graal_dumps/*/*_failure.log"],
     },
 
-    "weekly-compiler-test_vec16-labsjdk-latest-linux-amd64": {},
-    "weekly-compiler-test_avx0-labsjdk-latest-linux-amd64": {},
-    "weekly-compiler-test_avx1-labsjdk-latest-linux-amd64": {},
+    "compiler-test_vec16-labsjdk-latest-linux-amd64": {},
+    "compiler-test_avx0-labsjdk-latest-linux-amd64": {},
+    "compiler-test_avx1-labsjdk-latest-linux-amd64": {},
 
-    "weekly-compiler-test_jtt_phaseplan_fuzzing-labsjdk-latest-linux-amd64": {
+    "compiler-test_jtt_phaseplan_fuzzing-labsjdk-latest-linux-amd64": {
       notify_groups: [],
       notify_emails: ["[email protected]"],
     },
 
-    "weekly-compiler-benchmarktest-labsjdk-latestDebug-linux-amd64": t("6:00:00"),
-    "weekly-compiler-benchmarktest-labsjdk-latestDebug-darwin-aarch64": t("6:00:00"),
+    "compiler-benchmarktest-labsjdk-latestDebug-linux-amd64": t("6:00:00"),
+    "compiler-benchmarktest-labsjdk-latestDebug-darwin-aarch64": t("6:00:00"),
 
-    "weekly-compiler-coverage*": {},
+    "compiler-coverage*": {},
 
-    "weekly-compiler-test_serialgc-labsjdk-latest-linux-amd64": {},
-    "weekly-compiler-test_serialgc-labsjdk-latest-linux-aarch64": {},
-    "weekly-compiler-test_serialgc-labsjdk-latest-darwin-amd64": {},
-    "weekly-compiler-test_serialgc-labsjdk-latest-darwin-aarch64": {},
+    "compiler-test_serialgc-labsjdk-latest-linux-amd64": {},
+    "compiler-test_serialgc-labsjdk-latest-linux-aarch64": {},
+    "compiler-test_serialgc-labsjdk-latest-darwin-amd64": {},
+    "compiler-test_serialgc-labsjdk-latest-darwin-aarch64": {},
 
-    "weekly-compiler-truffle_xcomp_serialgc-labsjdk-latest-linux-amd64": {},
-    "weekly-compiler-truffle_xcomp_serialgc-labsjdk-latest-linux-aarch64": {},
+    "compiler-truffle_xcomp_serialgc-labsjdk-latest-linux-amd64": {},
+    "compiler-truffle_xcomp_serialgc-labsjdk-latest-linux-aarch64": {},
   },
 
   # This map defines overrides and field extensions for monthly builds.
@@ -329,38 +369,46 @@
              include_common_os_arch=true,
              jdk_name="labsjdk",
              gates_manifest=gates,
+             tier1_manifest=tier1s,
+             tier2_manifest=tier2s,
+             tier3_manifest=tier3s,
              dailies_manifest=dailies,
              weeklies_manifest=weeklies,
              monthlies_manifest=monthlies):: {
-    local base_name = "%s-%s-%s-%s-%s" % [suite, task, jdk_name, if std.startsWith(jdk, "Latest") then "l" + jdk[1:] else jdk, os_arch],
-    local gate_name = "gate-" + base_name,
-    local daily_name = "daily-" + base_name,
-    local weekly_name = "weekly-" + base_name,
-    local monthly_name = "monthly-" + base_name,
-
-    local is_gate = $.manifest_match(gates_manifest, gate_name),
-    local is_daily = $.manifest_match(dailies_manifest, daily_name),
-    local is_monthly = $.manifest_match(monthlies_manifest, monthly_name),
-    local is_weekly = !is_gate && !is_daily && !is_monthly, # Default to weekly
+    local build_name = "%s-%s-%s-%s-%s" % [suite, task, jdk_name, if std.startsWith(jdk, "Latest") then "l" + jdk[1:] else jdk, os_arch],
+
+    local is_gate = $.manifest_match(gates_manifest, build_name),
+    local is_tier1 = $.manifest_match(tier1_manifest, build_name),
+    local is_tier2 = $.manifest_match(tier2_manifest, build_name),
+    local is_tier3 = $.manifest_match(tier3_manifest, build_name),
+    local is_daily = $.manifest_match(dailies_manifest, build_name),
+    local is_monthly = $.manifest_match(monthlies_manifest, build_name),
+    local is_weekly = !is_gate && !is_tier1 && !is_tier2 && !is_tier3 && !is_daily && !is_monthly, # Default to weekly
     local is_windows = utils.contains(os_arch, "windows"),
     local extra = if is_gate then
-        $.get(gates_manifest, gate_name, {})
+        $.get(gates_manifest, build_name, {})
+      else if is_tier1 then
+        $.get(tier1_manifest, build_name, {})
+      else if is_tier2 then
+        $.get(tier2_manifest, build_name, {})
+      else if is_tier3 then
+        $.get(tier3_manifest, build_name, {})
       else if is_daily then
-        $.get(dailies_manifest, daily_name, {})
+        $.get(dailies_manifest, build_name, {})
       else if is_weekly then
-        $.get(weeklies_manifest, weekly_name, {})
+        $.get(weeklies_manifest, build_name, {})
       else if is_monthly then
-        $.get(monthlies_manifest, monthly_name, {}),
+        $.get(monthlies_manifest, build_name, {}),
 
     build: {
-      name: if is_gate   then gate_name
-       else if is_daily  then daily_name
-       else if is_weekly then weekly_name
-       else                   monthly_name
+      name: build_name
     } +
       (s + extra_tasks)[task] +
       c["%s%s" % [jdk_name, jdk]] +
       (if include_common_os_arch then c[std.strReplace(os_arch, "-", "_")] else {}) +
+      (if is_tier1 then s.tier1 else {}) +
+      (if is_tier2 then s.tier2 else {}) +
+      (if is_tier3 then s.tier3 else {}) +
       (if is_daily then s.daily else {}) +
       (if is_weekly then s.weekly else {}) +
       (if is_monthly then s.monthly else {}) +
@@ -410,6 +458,8 @@
   local all_platforms_builds = [self.make_build(jdk, os_arch, task).build + galahad.exclude
     for task in [
       "test",
+      "unittest_compiler",
+      "unittest_truffle",
       "truffle_xcomp",
       "ctw",
       "ctw_economy",
@@ -455,6 +505,8 @@
       ]
       for task in [
         "test_zgc",
+        "unittest_compiler_zgc",
+        "unittest_truffle_zgc",
         "truffle_xcomp_zgc",
         "ctw_zgc",
         "benchmarktest_zgc",

compiler/ci/ci_includes/baseline-benchmarks.jsonnet

@@ -17,10 +17,11 @@
 
   local hotspot_profiling_builds = std.flattenArrays([
     [
-    c.monthly + hw.x52  + jdk + cc.c2 + suite + cc.enable_profiling   + { job_prefix:: "bench-compiler-profiling" },
-    c.monthly + hw.a12c + jdk + cc.c2 + suite + cc.enable_profiling   + { job_prefix:: "bench-compiler-profiling" },
-    c.monthly + hw.x52  + jdk + cc.c2 + suite + cc.footprint_tracking + { job_prefix:: "bench-compiler-footprint" },
-    c.monthly + hw.a12c + jdk + cc.c2 + suite + cc.footprint_tracking + { job_prefix:: "bench-compiler-footprint" }
+    c.monthly + hw.x52  + jdk + cc.c2 + suite + cc.enable_profiling    + { job_prefix:: "bench-compiler-profiling" },
+    c.monthly + hw.a12c + jdk + cc.c2 + suite + cc.enable_profiling    + { job_prefix:: "bench-compiler-profiling" },
+    c.monthly + hw.x52  + jdk + cc.c2 + suite + cc.footprint_tracking  + { job_prefix:: "bench-compiler-footprint" },
+    c.monthly + hw.a12c + jdk + cc.c2 + suite + cc.footprint_tracking  + { job_prefix:: "bench-compiler-footprint" },
+    c.monthly + hw.x52_root + jdk + cc.c2 + suite + cc.energy_tracking + { job_prefix:: "bench-compiler-energy" }
     ]
   for jdk in cc.product_jdks
   for suite in bench.groups.main_suites

compiler/ci/ci_includes/bootstrap_extra.libsonnet

@@ -5,8 +5,8 @@
 
   # See definition of `gate_jobs` local variable in ../ci_common/gate.jsonnet
   local gate_jobs = {
-    "gate-compiler-bootstrap-labsjdk-latest-linux-amd64": g.many_cores,
-    "gate-compiler-bootstrap_economy-labsjdk-latest-linux-amd64": g.many_cores,
+    "compiler-bootstrap-labsjdk-latest-linux-amd64": {},
+    "compiler-bootstrap_economy-labsjdk-latest-linux-amd64": {},
   },
   local gates = g.as_gates(gate_jobs),
   local dailies = g.as_dailies(gate_jobs),