[GR-63547] Adopt the new unchained standalones for Sulong

PullRequest: graal/20381

Author: eregon

sdk/mx.sdk/mx_sdk_vm.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -263,7 +263,8 @@ def __init__(self,
                  extra_installable_qualifiers=None,
                  has_relative_home=True,
                  jvm_configs=None,
-                 extra_native_targets=None):
+                 extra_native_targets=None,
+                 final_stage_only=False):
         """
         :param suite mx.Suite: the suite this component belongs to
         :type name: str
@@ -281,6 +282,7 @@ def __init__(self,
                 'priority': -1,  # 0 is invalid; < 0 prepends to the default configs; > 0 appends
             }
         :param extra_native_targets: list of str, enables extra targets in multi-target projects.
+        :param final_stage_only: bool, this component should be only included in the final GraalVM, not in stage1
         :type license_files: list[str]
         :type third_party_license_files: list[str]
         :type polyglot_lib_build_args: list[str]
@@ -305,6 +307,7 @@ def __init__(self,
         :type extra_installable_qualifiers: list[str] | None
         :type has_relative_home: bool
         :type jvm_configs: list[dict] or None
+        :type final_stage_only: bool
         """
         if dependencies is None:
             mx.logv('Component {} does not specify dependencies'.format(name))
@@ -343,6 +346,7 @@ def __init__(self,
         self.has_relative_home = has_relative_home
         self.jvm_configs = jvm_configs or []
         self.extra_native_targets = extra_native_targets
+        self.final_stage_only = final_stage_only
 
         if supported is not None or early_adopter:
             if stability is not None:

sdk/mx.sdk/mx_sdk_vm_impl.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2019, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -166,6 +166,8 @@ def add_dependencies(dependencies, excludes=True):
             components = dependencies[:]
             while components:
                 component = components.pop(0)
+                if component.final_stage_only and stage1:
+                    continue
                 if component not in components_to_build and not (excludes and is_excluded(component)):
                     components_to_build.append(component)
                     components.extend(component.direct_dependencies())
@@ -495,11 +497,7 @@ def _add_link(_dest, _target, _component=None, _dest_base_name=None):
             if _linkname != dest_base_name:
                 if mx.is_windows():
                     if _target.endswith('.exe') or _target.endswith('.cmd'):
-                        link_template_name = join(_suite.mxDir, 'vm', 'exe_link_template.cmd')
-                        with open(link_template_name, 'r') as template:
-                            _template_subst = mx_subst.SubstitutionEngine(mx_subst.string_substitutions)
-                            _template_subst.register_no_arg('target', normpath(_linkname))
-                            contents = _template_subst.substitute(template.read())
+                        contents = mx_sdk_vm_ng._make_windows_link(_linkname)
                         full_dest = _dest + dest_base_name[:-len('.exe')] + '.cmd'
                         _add(layout, full_dest, 'string:{}'.format(contents), _component)
                         return full_dest
@@ -2753,11 +2751,7 @@ def add_files_from_component(comp, path_prefix, excluded_paths):
                             link_target = relpath(launcher_dest, start=dirname(link_dest))
                             if mx.is_windows():
                                 if link_target.endswith('.exe') or link_target.endswith('.cmd'):
-                                    link_template_name = join(_suite.mxDir, 'vm', 'exe_link_template.cmd')
-                                    with open(link_template_name, 'r') as template:
-                                        _template_subst = mx_subst.SubstitutionEngine(mx_subst.string_substitutions)
-                                        _template_subst.register_no_arg('target', normpath(link_target))
-                                        contents = _template_subst.substitute(template.read())
+                                    contents = mx_sdk_vm_ng._make_windows_link(link_target)
                                     full_dest = link_dest[:-len('.exe')] + '.cmd'
                                     layout.setdefault(full_dest, []).append({
                                         'source_type': 'string',

sdk/mx.sdk/mx_sdk_vm_ng.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -44,7 +44,7 @@
 import sys
 from abc import ABCMeta, abstractmethod
 from os import listdir, linesep
-from os.path import join, exists, isfile, basename, relpath, isdir, isabs, dirname
+from os.path import join, exists, isfile, basename, relpath, isdir, isabs, dirname, normpath
 from typing import Tuple
 
 import mx
@@ -381,6 +381,9 @@ def __init__(self, args, project: NativeImageProject):
             max_parallelism = 12
         super().__init__(project, args, min(max_parallelism, mx.cpu_count()))
 
+    def newestOutput(self):
+        return mx.TimeStampFile.newest([_path for _path, _ in self.subject.getArchivableResults()])
+
     def get_build_args(self):
         experimental_build_args = [
             '-H:+GenerateBuildArtifactsFile',  # generate 'build-artifacts.json'
@@ -931,14 +934,57 @@ def witness_contents(self):
         return f"roots: {', '.join(self.subject.root_components)}\nignored: {', '.join(self.subject.ignore_components)}"
 
 
+def _make_windows_link(link_target):
+    link_template_name = join(_suite.mxDir, 'vm', 'exe_link_template.cmd')
+    with open(link_template_name, 'r') as template:
+        _template_subst = mx_subst.SubstitutionEngine(mx_subst.string_substitutions)
+        _template_subst.register_no_arg('target', normpath(link_target))
+        return _template_subst.substitute(template.read())
+
+
+class ToolchainToolDistribution(mx.LayoutDirDistribution):
+    def __init__(self, suite, name=None, deps=None, excludedLibs=None, platformDependent=True, theLicense=None, defaultBuild=True, **kw_args):
+        self.tool_project = _require(kw_args, 'tool_project', suite, name)
+        self.tool_links = _require(kw_args, 'tool_links', suite, name)
+
+        layout = {
+            './': [{
+                "source_type": "dependency",
+                "dependency": self.tool_project,
+            }]
+        }
+
+        super().__init__(suite, name=name, deps=[], layout=layout, path=None, theLicense=theLicense, platformDependent=True, defaultBuild=defaultBuild)
+
+    def resolveDeps(self):
+        self.tool_project = mx.project(self.tool_project)
+        _, main_tool_name = next(self.tool_project.getArchivableResults(single=True))
+
+        def _add_link(name, target):
+            if mx.is_windows():
+                # ignore indirect symlinks on windows and link everything directly to the main tool
+                # otherwise we lose the original program name
+                self.layout[f'./{name}.cmd'] = f'string:{_make_windows_link(main_tool_name)}'
+            else:
+                self.layout[f'./{name}'] = f'link:{target}'
+
+        for tool in self.tool_links:
+            _add_link(tool, main_tool_name)
+            alt_names = self.tool_links[tool]
+            for alt_name in alt_names:
+                _add_link(alt_name, tool)
+
+        super().resolveDeps()
+
+
 if mx.is_windows():
     DeliverableArchiveSuper = mx.LayoutZIPDistribution
 else:
     DeliverableArchiveSuper = mx.LayoutTARDistribution
 
 
 class DeliverableStandaloneArchive(DeliverableArchiveSuper):
-    def __init__(self, suite, name=None, deps=None, excludedLibs=None, platformDependent=True, theLicense=None, **kw_args):
+    def __init__(self, suite, name=None, deps=None, excludedLibs=None, platformDependent=True, theLicense=None, defaultBuild=True, **kw_args):
         standalone_dir_dist = _require(kw_args, 'standalone_dist', suite, name)
         community_archive_name = _require(kw_args, 'community_archive_name', suite, name)
         enterprise_archive_name = _require(kw_args, 'enterprise_archive_name', suite, name)
@@ -959,11 +1005,16 @@ def __init__(self, suite, name=None, deps=None, excludedLibs=None, platformDepen
             dist_name = 'STANDALONE_' + community_archive_name.upper().replace('-', '_')
 
         layout = {
-            f'{dir_name}/': f'dependency:{standalone_dir_dist}/*'
+            f'{dir_name}/': {
+                "source_type": "dependency",
+                "dependency": standalone_dir_dist,
+                "path": "*",
+                "dereference": "never",
+            }
         }
         self.standalone_dir_dist = standalone_dir_dist
         maven = { 'groupId': 'org.graalvm', 'tag': 'standalone' }
-        super().__init__(suite, name=dist_name, deps=[], layout=layout, path=None, theLicense=theLicense, platformDependent=True, path_substitutions=path_substitutions, string_substitutions=string_substitutions, maven=maven)
+        super().__init__(suite, name=dist_name, deps=[], layout=layout, path=None, theLicense=theLicense, platformDependent=True, path_substitutions=path_substitutions, string_substitutions=string_substitutions, maven=maven, defaultBuild=defaultBuild)
         self.buildDependencies.append(standalone_dir_dist)
 
     def resolveDeps(self):

sulong/ci/ci.jsonnet

@@ -39,6 +39,8 @@ local sc = (import "ci_common/sulong-common.jsonnet");
         "<graal>/regex/**",
         "<graal>/java-benchmarks/**",
       ] + (if standalone then [
+        # tools suite (included in standalone)
+        "<graal>/tools/**",
         # substratevm and its dependencies
         "<graal>/substratevm/**",
         # vm and its dependencies
@@ -56,21 +58,21 @@ local sc = (import "ci_common/sulong-common.jsonnet");
     run+: [
       ["mx", "build", "--dependencies", "SULONG_TEST"],
       ["mx", "unittest", "--verbose", "-Dsulongtest.toolchainPathPattern=SULONG_BOOTSTRAP_TOOLCHAIN", "ToolchainAPITest"],
-      ["mx", "--env", "toolchain-only", "build"],
-      ["set-export", "SULONG_BOOTSTRAP_GRAALVM", ["mx", "--quiet", "--no-warning", "--env", "toolchain-only", "graalvm-home"]],
-      ["mx", "unittest", "--verbose", "-Dsulongtest.toolchainPathPattern=GRAALVM_TOOLCHAIN_ONLY", "ToolchainAPITest"],
+      ["mx", "--env", "ce-llvm-standalones", "build", "--dependencies", "SULONG_JVM_STANDALONE"],
+      ["set-export", "SULONG_BOOTSTRAP_STANDALONE", ["mx", "--quiet", "--no-warning", "--env", "ce-llvm-standalones", "path", "--output", "SULONG_JVM_STANDALONE"]],
+      ["mx", "unittest", "--verbose", "-Dsulongtest.toolchainPathPattern=SULONG_JVM_STANDALONE", "ToolchainAPITest"],
     ],
   },
 
   regular_builds:: [
     $.sulong + $.gate(style=true) + sc.labsjdkLatest + sc.linux_amd64 + sc.style + { name: "gate-sulong-style-fullbuild-jdk-latest-linux-amd64" },
-    $.sulong + $.gate(standalone=true) + sc.labsjdkLatest + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.requireGCC + sc.gateTags("build,sulongMisc,parser") + $.sulong_test_toolchain + { name: "gate-sulong-misc-parser-jdk-latest-linux-amd64" },
+    $.sulong + $.gate(standalone=true) + sc.labsjdkLatest + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.gateTags("build,sulongMisc,parser") + $.sulong_test_toolchain + { name: "gate-sulong-misc-parser-jdk-latest-linux-amd64" },
     $.sulong + $.gate() + sc.labsjdkLatest + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.gateTags("build,gcc_c") + { name: "gate-sulong-gcc_c-jdk-latest-linux-amd64", timelimit: "45:00" },
     $.sulong + $.gate() + sc.labsjdkLatest + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.gateTags("build,gcc_cpp") + { name: "gate-sulong-gcc_cpp-jdk-latest-linux-amd64", timelimit: "45:00" },
 
     $.sulong + $.gate() + sc.labsjdkLatest + sc.darwin_amd64 + sc.llvmBundled + sc.gateTags(basicTags) + { name: "gate-sulong-basic-nwcc-llvm-jdk-latest-darwin-amd64", timelimit: "0:45:00", capabilities+: ["ram16gb"] },
 
-    $.sulong + $.gate() + sc.labsjdkLatest + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.requireGCC + sc.gateTags(basicTags) + { name: "gate-sulong-basic-nwcc-llvm-jdk-latest-linux-amd64" },
+    $.sulong + $.gate() + sc.labsjdkLatest + sc.linux_amd64 + sc.llvmBundled + sc.requireGMP + sc.gateTags(basicTags) + { name: "gate-sulong-basic-nwcc-llvm-jdk-latest-linux-amd64" },
 
     $.sulong + $.gate() + sc.labsjdkLatest + sc.linux_aarch64 + sc.llvmBundled + sc.requireGMP + sc.gateTags(basicTagsNoNWCC) + { name: "gate-sulong-basic-llvm-jdk-latest-linux-aarch64", timelimit: "30:00" },
 

sulong/ci/ci_common/sulong-common.jsonnet

@@ -158,7 +158,7 @@ local sulong_deps = common.deps.sulong;
     extra_gate_args+:: ["--strict-mode"],
   },
 
-  coverage(builds):: $.llvmBundled + $.requireGMP + $.optionalGCC + $.mxGate + {
+  coverage(builds):: $.llvmBundled + $.requireGMP + $.mxGate + {
       local sameArchBuilds = std.filter(function(b) b.os == self.os && b.arch == self.arch, builds),
       local allTags = std.set(std.flattenArrays([b.gateTags for b in sameArchBuilds if std.objectHasAll(b, "gateTags")])),
       local coverageTags = std.setDiff(allTags, ["build", "build-all", "fullbuild", "style"]),
@@ -184,17 +184,6 @@ local sulong_deps = common.deps.sulong;
 
   llvmBundled:: {},
 
-  requireGCC:: {
-    packages+: {
-      gcc: "==6.1.0",
-    },
-  },
-
-  # like requireGCC, but only on linux/amd64, ignored otherwise
-  optionalGCC:: {
-    packages+: if self.os == "linux" && self.arch == "amd64" then $.requireGCC.packages else {},
-  },
-
   requireGMP:: {
     packages+: if self.os == "darwin" && self.arch == "aarch64" then {
         libgmp: "==6.2.1",

sulong/docs/contributor/TOOLCHAIN.md

@@ -159,13 +159,13 @@ On the implementation side, _the toolchain_ consists of multiple ingredients:
   The goal is to produce a GraalVM LLVM runtime executable result by simply pointing any build system to those wrappers,
   for example via `CC`/`CXX` environment variables or by setting `PATH`.
 
-## Using a Prebuilt GraalVM as a Bootstrapping Toolchain
+## Using a Prebuilt standalone as a Bootstrapping Toolchain
 
-To speed up toolchain compilation during development, the `SULONG_BOOTSTRAP_GRAALVM` environment variable can be set
-to a _prebuilt_ GraalVM. Sulong comes with a configuration file that makes building a bootstrapping GraalVM easy:
+To speed up toolchain compilation during development, the `SULONG_BOOTSTRAP_STANDALONE` environment variable can be set
+to a _prebuilt_ standalone.
 
 ```bash
-$ mx --env toolchain-only build
-$ export SULONG_BOOTSTRAP_GRAALVM=`mx --env toolchain-only graalvm-home`
+$ mx --env ce-llvm-standalones build --dependencies SULONG_JVM_STANDALONE
+$ export SULONG_BOOTSTRAP_STANDALONE=`mx --env ce-llvm-standalones path --output SULONG_JVM_STANDALONE`
 ```
 > **WARNING**: *The bootstrapping GraalVM will not be rebuilt automatically. You are responsible for keeping it up-to-date.*

sulong/mx.sulong/ce-llvm-standalones

@@ -1,3 +1,5 @@
-DYNAMIC_IMPORTS=/substratevm,/vm
-COMPONENTS=llp,llrl,llrlf,llrn,lg,svmnfi,tflm,tflsm
-BUILD_TARGETS=GRAALVM_STANDALONES
+DYNAMIC_IMPORTS=/tools,/compiler,/substratevm
+COMPONENTS=SubstrateVM,Truffle SVM Macro,LibGraal
+NATIVE_IMAGES=true
+BUILD_TARGETS=SULONG_NATIVE_STANDALONE,SULONG_JVM_STANDALONE
+GRAALVM_SKIP_ARCHIVE=true

sulong/mx.sulong/mx_sulong.py

@@ -47,6 +47,7 @@
 import mx_sulong_gate
 import mx_sulong_unittest #pylint: disable=unused-import
 import mx_sulong_llvm_config
+import mx_truffle
 
 # re-export custom mx project classes so they can be used from suite.py
 from mx_cmake import CMakeNinjaProject #pylint: disable=unused-import
@@ -58,6 +59,7 @@
 from mx_sulong_suite_constituents import DocumentationProject #pylint: disable=unused-import
 from mx_sulong_suite_constituents import HeaderProject #pylint: disable=unused-import
 from mx_sulong_suite_constituents import CopiedNativeProject #pylint: disable=unused-import
+from mx_sdk_vm_ng import StandaloneLicenses, ThinLauncherProject, NativeImageLibraryProject, NativeImageExecutableProject, LanguageLibraryProject, DynamicPOMDistribution, DeliverableStandaloneArchive, ToolchainToolDistribution  # pylint: disable=unused-import
 
 if sys.version_info[0] < 3:
     def _decode(x):
@@ -102,6 +104,36 @@ def sulong_prefix_path(name):
 
 mx_subst.results_substitutions.register_with_arg('sulong_prefix', sulong_prefix_path)
 
+# Functions called from suite.py
+
+def has_suite(name):
+    return mx.suite(name, fatalIfMissing=False)
+
+def is_ee():
+    return has_suite('sulong-managed')
+
+def sulong_standalone_deps():
+    deps = mx_truffle.resolve_truffle_dist_names()
+    if is_ee():
+        # SULONG_ENTERPRISE and SULONG_MANAGED do not belong in the EE standalone of SULONG_NATIVE, but we want a single definition of libllvmvm.
+        # So we compromise here by including them. We do not use or distribute the EE standalone of SULONG_NATIVE so it does not matter.
+        # See also the comments in suite.py, in SULONG_*_STANDALONE_RELEASE_ARCHIVE.
+        deps += [
+            'sulong-managed:SULONG_ENTERPRISE',
+            'sulong-managed:SULONG_MANAGED',
+            'sulong-managed:SULONG_ENTERPRISE_NATIVE',
+        ]
+    return deps
+
+def libllvmvm_build_args():
+    if is_ee() and not mx.is_windows():
+        return [
+            '-H:+AuxiliaryEngineCache',
+            '-H:ReservedAuxiliaryImageBytes=2145482548',
+        ]
+    else:
+        return []
+
 def testLLVMImage(image, imageArgs=None, testFilter=None, libPath=True, test=None, unittestArgs=None):
     mx_sulong_gate.testLLVMImage(image, imageArgs, testFilter, libPath, test, unittestArgs)
 
@@ -281,7 +313,11 @@ def get_lli_path(fatalIfMissing=True):
             useJvm = False
         else:
             mx.abort(f"Unknown standalone type {standaloneMode}.")
-        path = mx_sdk_vm_impl.standalone_home("llvm", useJvm)
+        if is_ee():
+            dist = "SULONG_MANAGED_JVM_STANDALONE" if useJvm else "SULONG_MANAGED_NATIVE_STANDALONE"
+        else:
+            dist = "SULONG_JVM_STANDALONE" if useJvm else "SULONG_NATIVE_STANDALONE"
+        path = mx.distribution(dist).output
         return os.path.join(path, 'bin', mx_subst.path_substitutions.substitute('<exe:lli>'))
 
 
@@ -387,13 +423,9 @@ def _get_toolchain_tool(name_tool):
 
 def create_toolchain_root_provider(name, dist):
     def provider():
-        bootstrap_graalvm = mx.get_env('SULONG_BOOTSTRAP_GRAALVM')
-        if bootstrap_graalvm:
-            ret = os.path.join(bootstrap_graalvm, 'jre', 'languages', 'llvm', name)
-            if os.path.exists(ret): # jdk8 based graalvm
-                return ret
-            else: # jdk11+ based graalvm
-                return os.path.join(bootstrap_graalvm, 'languages', 'llvm', name)
+        bootstrap_standalone = mx.get_env('SULONG_BOOTSTRAP_STANDALONE')
+        if bootstrap_standalone:
+            return os.path.join(bootstrap_standalone, 'lib', 'sulong', name)
         return mx.distribution(dist).get_output()
     return provider
 
@@ -408,7 +440,8 @@ def _lib_sub(program):
     return mx_subst.path_substitutions.substitute("<lib:{}>".format(program))
 
 class ToolchainConfig(object):
-    # Please keep this list in sync with Toolchain.java (method documentation) and ToolchainImpl.java (lookup switch block).
+    # Please keep this list in sync with Toolchain.java (method documentation) and ToolchainImpl.java (lookup switch block)
+    # and NativeToolchainWrapper.
     _llvm_tool_map = ["ar", "nm", "objcopy", "objdump", "ranlib", "readelf", "readobj", "strip"]
     _tool_map = {
         "CC": ["graalvm-{name}-clang", "graalvm-clang", "clang", "cc", "gcc"],