[GR-65148] Deploy the new unchained standalones

Author: eregon

sdk/mx.sdk/mx_sdk_vm_ng.py

@@ -1077,6 +1077,8 @@ def __init__(self, suite, name=None, deps=None, excludedLibs=None, platformDepen
         }
         self.standalone_dir_dist = standalone_dir_dist
         maven = { 'groupId': 'org.graalvm', 'tag': 'standalone' }
+        assert theLicense is None, "the 'license' attribute is ignored for DeliverableStandaloneArchive"
+        theLicense = ['GFTC' if is_enterprise() else 'UPL']
         super().__init__(suite, name=dist_name, deps=[], layout=layout, path=None, theLicense=theLicense, platformDependent=True, path_substitutions=path_substitutions, string_substitutions=string_substitutions, maven=maven, defaultBuild=defaultBuild)
         self.buildDependencies.append(standalone_dir_dist)
 
@@ -1085,7 +1087,6 @@ def resolveDeps(self):
         resolved = [self.standalone_dir_dist]
         self._resolveDepsHelper(resolved)
         self.standalone_dir_dist = resolved[0]
-        self.theLicense = self.standalone_dir_dist.theLicense
 
     def get_artifact_metadata(self):
         return {'edition': 'ee' if is_enterprise() else 'ce', 'type': 'standalone', 'project': 'graal'}

vm/ci/ci_common/common.jsonnet

@@ -149,9 +149,13 @@ local devkits = graal_common.devkits;
   svm_common: graal_common.deps.svm,
   svm_common_windows_amd64(jdk): self.svm_common + graal_common.devkits["windows-jdk" + jdk],
 
-  maven_deploy_sdk:                     ['--suite', 'sdk', 'maven-deploy', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'],
+  maven_deploy_sdk:      ['--suite', 'sdk', 'maven-deploy', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'],
   deploy_artifacts_sdk(os, base_dist_name=null): (if base_dist_name != null then ['--base-dist-name=' + base_dist_name] else []) + ['--suite', 'sdk', 'deploy-artifacts', '--uploader', if os == 'windows' then 'artifact_uploader.cmd' else 'artifact_uploader'],
 
+  maven_deploy_all_suites: ['maven-deploy', '--all-suites', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'],
+  deploy_artifacts_all_suites(os): ['deploy-artifacts', '--all-suites', '--uploader', if os == 'windows' then 'artifact_uploader.cmd' else 'artifact_uploader'],
+
+  # All 3 used in vm.jsonnet
   maven_deploy_sdk_base:                    self.maven_deploy_sdk +                     ['--tags', 'graalvm', vm.binaries_repository],
   artifact_deploy_sdk_base(os, base_dist_name): self.deploy_artifacts_sdk(os, base_dist_name) + ['--tags', 'graalvm'],
   deploy_sdk_base(os, base_dist_name=null):     [self.mx_vm_common + vm.vm_profiles + self.maven_deploy_sdk_base, self.mx_vm_common + vm.vm_profiles + self.artifact_deploy_sdk_base(os, base_dist_name)],
@@ -160,16 +164,16 @@ local devkits = graal_common.devkits;
   artifact_deploy_sdk_base_dry_run(os, base_dist_name): self.deploy_artifacts_sdk(os, base_dist_name) + ['--tags', 'graalvm', '--dry-run'],
   deploy_sdk_base_dry_run(os, base_dist_name=null):     [self.mx_vm_common + vm.vm_profiles + self.maven_deploy_sdk_base_dry_run, self.mx_vm_common + vm.vm_profiles + self.artifact_deploy_sdk_base_dry_run(os, base_dist_name)],
 
-  deploy_sdk_components(os, tags): [
-    $.mx_vm_complete + self.maven_deploy_sdk + ['--tags', tags, vm.binaries_repository],
-    $.mx_vm_complete + self.deploy_artifacts_sdk(os) + ['--tags', tags]
+  deploy_standalones(os, tags): [
+    $.mx_vm_complete + self.maven_deploy_all_suites + ['--tags', tags, vm.binaries_repository],
+    $.mx_vm_complete + self.deploy_artifacts_all_suites(os) + ['--tags', tags]
   ],
 
-  maven_deploy_sdk_components_dry_run: self.maven_deploy_sdk + ['--tags', 'standalone', '--dry-run', vm.binaries_repository],
-  artifact_deploy_sdk_components_dry_run(os): self.deploy_artifacts_sdk(os) + ['--tags', 'standalone', '--dry-run'],
-  deploy_sdk_components_dry_run(os): [
-    $.mx_vm_complete + self.maven_deploy_sdk_components_dry_run,
-    $.mx_vm_complete + self.artifact_deploy_sdk_components_dry_run(os)
+  maven_deploy_standalones_dry_run: self.maven_deploy_all_suites + ['--tags', 'standalone', '--dry-run', vm.binaries_repository],
+  artifact_deploy_standalones_dry_run(os): self.deploy_artifacts_all_suites(os) + ['--tags', 'standalone', '--dry-run'],
+  deploy_standalones_dry_run(os): [
+    $.mx_vm_complete + self.maven_deploy_standalones_dry_run,
+    $.mx_vm_complete + self.artifact_deploy_standalones_dry_run(os)
   ],
 
   ruby_vm_build: self.svm_common + self.sulong + self.truffleruby + vm.custom_vm,
@@ -528,24 +532,19 @@ local devkits = graal_common.devkits;
     timelimit: "1:00:00"
   },
 
-  deploy_graalvm_components(java_version, standalones, record_file_sizes=false): vm.check_structure + {
-    build_deps:: std.join(',', []
-      + (if (record_file_sizes) then ['GRAALVM'] else [])
-      + (if (standalones) then ['GRAALVM_STANDALONES'] else [])
-    ),
+  deploy_graalvm_standalones(java_version, record_file_sizes=false): vm.check_structure + {
+    build_deps:: '{MAVEN_TAG_DISTRIBUTIONS:standalone}',
 
-    tags:: std.join(',', []
-      + (if (standalones) then ['standalone'] else [])
-    ),
+    tags:: 'standalone',
 
     run: $.patch_env(self.os, self.arch, java_version) + [
       $.mx_vm_complete + ['graalvm-show'],
       $.mx_vm_complete + ['build', '--dependencies', self.build_deps],
     ]
-    + $.deploy_sdk_components(self.os, self.tags)
+    + $.deploy_standalones(self.os, self.tags)
     + (
       if (record_file_sizes) then [
-        $.mx_vm_complete + $.record_file_sizes,
+        $.mx_vm_complete + $.record_file_sizes + ['--', 'standalones'],
         $.upload_file_sizes,
       ] else []
     ),
@@ -554,20 +553,21 @@ local devkits = graal_common.devkits;
   },
 
   #
-  # Deploy GraalVM Base and Standalones
+  # Deploy Truffle Languages Standalones
+  # `Deploy GraalVM Base` is done in common-runspec.jsonnet.
   # NOTE: After adding or removing deploy jobs, please make sure you modify ce-release-artifacts.json accordingly.
   #
 
   # Linux/AMD64
-  deploy_vm_standalones_javaLatest_linux_amd64: vm.vm_java_Latest + self.full_vm_build + self.linux_deploy + self.vm_base('linux', 'amd64', 'daily', deploy=true) + self.deploy_graalvm_components('latest', standalones=true, record_file_sizes=true) + {name: 'daily-deploy-vm-standalones-java-latest-linux-amd64', notify_groups:: ["deploy"]},
+  deploy_vm_standalones_javaLatest_linux_amd64: vm.vm_java_Latest + self.full_vm_build + self.linux_deploy + self.vm_base('linux', 'amd64', 'daily', deploy=true) + self.deploy_graalvm_standalones('latest', record_file_sizes=true) + {name: 'daily-deploy-vm-standalones-java-latest-linux-amd64', notify_groups:: ["deploy"]},
   # Linux/AARCH64
-  deploy_vm_standalones_javaLatest_linux_aarch64: vm.vm_java_Latest + self.full_vm_build + self.linux_deploy + self.vm_base('linux', 'aarch64', 'daily', deploy=true) + self.deploy_graalvm_components('latest', standalones=true) + {name: 'daily-deploy-vm-standalones-java-latest-linux-aarch64', notify_groups:: ["deploy"], capabilities+: ["!xgene3"]},
+  deploy_vm_standalones_javaLatest_linux_aarch64: vm.vm_java_Latest + self.full_vm_build + self.linux_deploy + self.vm_base('linux', 'aarch64', 'daily', deploy=true) + self.deploy_graalvm_standalones('latest') + {name: 'daily-deploy-vm-standalones-java-latest-linux-aarch64', notify_groups:: ["deploy"], capabilities+: ["!xgene3"]},
   # Darwin/AMD64
-  deploy_vm_standalones_javaLatest_darwin_amd64: vm.vm_java_Latest + self.full_vm_build + self.darwin_deploy + self.vm_base('darwin', 'amd64', 'daily', deploy=true, jdk_hint='Latest') + self.deploy_graalvm_components('latest', standalones=true) + {name: 'daily-deploy-vm-standalones-java-latest-darwin-amd64', capabilities+: ["!macmini_late_2014"], notify_groups:: ["deploy"], timelimit: '3:00:00'},
+  deploy_vm_standalones_javaLatest_darwin_amd64: vm.vm_java_Latest + self.full_vm_build + self.darwin_deploy + self.vm_base('darwin', 'amd64', 'daily', deploy=true, jdk_hint='Latest') + self.deploy_graalvm_standalones('latest') + {name: 'daily-deploy-vm-standalones-java-latest-darwin-amd64', capabilities+: ["!macmini_late_2014"], notify_groups:: ["deploy"], timelimit: '3:00:00'},
   # Darwin/AARCH64
-  deploy_vm_standalones_javaLatest_darwin_aarch64: vm.vm_java_Latest + self.full_vm_build + self.darwin_deploy + self.vm_base('darwin', 'aarch64', 'daily', deploy=true) + self.deploy_graalvm_components('latest', standalones=true) + {name: 'daily-deploy-vm-standalones-java-latest-darwin-aarch64', notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '3:00:00'},
+  deploy_vm_standalones_javaLatest_darwin_aarch64: vm.vm_java_Latest + self.full_vm_build + self.darwin_deploy + self.vm_base('darwin', 'aarch64', 'daily', deploy=true) + self.deploy_graalvm_standalones('latest') + {name: 'daily-deploy-vm-standalones-java-latest-darwin-aarch64', notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '3:00:00'},
   # Windows/AMD64
-  deploy_vm_standalones_javaLatest_windows_amd64: vm.vm_java_Latest + self.svm_common_windows_amd64('Latest') + self.js_windows_common + self.sulong + self.vm_base('windows', 'amd64', 'daily', deploy=true, jdk_hint='Latest') + self.deploy_graalvm_components('latest', standalones=true) + self.deploy_build + {name: 'daily-deploy-vm-standalones-java-latest-windows-amd64', timelimit: '2:30:00', notify_groups:: ["deploy"]},
+  deploy_vm_standalones_javaLatest_windows_amd64: vm.vm_java_Latest + self.svm_common_windows_amd64('Latest') + self.js_windows_common + self.sulong + self.vm_base('windows', 'amd64', 'daily', deploy=true, jdk_hint='Latest') + self.deploy_graalvm_standalones('latest') + self.deploy_build + {name: 'daily-deploy-vm-standalones-java-latest-windows-amd64', timelimit: '2:30:00', notify_groups:: ["deploy"]},
 
   local sulong_vm_tests = self.svm_common + self.sulong + vm.custom_vm + self.vm_base('linux', 'amd64', 'gate') + {
      run: [

vm/mx.vm/ce-aarch64-complete

@@ -1,4 +1,4 @@
+# Same as the ce-aarch64 env, except more dynamic imports and no NATIVE_IMAGES list, and no fastr
 DYNAMIC_IMPORTS=/compiler,/graal-js,/graal-nodejs,/regex,/sdk,/substratevm,/sulong,/tools,/truffle,/wasm,graalpython,truffleruby
-COMPONENTS=antlr4,cmp,cov,dap,gvm,gwa,icu4j,ins,insight,insightheap,js,jsl,jss,lg,llp,llrc,llrl,llrn,lsp,nfi-libffi,ni,nic,nil,njs,njsl,pro,pyn,pynl,rby,rbyl,rgx,sdk,sdkl,svm,svmjdwp,svmt,svml,svmnfi,svmsl,tfl,tfla,tflc,tflm,truffle-json,vvm
-NATIVE_IMAGES=lib:pythonvm,graalvm-native-binutil,graalvm-native-clang,graalvm-native-clang-cl,graalvm-native-clang++,graalvm-native-flang,graalvm-native-ld,lib:jsvm,lib:graal-nodejs,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent,lib:llvmvm,native-image,lib:rubyvm,lib:wasmvm
-DISABLE_INSTALLABLES=False
+COMPONENTS=cmp,gvm,lg,ni,nic,nil,sdkni,svm,svmjdwp,svmt,svmsl,tflc,tflsm
+NON_REBUILDABLE_IMAGES=lib:jvmcicompiler

vm/mx.vm/ce-complete

@@ -1,4 +1,4 @@
+# Same as the ce env, except more dynamic imports and no NATIVE_IMAGES list
 DYNAMIC_IMPORTS=/compiler,/graal-js,/graal-nodejs,/regex,/sdk,/substratevm,/sulong,/tools,/truffle,/wasm,fastr,graalpython,truffleruby
-COMPONENTS=antlr4,cmp,cov,dap,gvm,gwa,icu4j,xz,ins,insight,insightheap,js,jsl,jss,lg,llp,llrc,llrl,llrn,lsp,nfi-libffi,nfi,ni,nic,nil,njs,njsl,poly,pro,pyn,pynl,R,rby,rbyl,rgx,sdk,sdkl,svm,svmjdwp,svmt,svml,svmnfi,svmsl,tfl,tfla,tflc,tflm,truffle-json,vvm
-NATIVE_IMAGES=lib:pythonvm,graalvm-native-binutil,graalvm-native-clang,graalvm-native-clang-cl,graalvm-native-clang++,graalvm-native-flang,graalvm-native-ld,lib:jsvm,lib:graal-nodejs,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent,lib:llvmvm,native-image,lib:rubyvm,lib:wasmvm
-DISABLE_INSTALLABLES=false
+COMPONENTS=cmp,gvm,lg,ni,nic,nil,sdkni,svm,svmjdwp,svmt,svmsl,tflc,tflsm
+NON_REBUILDABLE_IMAGES=lib:jvmcicompiler

vm/mx.vm/ce-darwin-aarch64-complete

@@ -1,4 +1,4 @@
+# Same as the ce-darwin-aarch64 env, except more dynamic imports and no NATIVE_IMAGES list, and no fastr
 DYNAMIC_IMPORTS=/compiler,/graal-js,/graal-nodejs,/regex,/sdk,/substratevm,/sulong,/tools,/truffle,/wasm,graalpython,truffleruby
-COMPONENTS=antlr4,cmp,cov,dap,gvm,gwa,icu4j,ins,insight,insightheap,js,jsl,jss,lg,llp,llrc,llrl,llrn,lsp,nfi-libffi,nfi,ni,nic,nil,njs,njsl,pro,pyn,pynl,rby,rbyl,rgx,sdk,sdkl,svm,svmjdwp,svmt,svmnfi,svmsl,tfl,tfla,tflc,tflm,truffle-json,vvm
-NATIVE_IMAGES=lib:pythonvm,graalvm-native-binutil,graalvm-native-clang,graalvm-native-clang-cl,graalvm-native-clang++,graalvm-native-flang,graalvm-native-ld,lib:jsvm,lib:graal-nodejs,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent,lib:llvmvm,native-image,lib:rubyvm,lib:wasmvm
-DISABLE_INSTALLABLES=false
+COMPONENTS=cmp,gvm,lg,ni,nic,nil,sdkni,svm,svmjdwp,svmt,svmsl,tflc,tflsm
+NON_REBUILDABLE_IMAGES=lib:jvmcicompiler

vm/mx.vm/ce-darwin-complete

@@ -1,4 +1,4 @@
+# Same as the ce-darwin env, except more dynamic imports and no NATIVE_IMAGES list, and no fastr
 DYNAMIC_IMPORTS=/compiler,/graal-js,/graal-nodejs,/regex,/sdk,/substratevm,/sulong,/tools,/truffle,/wasm,graalpython,truffleruby
-COMPONENTS=antlr4,cmp,cov,dap,gvm,gwa,icu4j,xz,ins,insight,insightheap,js,jsl,jss,lg,llp,llrc,llrl,llrn,lsp,nfi-libffi,nfi,ni,nic,nil,njs,njsl,poly,pro,pyn,pynl,rby,rbyl,rgx,sdk,sdkl,svm,svmjdwp,svmt,svml,svmnfi,svmsl,tfl,tfla,tflc,tflm,truffle-json,vvm
-NATIVE_IMAGES=lib:pythonvm,graalvm-native-binutil,graalvm-native-clang,graalvm-native-clang-cl,graalvm-native-clang++,graalvm-native-flang,graalvm-native-ld,lib:jsvm,lib:graal-nodejs,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent,lib:llvmvm,native-image,lib:rubyvm,lib:wasmvm
-DISABLE_INSTALLABLES=false
+COMPONENTS=cmp,gvm,lg,ni,nic,nil,sdkni,svm,svmjdwp,svmt,svmsl,tflc,tflsm
+NON_REBUILDABLE_IMAGES=lib:jvmcicompiler

vm/mx.vm/ce-win-complete

@@ -1,4 +1,4 @@
+# Same as the ce-win env, except more dynamic imports and no NATIVE_IMAGES list, and no truffleruby,fastr
 DYNAMIC_IMPORTS=/compiler,/graal-js,/graal-nodejs,/regex,/sdk,/substratevm,/sulong,/tools,/truffle,/wasm,graalpython
-COMPONENTS=antlr4,cmp,cov,dap,gvm,gwa,icu4j,ins,insight,insightheap,js,jsl,jss,lg,llp,llrc,llrl,llrn,lsp,nfi-libffi,nfi,ni,nic,nil,njs,njsl,pro,pyn,pynl,rgx,sdk,sdkl,svm,svmjdwp,svmt,svmnfi,svmsl,tfl,tfla,tflc,tflm,truffle-json,vvm
-NATIVE_IMAGES=lib:pythonvm,graalvm-native-binutil,graalvm-native-clang,graalvm-native-clang-cl,graalvm-native-clang++,graalvm-native-flang,graalvm-native-ld,lib:jsvm,lib:graal-nodejs,lib:jvmcicompiler,lib:native-image-agent,lib:native-image-diagnostics-agent,lib:llvmvm,native-image,lib:wasmvm
-DISABLE_INSTALLABLES=False
+COMPONENTS=cmp,gvm,lg,ni,nic,nil,sdkni,svm,svmjdwp,svmt,svmsl,tflc,tflsm
+NON_REBUILDABLE_IMAGES=lib:jvmcicompiler

vm/mx.vm/mx_vm_benchmark.py

@@ -34,6 +34,7 @@
 import mx_sdk_benchmark
 import mx_sdk_vm
 import mx_sdk_vm_impl
+import mx_sdk_vm_ng
 from mx_benchmark import DataPoint, DataPoints
 from mx_sdk_benchmark import GraalVm, NativeImageVM
 
@@ -344,7 +345,6 @@ def get_vm_registry(self):
 
     def runAndReturnStdOut(self, benchmarks, bmSuiteArgs):
         vm = self.get_vm_registry().get_vm_from_suite_args(bmSuiteArgs)
-        vm.extract_vm_info(self.vmArgs(bmSuiteArgs))
         host_vm = None
         if isinstance(vm, mx_benchmark.GuestVm):
             host_vm = vm.host_vm()
@@ -359,21 +359,34 @@ def runAndReturnStdOut(self, benchmarks, bmSuiteArgs):
             "guest-vm-config": self.guest_vm_config_name(host_vm, vm),
         }
 
+        def get_size_message(image_name, image_location):
+            return FileSizeBenchmarkSuite.SZ_MSG_PATTERN.format(image_name, getsize(image_location), image_location)
+
+        bmSuiteArgs = bmSuiteArgs or ['base']
         out = ""
-        output_root = mx_sdk_vm_impl.get_final_graalvm_distribution().get_output_root()
 
-        def get_size_message(image_name, image_location):
-            return FileSizeBenchmarkSuite.SZ_MSG_PATTERN.format(image_name, getsize(os.path.join(output_root, image_location)), image_location, output_root)
-
-        for location in mx_sdk_vm_impl.get_all_native_image_locations(include_libraries=True, include_launchers=False, abs_path=False):
-            lib_name = 'lib:' + mx_sdk_vm_impl.remove_lib_prefix_suffix(basename(location))
-            out += get_size_message(lib_name, location)
-        for location in mx_sdk_vm_impl.get_all_native_image_locations(include_libraries=False, include_launchers=True, abs_path=False):
-            launcher_name = mx_sdk_vm_impl.remove_exe_suffix(basename(location))
-            out += get_size_message(launcher_name, location)
+        for arg in bmSuiteArgs:
+            if arg == 'standalones':
+                # Standalones
+                for project in mx.projects():
+                    if isinstance(project, mx_sdk_vm_ng.NativeImageProject):
+                        native_image = project.output_file()
+                        if os.path.exists(native_image):
+                            out += get_size_message(project.options_file_name(), native_image)
+            elif arg == 'base':
+                # GraalVM base
+                output_root = mx_sdk_vm_impl.get_final_graalvm_distribution().get_output_root()
+                for location in mx_sdk_vm_impl.get_all_native_image_locations(include_libraries=True, include_launchers=False, abs_path=False):
+                    lib_name = 'lib:' + mx_sdk_vm_impl.remove_lib_prefix_suffix(basename(location))
+                    out += get_size_message(lib_name, os.path.join(output_root, location))
+                for location in mx_sdk_vm_impl.get_all_native_image_locations(include_libraries=False, include_launchers=True, abs_path=False):
+                    launcher_name = mx_sdk_vm_impl.remove_exe_suffix(basename(location))
+                    out += get_size_message(launcher_name, os.path.join(output_root, location))
+            else:
+                mx.abort("FileSizeBenchmarkSuite expects 'base' or 'standalones' in bench suite arguments but got " + arg)
+
         if out:
             mx.log(out, end='')
-        dims.update(vm.dimensions(output_root, bmSuiteArgs, 0, out))
         return 0, out, dims
 
     def rules(self, output, benchmarks, bmSuiteArgs):