From d5d9ae249672659c8bdd2adb55ea1221b2e3326c Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 26 May 2025 18:34:14 +0200 Subject: [PATCH 1/5] Remove vm_common.{sulong,truffleruby,graalpy,wasm,svm_common} * And use common.deps instead. * Also removed ruby_vm_build and ruby_python_vm_build which are unused. --- vm/ci/ci_common/common-bench.jsonnet | 10 +++++----- vm/ci/ci_common/common.jsonnet | 25 +++++-------------------- vm/ci/ci_common/libgraal.jsonnet | 2 +- vm/ci/ci_includes/vm-native.jsonnet | 8 ++++---- vm/ci/ci_includes/vm.jsonnet | 2 +- 5 files changed, 16 insertions(+), 31 deletions(-) diff --git a/vm/ci/ci_common/common-bench.jsonnet b/vm/ci/ci_common/common-bench.jsonnet index 079a6c364017..656bf7ffff4d 100644 --- a/vm/ci/ci_common/common-bench.jsonnet +++ b/vm/ci/ci_common/common-bench.jsonnet @@ -23,7 +23,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet'; vm_bench_common: self.vm_bench_base(machine_name='x52') + { capabilities+: ['tmpfs25g'] }, - vm_bench_js_linux_amd64(bench_suite=null): vm.vm_java_Latest + vm_common.svm_common + vm_common.sulong + vm.custom_vm + self.vm_bench_common + { + vm_bench_js_linux_amd64(bench_suite=null): vm.vm_java_Latest + common.deps.svm + common.deps.sulong + vm.custom_vm + self.vm_bench_common + { cmd_base:: vm_common.mx_vm_common + ['--dynamicimports', 'js-benchmarks', 'benchmark', '--results-file', self.result_file], config_base:: ['--js-vm=graal-js', '--js-vm-config=default', '--jvm=graalvm-${VM_ENV}'], setup+: [ @@ -59,7 +59,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet'; base_cmd:: ['mx', '--env', env, '--dy', 'polybenchmarks'], }, - vm_bench_polybenchmarks_linux_build: vm_common.svm_common + vm_common.truffleruby + vm_common.graalpy + vm.custom_vm + vm.vm_java_Latest + self.polybench_hpc_linux_common(shape='e4_36_256') + self.vm_bench_polybenchmarks_base(env='polybench-${VM_ENV}') + { + vm_bench_polybenchmarks_linux_build: common.deps.svm + common.deps.truffleruby + common.deps.graalpy + vm.custom_vm + vm.vm_java_Latest + self.polybench_hpc_linux_common(shape='e4_36_256') + self.vm_bench_polybenchmarks_base(env='polybench-${VM_ENV}') + { setup+: [ self.base_cmd + ['sforceimports'], ], @@ -80,7 +80,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet'; }, # TODO (GR-60584): re-enable espresso polybench jobs once polybench is unchained - vm_bench_polybenchmarks_linux_common(vm_config='jvm', is_gate=false, suite='default:~r[.*jar]', shape=null): vm_common.svm_common + vm_common.truffleruby + vm.custom_vm + vm.vm_java_Latest + self.polybench_hpc_linux_common(shape=shape) + self.vm_bench_polybenchmarks_base(env='polybench-${VM_ENV}') { + vm_bench_polybenchmarks_linux_common(vm_config='jvm', is_gate=false, suite='default:~r[.*jar]', shape=null): common.deps.svm + common.deps.truffleruby + vm.custom_vm + vm.vm_java_Latest + self.polybench_hpc_linux_common(shape=shape) + self.vm_bench_polybenchmarks_base(env='polybench-${VM_ENV}') { bench_cmd:: self.base_cmd + ['benchmark', '--results-file', self.result_file], setup+: [ self.base_cmd + ['sforceimports'], @@ -106,7 +106,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet'; timelimit: if (is_gate) then '1:00:00' else '1:30:00', } + (if is_gate then self.vm_bench_base(machine_name=null) else self.vm_bench_common), - vm_bench_polybench_linux_common(env='polybench-${VM_ENV}', fail_fast=false, skip_machine=false): (if skip_machine then self.vm_bench_base(machine_name=null) else self.vm_bench_common) + vm_common.svm_common + vm_common.truffleruby + vm_common.graalpy + vm.custom_vm + vm_common.wasm { + vm_bench_polybench_linux_common(env='polybench-${VM_ENV}', fail_fast=false, skip_machine=false): (if skip_machine then self.vm_bench_base(machine_name=null) else self.vm_bench_common) + common.deps.svm + common.deps.truffleruby + common.deps.graalpy + common.deps.wasm + vm.custom_vm { base_cmd:: ['mx', '--env', env], bench_cmd:: self.base_cmd + ['benchmark'] + (if (fail_fast) then ['--fail-fast'] else []), interpreter_bench_cmd(vmConfig):: self.bench_cmd + @@ -275,7 +275,7 @@ local repo_config = import '../../../ci/repo-configuration.libsonnet'; notify_groups:: ['compiler_bench'] }, - vm_bench_polybench_nfi_linux_amd64: self.vm_bench_common + vm_common.svm_common + self.vm_bench_polybench_nfi, + vm_bench_polybench_nfi_linux_amd64: self.vm_bench_common + common.deps.svm + self.vm_bench_polybench_nfi, local builds = [ # We used to expand `${common_vm_linux}` here to work around some limitations in the version of pyhocon that we use in the CI diff --git a/vm/ci/ci_common/common.jsonnet b/vm/ci/ci_common/common.jsonnet index 32161bc71b32..d97b97dd155a 100644 --- a/vm/ci/ci_common/common.jsonnet +++ b/vm/ci/ci_common/common.jsonnet @@ -73,24 +73,12 @@ local devkits = graal_common.devkits; }, }, - # SULONG - sulong: graal_common.deps.sulong, - - # TRUFFLERUBY, needs OpenSSL 1.0.2+, so OracleLinux 7+ - truffleruby: graal_common.deps.sulong + graal_common.deps.truffleruby, - fastr_no_recommended: { environment+: { FASTR_NO_RECOMMENDED: 'true' }, }, - # GRAALPYTHON - graalpy: self.sulong + graal_common.deps.graalpy, - - # WASM - wasm: graal_common.deps.wasm, - vm_linux_amd64_common: graal_common.deps.svm { capabilities+: ['manycores', 'ram16gb', 'fast'], }, @@ -146,8 +134,7 @@ local devkits = graal_common.devkits; mx_vm_complete: vm.mx_cmd_base_no_env + ['--env', '${VM_ENV}-complete'] + self.mx_vm_cmd_suffix, // svm_common includes the dependencies for all platforms besides windows amd64 - svm_common: graal_common.deps.svm, - svm_common_windows_amd64(jdk): self.svm_common + graal_common.devkits["windows-jdk" + jdk], + svm_common_windows_amd64(jdk): graal_common.deps.svm + graal_common.devkits["windows-jdk" + jdk], maven_deploy_sdk: ['--suite', 'sdk', 'maven-deploy', '--validate', 'none', '--all-distribution-types', '--with-suite-revisions-metadata'], deploy_artifacts_sdk(os, base_dist_name=null): (if base_dist_name != null then ['--base-dist-name=' + base_dist_name] else []) + ['--suite', 'sdk', 'deploy-artifacts', '--uploader', if os == 'windows' then 'artifact_uploader.cmd' else 'artifact_uploader'], @@ -176,9 +163,7 @@ local devkits = graal_common.devkits; $.mx_vm_complete + self.artifact_deploy_standalones_dry_run(os) ], - ruby_vm_build: self.svm_common + self.sulong + self.truffleruby + vm.custom_vm, - ruby_python_vm_build: self.ruby_vm_build + self.graalpy, - full_vm_build: self.ruby_python_vm_build + graal_common.deps.fastr, + full_vm_build: graal_common.deps.svm + graal_common.deps.sulong + graal_common.deps.truffleruby + graal_common.deps.graalpy + graal_common.deps.fastr + vm.custom_vm, graalvm_complete_build_deps(edition, os, arch, java_version): local java_deps(edition) = { @@ -201,7 +186,7 @@ local devkits = graal_common.devkits; if (os == 'windows') then if (arch == 'amd64') then # Windows/AMD64 - java_deps(edition) + (if (java_version == 'latest') then self.svm_common_windows_amd64("Latest") else self.svm_common_windows_amd64(java_version)) + self.js_windows_common + self.sulong + java_deps(edition) + (if (java_version == 'latest') then self.svm_common_windows_amd64("Latest") else self.svm_common_windows_amd64(java_version)) + self.js_windows_common + graal_common.deps.sulong else error 'Unknown windows arch: ' + arch else if (os == 'linux' || os == 'darwin') then @@ -567,9 +552,9 @@ local devkits = graal_common.devkits; # Darwin/AARCH64 deploy_vm_standalones_javaLatest_darwin_aarch64: vm.vm_java_Latest + self.full_vm_build + self.darwin_deploy + self.vm_base('darwin', 'aarch64', 'daily', deploy=true) + self.deploy_graalvm_standalones('latest') + {name: 'daily-deploy-vm-standalones-java-latest-darwin-aarch64', notify_groups:: ["deploy"], notify_emails+: ["bernhard.urban-forster@oracle.com"], timelimit: '3:00:00'}, # Windows/AMD64 - deploy_vm_standalones_javaLatest_windows_amd64: vm.vm_java_Latest + self.svm_common_windows_amd64('Latest') + self.js_windows_common + self.sulong + self.vm_base('windows', 'amd64', 'daily', deploy=true, jdk_hint='Latest') + self.deploy_graalvm_standalones('latest') + self.deploy_build + {name: 'daily-deploy-vm-standalones-java-latest-windows-amd64', timelimit: '2:30:00', notify_groups:: ["deploy"]}, + deploy_vm_standalones_javaLatest_windows_amd64: vm.vm_java_Latest + self.svm_common_windows_amd64('Latest') + self.js_windows_common + graal_common.deps.sulong + self.vm_base('windows', 'amd64', 'daily', deploy=true, jdk_hint='Latest') + self.deploy_graalvm_standalones('latest') + self.deploy_build + {name: 'daily-deploy-vm-standalones-java-latest-windows-amd64', timelimit: '2:30:00', notify_groups:: ["deploy"]}, - local sulong_vm_tests = self.svm_common + self.sulong + vm.custom_vm + self.vm_base('linux', 'amd64', 'gate') + { + local sulong_vm_tests = graal_common.deps.svm + graal_common.deps.sulong + vm.custom_vm + self.vm_base('linux', 'amd64', 'gate') + { run: [ ['export', 'SVM_SUITE=' + vm.svm_suite], ['mx', '--dynamicimports', '$SVM_SUITE,/sulong', '--disable-polyglot', '--disable-libpolyglot', 'gate', '--no-warning-as-error', '--tags', 'build,sulong'], diff --git a/vm/ci/ci_common/libgraal.jsonnet b/vm/ci/ci_common/libgraal.jsonnet index a9c68642494b..506c8e7120ab 100644 --- a/vm/ci/ci_common/libgraal.jsonnet +++ b/vm/ci/ci_common/libgraal.jsonnet @@ -107,7 +107,7 @@ local galahad = import '../../../ci/ci_common/galahad-common.libsonnet'; if (os_arch == 'windows-amd64') then c.svm_common_windows_amd64(jdk) else - c.svm_common, + graal_common.deps.svm, local all_os_arches = [ "linux-amd64", diff --git a/vm/ci/ci_includes/vm-native.jsonnet b/vm/ci/ci_includes/vm-native.jsonnet index 7d93d8e85bd4..3a88cb52d4b0 100644 --- a/vm/ci/ci_includes/vm-native.jsonnet +++ b/vm/ci/ci_includes/vm-native.jsonnet @@ -4,7 +4,7 @@ local vm_common = import '../ci_common/common.jsonnet'; local graal_common = import '../../../ci/ci_common/common.jsonnet'; { - local truffle_native_tck = vm_common.svm_common + { + local truffle_native_tck = graal_common.deps.svm + { run+: [ ['mx', '--env', 'ce', '--dynamicimports', '/tools', '--native-images=lib:jvmcicompiler', 'gate', '--tags', 'build,truffle-native-tck,truffle-native-tck-sl'], ], @@ -14,7 +14,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; name: self.targets[0] + '-vm-truffle-native-tck-labs' + self.jdk_name + '-linux-amd64', }, - local truffle_native_tck_wasm = vm_common.svm_common + { + local truffle_native_tck_wasm = graal_common.deps.svm + { run+: [ ['mx', '--env', 'ce', '--dynamicimports', '/wasm', '--native-images=lib:jvmcicompiler', 'gate', '--tags', 'build,truffle-native-tck-wasm'], ], @@ -24,7 +24,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; name: self.targets[0] + '-vm-truffle-native-tck-wasm-labs' + self.jdk_name + '-linux-amd64', }, - local truffle_maven_downloader = vm_common.svm_common + vm_common.sulong + { + local truffle_maven_downloader = graal_common.deps.svm + graal_common.deps.sulong + { run+: [ ['export', 'SVM_SUITE=' + vm.svm_suite], ['mx', '--env', 'ce-llvm', '--native-images=', 'gate', '--no-warning-as-error', '--tags', 'build,maven-downloader'], @@ -39,7 +39,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; }, local builds = [ - vm.vm_java_Latest + vm_common.svm_common + vm_common.sulong + vm_common.graalpy + vm.custom_vm + vm_common.vm_base('linux', 'amd64', 'gate') + { + vm.vm_java_Latest + graal_common.deps.svm + graal_common.deps.sulong + graal_common.deps.graalpy + vm.custom_vm + vm_common.vm_base('linux', 'amd64', 'gate') + { run+: [ ['export', 'SVM_SUITE=' + vm.svm_suite], ['mx', '--dynamicimports', '$SVM_SUITE,graalpython', '--disable-polyglot', '--disable-libpolyglot', '--force-bash-launchers=lli,native-image', 'gate', '--no-warning-as-error', '--tags', 'build,python'], diff --git a/vm/ci/ci_includes/vm.jsonnet b/vm/ci/ci_includes/vm.jsonnet index 290b0754f8f0..93e66b6f6007 100644 --- a/vm/ci/ci_includes/vm.jsonnet +++ b/vm/ci/ci_includes/vm.jsonnet @@ -118,7 +118,7 @@ local graal_common = import '../../../ci/ci_common/common.jsonnet'; ], name: 'gate-vm-unittest-windows-amd64', }, ["sdk", "truffle", "vm"]), - self.vm_java_Latest + vm_common.vm_base('linux', 'amd64', 'gate') + vm_common.sulong + { + self.vm_java_Latest + vm_common.vm_base('linux', 'amd64', 'gate') + graal_common.deps.sulong + { environment+: { DYNAMIC_IMPORTS: '/tools,/substratevm,/sulong', NATIVE_IMAGES: 'polyglot', From b7715868d551b423a98351fc30d41822170135f3 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Mon, 26 May 2025 18:44:12 +0200 Subject: [PATCH 2/5] No more gu --- sdk/mx.sdk/mx_sdk_vm_impl.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sdk/mx.sdk/mx_sdk_vm_impl.py b/sdk/mx.sdk/mx_sdk_vm_impl.py index 256cd3ea0931..cb9b23649c53 100644 --- a/sdk/mx.sdk/mx_sdk_vm_impl.py +++ b/sdk/mx.sdk/mx_sdk_vm_impl.py @@ -102,8 +102,6 @@ def unicode_utf8(string): default_components = [] -USE_LEGACY_GU = mx.str_to_bool(mx.get_env('LEGACY_GU', 'false')) - mx.add_argument('--base-dist-name', help='Sets the name of the GraalVM base image ( for complete, ruby ... images), default to "base"', default='base') From 2b263875592956f5b6f45d6dd5cc55cde82fe7ab Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Tue, 27 May 2025 15:58:24 +0200 Subject: [PATCH 3/5] Update overlay --- graal-common.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graal-common.json b/graal-common.json index 7d430a4709b5..53dbe718a07f 100644 --- a/graal-common.json +++ b/graal-common.json @@ -1,6 +1,6 @@ { "README": "This file contains definitions that are useful for the jsonnet CI files of the graal and graal-enterprise repositories.", "ci": { - "overlay": "aa82bb70f9d2919c706ff28ef3331cab574d5e51" + "overlay": "ed6845f6ba14623db7c6e2f71f0ed9f6b0efb708" } } From 1621305c9ac3408816aa2f380b2fb8b226a7d4f1 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Tue, 27 May 2025 17:51:35 +0200 Subject: [PATCH 4/5] Remove unused gate tags --- vm/mx.vm/mx_vm_gate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/vm/mx.vm/mx_vm_gate.py b/vm/mx.vm/mx_vm_gate.py index 73e3d390c1b0..6165360e3f75 100644 --- a/vm/mx.vm/mx_vm_gate.py +++ b/vm/mx.vm/mx_vm_gate.py @@ -56,7 +56,6 @@ class VmGateTasks: sulong = 'sulong' sulong_aot = 'sulong-aot' graal_js_all = 'graal-js' - graal_js_smoke = 'graal-js-smoke' graal_js_tests = 'graal-js-tests' graal_js_tests_compiled = 'graal-js-tests-compiled' graal_nodejs = 'graal-nodejs' @@ -64,7 +63,6 @@ class VmGateTasks: ruby = 'ruby' python = 'python' fastr = 'fastr' - graalpython = 'graalpython' integration = 'integration' tools = 'tools' libgraal = 'libgraal' From 450f46073a76d6ce74d43db47cbe3b7a27a7a674 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Thu, 5 Jun 2025 15:26:39 +0200 Subject: [PATCH 5/5] Building graalpy requires cmake --- ci/common.jsonnet | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/ci/common.jsonnet b/ci/common.jsonnet index 8f99dbc1aace..74d4ac1dc9b2 100644 --- a/ci/common.jsonnet +++ b/ci/common.jsonnet @@ -205,6 +205,12 @@ local common_json = import "../common.json"; }, }, + cmake:: { + packages+: { + cmake: "==3.22.2", + }, + }, + gradle:: { downloads+: { GRADLE_JAVA_HOME: jdks_data["oraclejdk21"], @@ -221,10 +227,8 @@ local common_json = import "../common.json"; # GR-49566: SpotBugs does not yet run on JDK 22 spotbugs: code_tools, - sulong:: { - packages+: { - cmake: "==3.22.2", - } + if self.os == "windows" then { + sulong:: self.cmake + { + packages+: if self.os == "windows" then { msvc_source: "==14.0", } else {}, }, @@ -259,7 +263,7 @@ local common_json = import "../common.json"; } else {}, }, - graalpy:: self.gradle + { + graalpy:: self.gradle + self.cmake + { packages+: if (self.os == "linux") then { libffi: '>=3.2.1', bzip2: '>=1.0.6',