[GR-48742] [GR-48754] Improve deployment of artifacts.

PullRequest: graal/15618

Author: ansalond

common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "6.49.1",
+  "mx_version": "6.50.2",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {

graal-common.json

@@ -1,6 +1,6 @@
 {
   "README": "This file contains definitions that are useful for the jsonnet CI files of the graal and graal-enterprise repositories.",
   "ci": {
-    "overlay": "f85c63d782782fc566bd661a1d638f8f9bb65b2e"
+    "overlay": "f197d31e391f68e62e7b7897e8e161b50e5e880d"
   }
 }

sdk/mx.sdk/mx_sdk_vm_impl.py

@@ -3498,12 +3498,25 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
     :type register_project: (mx.Project) -> None
     :type register_distribution: (mx.Distribution) -> None
     """
-    with_debuginfo = []
+    main_dists = {
+        'graalvm': [],
+        'graalvm_installables': [],
+        'graalvm_standalones': [],
+    }
     with_non_rebuildable_configs = False
+
+    def register_main_dist(dist, label):
+        assert label in main_dists, f"Unknown label: '{label}'. Expected: '{list(main_dists.keys())}'"
+        register_distribution(dist)
+        main_dists[label].append(dist.name)
+        if _debuginfo_dists():
+            if _get_svm_support().is_debug_supported() or mx.get_opts().strip_jars or with_non_rebuildable_configs:
+                debuginfo_dist = DebuginfoDistribution(dist)
+                register_distribution(debuginfo_dist)
+                main_dists[label].append(debuginfo_dist.name)
+
     _final_graalvm_distribution = get_final_graalvm_distribution()
-    register_distribution(_final_graalvm_distribution)
-    with_debuginfo.append(_final_graalvm_distribution)
-    other_graalvm_artifact_names = []
+    register_main_dist(_final_graalvm_distribution, 'graalvm')
 
     # Add the macros if SubstrateVM is in stage1, as images could be created later with an installable Native Image
     with_svm = has_component('svm', stage1=True)
@@ -3602,32 +3615,16 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
         native_image_resources_filelist_project = NativeImageResourcesFileList(None, ni_resources_components, dir_name, deps)
         register_project(native_image_resources_filelist_project)
 
-    # Trivial distribution to trigger the build of the final GraalVM distribution and its dependencies
-    register_distribution(mx.LayoutDirDistribution(
-        suite=_suite,
-        name="GRAALVM",
-        deps=[_final_graalvm_distribution.name],
-        layout={
-            "./deps": "string:" + _final_graalvm_distribution.name,
-        },
-        path=None,
-        platformDependent=False,
-        theLicense=None,
-        defaultBuild=False,
-    ))
-
     # Create installables
-    installable_names = []
+    # installable_names = []
     for components in installables.values():
         main_component = _get_main_component(components)
         installable_component = GraalVmInstallableComponent(main_component, extra_components=[c for c in components if c != main_component])
-        installable_names.append(installable_component.name)
-        register_distribution(installable_component)
-        with_debuginfo.append(installable_component)
+        register_main_dist(installable_component, 'graalvm_installables')
 
     # Create standalones
     needs_java_standalone_jimage = False
-    standalone_names = []
+    # standalone_names = []
     for components in installables.values():
         main_component = _get_main_component(components)
         svm_support = _get_svm_support()
@@ -3647,9 +3644,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
                 else:
                     needs_java_standalone_jimage = True
                     java_standalone = GraalVmStandaloneComponent(get_component(main_component.name, fatalIfMissing=True), _final_graalvm_distribution, is_jvm=True, defaultBuild=False)
-                    standalone_names.append(java_standalone.name)
-                    register_distribution(java_standalone)
-                    with_debuginfo.append(java_standalone)
+                    register_main_dist(java_standalone, 'graalvm_standalones')
 
                     for library_config in _get_library_configs(main_component):
                         if isinstance(library_config, mx_sdk.LanguageLibraryConfig) and library_config.launchers:
@@ -3662,9 +3657,7 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
                 only_native_libraries = not main_component.library_configs or (svm_support.is_supported() and not _has_skipped_libraries(main_component))
                 if only_native_launchers and only_native_libraries:
                     native_standalone = GraalVmStandaloneComponent(get_component(main_component.name, fatalIfMissing=True), _final_graalvm_distribution, is_jvm=False, defaultBuild=False)
-                    standalone_names.append(native_standalone.name)
-                    register_distribution(native_standalone)
-                    with_debuginfo.append(native_standalone)
+                    register_main_dist(native_standalone, 'graalvm_standalones')
 
     if needs_java_standalone_jimage:
         java_standalone_jimage_jars = set()
@@ -3684,20 +3677,6 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
         )
         register_project(java_standalone_jimage)
 
-    # Trivial distribution to trigger the build of all standalones and their dependencies (they have `defaultBuild=False`)
-    register_distribution(mx.LayoutDirDistribution(
-        suite=_suite,
-        name="GRAALVM_STANDALONES",
-        deps=standalone_names,
-        layout={
-            "./standalone_deps_names": "string:" + ",".join(standalone_names),
-        },
-        path=None,
-        platformDependent=False,
-        theLicense=None,
-        defaultBuild=False,
-    ))
-
     if needs_stage1:
         if register_project:
             for component in registered_graalvm_components(stage1=True):
@@ -3744,21 +3723,30 @@ def mx_register_dynamic_suite_constituents(register_project, register_distributi
                 jimage_project=final_jimage_project,
         ))
 
-    if _debuginfo_dists():
-        if _get_svm_support().is_debug_supported() or mx.get_opts().strip_jars or with_non_rebuildable_configs:
-            for d in with_debuginfo:
-                debuginfo_dist = DebuginfoDistribution(d)
-                register_distribution(debuginfo_dist)
-                other_graalvm_artifact_names.append(debuginfo_dist.name)
+    # Trivial distributions to trigger the build of the final GraalVM distribution, installables, and standalones
+    all_main_dists = []
+    for label, dists in main_dists.items():
+        if dists:
+            all_main_dists += dists
+            register_distribution(mx.LayoutDirDistribution(
+                suite=_suite,
+                name=label.upper(),
+                deps=dists,
+                layout={
+                    "./deps": "string:" + ",".join(dists),
+                },
+                path=None,
+                platformDependent=False,
+                theLicense=None,
+                defaultBuild=False,
+            ))
 
-    # Trivial distribution to trigger the build of the final GraalVM distribution and of all standalones
-    all_artifacts_deps = ["GRAALVM", "GRAALVM_STANDALONES"] + installable_names + other_graalvm_artifact_names
     register_distribution(mx.LayoutDirDistribution(
         suite=_suite,
         name="ALL_GRAALVM_ARTIFACTS",
-        deps=all_artifacts_deps,
+        deps=all_main_dists,
         layout={
-            "./deps": "string:" + ",".join(all_artifacts_deps),
+            "./deps": "string:" + ",".join(all_main_dists),
         },
         path=None,
         platformDependent=False,
@@ -4280,6 +4268,11 @@ def graalvm_show(args, forced_graalvm_dist=None):
                 for _, cfg in sorted(jvm_configs.items()):
                     for config in cfg['configs']:
                         print(f" {config} (from {cfg['source']})")
+            if args.verbose:
+                for dist_name in 'GRAALVM', 'GRAALVM_INSTALLABLES', 'GRAALVM_STANDALONES', 'ALL_GRAALVM_ARTIFACTS':
+                    dist = mx.distribution(dist_name, fatalIfMissing=False)
+                    if dist is not None:
+                        print(f"Dependencies of the '{dist_name}' distribution:\n -", '\n - '.join(sorted(dep.name for dep in dist.deps)))
 
         if args.print_env:
             def _print_env(name, val):

vm/ci/ci_common/common.jsonnet

@@ -724,12 +724,6 @@ local devkits = graal_common.devkits;
   record_file_sizes:: ['benchmark', 'file-size:*', '--results-file', 'sizes.json'],
   upload_file_sizes:: ['bench-uploader.py', 'sizes.json'],
 
-  # These dummy artifacts are required by notify_releaser_build in order to trigger the releaser service
-  # after the deploy jobs have completed.
-  create_releaser_notifier_artifact:: [
-    ['python3', '-c', "from os import environ; open('../' + environ['BUILD_NAME'], 'a').close()"],
-  ],
-
   build_base_graalvm_image: [
     $.mx_vm_common + vm.vm_profiles + ['graalvm-show'],
     $.mx_vm_common + vm.vm_profiles + ['build'],
@@ -772,20 +766,34 @@ local devkits = graal_common.devkits;
     run: $.patch_env(self.os, self.arch, java_version) + vm.collect_profiles() + $.build_base_graalvm_image + [
       $.mx_vm_common + vm.vm_profiles + $.record_file_sizes,
       $.upload_file_sizes,
-    ] + $.deploy_sdk_base(self.os) + $.create_releaser_notifier_artifact + $.check_base_graalvm_image(self.os, self.arch, java_version),
+    ] + $.deploy_sdk_base(self.os) + $.check_base_graalvm_image(self.os, self.arch, java_version),
     notify_groups:: ['deploy'],
     timelimit: "1:00:00"
   },
 
-  deploy_graalvm_components(java_version, tags): vm.check_structure + {
+  deploy_graalvm_components(java_version, installables, standalones, record_file_sizes=false): vm.check_structure + {
+    build_deps:: std.join(',', []
+      + (if (record_file_sizes) then ['GRAALVM'] else [])
+      + (if (installables) then ['GRAALVM_INSTALLABLES'] else [])
+      + (if (standalones) then ['GRAALVM_STANDALONES'] else [])
+    ),
+
+    tags:: std.join(',', []
+      + (if (installables) then ['installable'] else [])
+      + (if (standalones) then ['standalone'] else [])
+    ),
+
     run: $.patch_env(self.os, self.arch, java_version) + [
       $.mx_vm_installables + ['graalvm-show'],
-      $.mx_vm_installables + ['build', '--dependencies', 'ALL_GRAALVM_ARTIFACTS'],
-      ['set-export', 'GRAALVM_HOME', $.mx_vm_installables + ['--quiet', '--no-warning', 'graalvm-home']],
-    ] + $.deploy_sdk_components(self.os, tags) + [
-      $.mx_vm_installables + $.record_file_sizes,
-      $.upload_file_sizes,
-    ] + $.create_releaser_notifier_artifact + vm.check_graalvm_complete_build($.mx_vm_installables, self.os, self.arch, java_version),
+      $.mx_vm_installables + ['build', '--dependencies', self.build_deps],
+    ]
+    + $.deploy_sdk_components(self.os, self.tags)
+    + (
+      if (record_file_sizes) then [
+        $.mx_vm_installables + $.record_file_sizes,
+        $.upload_file_sizes,
+      ] else []
+    ),
     notify_groups:: ['deploy'],
     timelimit: "1:30:00"
   },
@@ -801,7 +809,7 @@ local devkits = graal_common.devkits;
       ['set-export', 'GRAALVM_HOME', $.mx_vm_common + ['--quiet', '--no-warning', 'graalvm-home']],
       ['set-export', 'DACAPO_JAR', $.mx_vm_common + ['--quiet', '--no-warning', 'paths', '--download', 'DACAPO_MR1_2baec49']],
       ['${GRAALVM_HOME}/bin/java', '-jar', '${DACAPO_JAR}', 'luindex'],
-    ] + $.create_releaser_notifier_artifact,
+    ],
     notify_groups:: ['deploy'],
     timelimit: '1:45:00',
   },
@@ -813,25 +821,25 @@ local devkits = graal_common.devkits;
 
   # Linux/AMD64
   deploy_vm_base_java21_linux_amd64: vm.vm_java_21_llvm + self.full_vm_build_linux_amd64 + self.linux_deploy + self.deploy_vm_linux_amd64 + self.deploy_graalvm_base("java21") + {name: 'post-merge-deploy-vm-base-java21-linux-amd64', diskspace_required: vm.diskspace_required.java21_linux_amd64, notify_groups:: ["deploy"]},
-  deploy_vm_installables_standalones_java21_linux_amd64: vm.vm_java_21_llvm + self.full_vm_build_linux_amd64 + self.linux_deploy + self.deploy_daily_vm_linux_amd64 + self.deploy_graalvm_components("java21", "installable,standalone") + {name: 'daily-deploy-vm-installables-standalones-java21-linux-amd64', diskspace_required: vm.diskspace_required.java21_linux_amd64, notify_groups:: ["deploy"]},
+  deploy_vm_installables_standalones_java21_linux_amd64: vm.vm_java_21_llvm + self.full_vm_build_linux_amd64 + self.linux_deploy + self.deploy_daily_vm_linux_amd64 + self.deploy_graalvm_components("java21", installables=true, standalones=true, record_file_sizes=true) + {name: 'daily-deploy-vm-installables-standalones-java21-linux-amd64', diskspace_required: vm.diskspace_required.java21_linux_amd64, notify_groups:: ["deploy"]},
   # Linux/AARCH64
   deploy_vm_base_java21_linux_aarch64: vm.vm_java_21 + self.full_vm_build_linux_aarch64 + self.linux_deploy + self.deploy_daily_vm_linux_aarch64 + self.deploy_graalvm_base("java21") + {name: 'daily-deploy-vm-base-java21-linux-aarch64', notify_groups:: ["deploy"], timelimit: '1:30:00', capabilities+: ["!xgene3"]},
-  deploy_vm_installables_standalones_java21_linux_aarch64: vm.vm_java_21 + self.full_vm_build_linux_aarch64 + self.linux_deploy + self.deploy_daily_vm_linux_aarch64 + self.deploy_graalvm_components("java21", "installable,standalone") + {name: 'daily-deploy-vm-installables-standalones-java21-linux-aarch64', notify_groups:: ["deploy"], capabilities+: ["!xgene3"]},
+  deploy_vm_installables_standalones_java21_linux_aarch64: vm.vm_java_21 + self.full_vm_build_linux_aarch64 + self.linux_deploy + self.deploy_daily_vm_linux_aarch64 + self.deploy_graalvm_components("java21", installables=true, standalones=true) + {name: 'daily-deploy-vm-installables-standalones-java21-linux-aarch64', notify_groups:: ["deploy"], capabilities+: ["!xgene3"]},
 
   # Darwin/AMD64
   deploy_vm_base_java21_darwin_amd64: vm.vm_java_21_llvm + self.full_vm_build_darwin_amd64 + self.darwin_deploy + self.deploy_daily_vm_darwin_amd64 + self.deploy_graalvm_base("java21") + {name: 'daily-deploy-vm-base-java21-darwin-amd64', notify_groups:: ["deploy"], timelimit: '1:45:00'},
-  deploy_vm_installables_java21_darwin_amd64: vm.vm_java_21_llvm + self.full_vm_build_darwin_amd64 + self.darwin_deploy + self.deploy_weekly_vm_darwin_amd64 + self.deploy_graalvm_components("java21", "installable") + {name: 'weekly-deploy-vm-installables-java21-darwin-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"], timelimit: '3:00:00'},
-  deploy_vm_standalones_java21_darwin_amd64: vm.vm_java_21_llvm + self.full_vm_build_darwin_amd64 + self.darwin_deploy + self.deploy_daily_vm_darwin_amd64 + self.deploy_graalvm_components("java21", "standalone") + {name: 'daily-deploy-vm-standalones-java21-darwin-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"], timelimit: '3:00:00'},
+  deploy_vm_installables_java21_darwin_amd64: vm.vm_java_21_llvm + self.full_vm_build_darwin_amd64 + self.darwin_deploy + self.deploy_weekly_vm_darwin_amd64 + self.deploy_graalvm_components("java21", installables=true, standalones=false) + {name: 'weekly-deploy-vm-installables-java21-darwin-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"], timelimit: '3:00:00'},
+  deploy_vm_standalones_java21_darwin_amd64: vm.vm_java_21_llvm + self.full_vm_build_darwin_amd64 + self.darwin_deploy + self.deploy_daily_vm_darwin_amd64 + self.deploy_graalvm_components("java21", installables=false, standalones=true) + {name: 'daily-deploy-vm-standalones-java21-darwin-amd64', diskspace_required: "31GB", notify_groups:: ["deploy"], timelimit: '3:00:00'},
 
   # Darwin/AARCH64
   deploy_vm_base_java21_darwin_aarch64: vm.vm_java_21 + self.full_vm_build_darwin_aarch64 + self.darwin_deploy + self.deploy_daily_vm_darwin_aarch64 + self.deploy_graalvm_base("java21") + {name: 'daily-deploy-vm-base-java21-darwin-aarch64', notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '1:45:00'},
-  deploy_vm_installables_java21_darwin_aarch64: vm.vm_java_21 + self.full_vm_build_darwin_aarch64 + self.darwin_deploy + self.deploy_weekly_vm_darwin_aarch64 + self.deploy_graalvm_components("java21", "installable") + {name: 'weekly-deploy-vm-installables-java21-darwin-aarch64', diskspace_required: "31GB", notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '3:00:00'},
-  deploy_vm_standalones_java21_darwin_aarch64: vm.vm_java_21 + self.full_vm_build_darwin_aarch64 + self.darwin_deploy + self.deploy_daily_vm_darwin_aarch64 + self.deploy_graalvm_components("java21", "standalone") + {name: 'daily-deploy-vm-standalones-java21-darwin-aarch64', diskspace_required: "31GB", notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '3:00:00'},
+  deploy_vm_installables_java21_darwin_aarch64: vm.vm_java_21 + self.full_vm_build_darwin_aarch64 + self.darwin_deploy + self.deploy_weekly_vm_darwin_aarch64 + self.deploy_graalvm_components("java21", installables=true, standalones=false) + {name: 'weekly-deploy-vm-installables-java21-darwin-aarch64', diskspace_required: "31GB", notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '3:00:00'},
+  deploy_vm_standalones_java21_darwin_aarch64: vm.vm_java_21 + self.full_vm_build_darwin_aarch64 + self.darwin_deploy + self.deploy_daily_vm_darwin_aarch64 + self.deploy_graalvm_components("java21", installables=false, standalones=true) + {name: 'daily-deploy-vm-standalones-java21-darwin-aarch64', diskspace_required: "31GB", notify_groups:: ["deploy"], notify_emails+: ["[email protected]"], timelimit: '3:00:00'},
 
   # Windows/AMD64
   deploy_vm_base_java21_windows_amd64: vm.vm_java_21 + self.svm_common_windows_amd64("21") + self.js_windows_jdk21 + self.deploy_daily_vm_windows_jdk21 + self.deploy_graalvm_base("java21") + self.deploy_build + {name: 'daily-deploy-vm-base-java21-windows-amd64', notify_groups:: ["deploy"], timelimit: '1:30:00'},
-  deploy_vm_installables_java21_windows_amd64: vm.vm_java_21 + self.svm_common_windows_amd64("21") + self.js_windows_jdk21 + self.sulong_windows + self.deploy_weekly_vm_windows_jdk21 + self.deploy_graalvm_components("java21", "installable") + self.deploy_build + {name: 'weekly-deploy-vm-installables-java21-windows-amd64', diskspace_required: "31GB", timelimit: '2:30:00', notify_groups:: ["deploy"]},
-  deploy_vm_standalones_java21_windows_amd64: vm.vm_java_21 + self.svm_common_windows_amd64("21") + self.js_windows_jdk21 + self.sulong_windows + self.deploy_daily_vm_windows_jdk21 + self.deploy_graalvm_components("java21", "standalone") + self.deploy_build + {name: 'daily-deploy-vm-standalones-java21-windows-amd64', diskspace_required: "31GB", timelimit: '2:30:00', notify_groups:: ["deploy"]},
+  deploy_vm_installables_java21_windows_amd64: vm.vm_java_21 + self.svm_common_windows_amd64("21") + self.js_windows_jdk21 + self.sulong_windows + self.deploy_weekly_vm_windows_jdk21 + self.deploy_graalvm_components("java21", installables=true, standalones=false) + self.deploy_build + {name: 'weekly-deploy-vm-installables-java21-windows-amd64', diskspace_required: "31GB", timelimit: '2:30:00', notify_groups:: ["deploy"]},
+  deploy_vm_standalones_java21_windows_amd64: vm.vm_java_21 + self.svm_common_windows_amd64("21") + self.js_windows_jdk21 + self.sulong_windows + self.deploy_daily_vm_windows_jdk21 + self.deploy_graalvm_components("java21", installables=false, standalones=true) + self.deploy_build + {name: 'daily-deploy-vm-standalones-java21-windows-amd64', diskspace_required: "31GB", timelimit: '2:30:00', notify_groups:: ["deploy"]},
 
   #
   # Deploy the GraalVM Espresso artifact (GraalVM Base + espresso - native image)