Merge branch 'develop' into looseversion-test-alphabetical-prefix

Author: boegel

.github/workflows/container_tests.yml

@@ -97,7 +97,7 @@ jobs:
           # see https://docs.easybuild.io/en/latest/Containers.html
           curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
-          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/rockylinux-8.10-amd64'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
           singularity exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)
           singularity exec bzip2-1.0.8.sif bzip2 --help

.github/workflows/container_tests_apptainer.yml

@@ -91,7 +91,7 @@ jobs:
           # see https://docs.easybuild.io/en/latest/Containers.html
           curl -OL https://raw.githubusercontent.com/easybuilders/easybuild-easyconfigs/develop/easybuild/easyconfigs/b/bzip2/bzip2-1.0.8.eb
           export EASYBUILD_CONTAINERPATH=$PWD
-          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/centos-7.9-python3-amd64'
+          export EASYBUILD_CONTAINER_CONFIG='bootstrap=docker,from=ghcr.io/easybuilders/rockylinux-8.10-amd64'
           export EASYBUILD_CONTAINER_TYPE='apptainer'
           eb bzip2-1.0.8.eb --containerize --experimental --container-build-image
           apptainer exec bzip2-1.0.8.sif command -v bzip2 | grep '/app/software/bzip2/1.0.8/bin/bzip2' || (echo "Path to bzip2 '$which_bzip2' is not correct" && exit 1)

RELEASE_NOTES

@@ -4,9 +4,62 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://docs.easybuild.io/release-notes .
 
 
+v5.1.0 (26 May 2025)
+--------------------
+
+feature release
+
+- various enhancements, including:
+  - add support for data installations (#4474, #4873, #4874)
+  - allow specifying location for RPATH wrapper scripts via `rpath_wrappers_dir` (#4596)
+  - add a CUDA device code sanity check (#4692)
+  - add support for check_readelf_rpath easyconfig parameter to optionally skip RPATH checks (#4768)
+  - add support for using environment variables in value used in modextravars (#4855)
+  - print summary after the build in trace output (#4861, #4875)
+  - avoid leaking keys by mistake with `--upload-test-report` (#4877)
+  - obtain PR/commit diff via GitHub API rather than downloading `*.diff` file via github.com (#4878)
+  - support options for patch command (#4886)
+  - replace full trace message for extension check command with simple pass/fail message (#4892)
+- various bug fixes, including:
+  - also pass `rpath_include_dirs` when preparing build environment for extensions (#4596)
+  - fix `check_checksums` when `nosource: True` is used (#4806)
+  - fix help string of findPythonDeps.py script (#4821)
+  - take into account `job-output-dir` option in Slurm job backend (#4842)
+  - fix unbound variable in error case in `build_and_install_software` (#4843)
+  - avoid failure when only some passed easyconfigs exist (#4847)
+  - restore original value for non-list easyconfig parameter values that are considered for iterating over (#4848)
+  - add '-' before 'DMKL_ILP64' in $CFLAGS (#4850)
+  - enhance RPATH sanity check to skip anything whose absolute path resolves to outside the install dir (#4854)
+  - use new `ModEnvVarType.STRICT_PATH_WITH_FILES` with `CMAKE_LIBRARY_PATH` environment variable (#4858)
+  - fix `is_patch_for` for patch dicts (#4865)
+  - update fake module for each extension installed (#4868, #4888, #4895)
+  - implement exponential backoff in `download_file` (#4870, #4880)
+  - use `develop` branch for PRs that target removed `5.0.x` branch in `fetch_files_from_pr` (#4879)
+  - also ignore errors raised during test step when `--ignore-test-failure` is used (#4881)
+  - fix download progress bar (#4885)
+  - fix `--dep-graph` by using `graphviz` Python package (#4891)
+- fixes for test suite:
+  - fix `test_github_preview_pr` which got broken because there's no more easyconfigs for bzip2 1.0.6 in easyconfigs repo (#4827)
+  - fix tests after removal of 5.0.x branch (#4830)
+  - fix missed message in trivial equal-asserts (#4831)
+  - fix wrong variable name used for Python version in linting CI (#4839)
+  - fix failing GitHub integration CI tests (#4841)
+  - fix testsuite badge in README (#4845)
+- other changes:
+  - go back to using `develop` branch rather than `5.0.x` branch in GitHub Actions workflows (#4820)
+  - remove source tarball for Python 3.7.2 from test sources (#4828)
+  - delete .coveragerc (#4833)
+  - remove Python 2 constructs (#4834)
+  - remove superflous assignment in `_sanity_check_step` (#4851)
+  - replace deprecation warning about reproducible tarballs on Python older than 3.9 with a regular warning (#4852)
+  - move `EasyBlock.expand_module_search_path` into `ModuleEnvironmentVariable.expand_paths` (#4859)
+
+
 v5.0.0 (18 March 2025)
 ----------------------
 
+major release (includes breaking changes)
+
 - remove support for Python 2.7 and 3.5 (#4229, #4270, #4306, #4473, #4477, #4476, #4478, #4524, #4607, #4756, #4810, #4811) 
   - also run unit test suite with Python 3.12 + 3.13 (#4484, #4674)
 - changed defaults in EasyBuild configuration:

easybuild/_deprecated.py

@@ -31,6 +31,7 @@
 """
 import contextlib
 import functools
+import json
 import os
 import re
 import signal
@@ -96,6 +97,12 @@ def cache_aware_func(cmd, *args, **kwargs):
     return cache_aware_func
 
 
+def json_loads(body):
+    """Deprecated wrapper for json.loads"""
+    _log.deprecated("json_loads is deprecated, use json.loads", '6.0')
+    return json.loads(body)
+
+
 def get_output_from_process(proc, read_size=None, asynchronous=False, print_deprecation_warning=True):
     """
     Get output from running process (that was opened with subprocess.Popen).

easybuild/base/fancylogger.py

@@ -578,8 +578,7 @@ def logToFile(filename, enable=True, filehandler=None, name=None, max_bytes=MAX_
         'maxBytes': max_bytes,
         'backupCount': backup_count,
     }
-    if sys.version_info[0] >= 3:
-        handleropts['encoding'] = 'utf-8'
+    handleropts['encoding'] = 'utf-8'
     # logging to a file is going to create the file later on, so let's try to be helpful and create the path if needed
     directory = os.path.dirname(filename)
     if not os.path.exists(directory):

easybuild/framework/easyblock.py

@@ -98,9 +98,9 @@
 from easybuild.tools.filetools import is_sha256_checksum, mkdir, move_file, move_logs, read_file, remove_dir
 from easybuild.tools.filetools import remove_file, remove_lock, symlink, verify_checksum, weld_paths, write_file
 from easybuild.tools.hooks import (
-    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP, MODULE_STEP,
-    MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP, READY_STEP,
-    SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
+    BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EASYBLOCK, EXTENSIONS_STEP, EXTRACT_STEP, FETCH_STEP, INSTALL_STEP,
+    MODULE_STEP, MODULE_WRITE, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP, PREPARE_STEP,
+    READY_STEP, SANITYCHECK_STEP, SINGLE_EXTENSION, TEST_STEP, TESTCASES_STEP, load_hooks, run_hook,
 )
 from easybuild.tools.run import RunShellCmdError, raise_run_shell_cmd_error, run_shell_cmd
 from easybuild.tools.jenkins import write_to_xml
@@ -311,6 +311,9 @@ def __init__(self, ec, logfile=None):
         # initialize logger
         self._init_log()
 
+        # number of iterations
+        self.iter_cnt = self.det_iter_cnt()
+
         # try and use the specified group (if any)
         group_name = build_option('group')
         group_spec = self.cfg['group']
@@ -448,7 +451,7 @@ def get_checksum_for(self, checksums, filename=None, index=None):
         if checksum and chksum_input_git is not None:
             # ignore any checksum for given filename due to changes in https://github.com/python/cpython/issues/90021
             # tarballs made for git repos are not reproducible when created with Python < 3.9
-            if sys.version_info[0] >= 3 and sys.version_info[1] < 9:
+            if sys.version_info < (3, 9):
                 print_warning(
                     "Reproducible tarballs of Git repos are only possible when using Python 3.9+ to run EasyBuild. "
                     f"Skipping checksum verification of {chksum_input} since Python < 3.9 is used."
@@ -2387,7 +2390,8 @@ def handle_iterate_opts(self):
                     self.log.debug("Iterating opt %s: %s", opt, self.iter_opts[opt])
 
             if self.iter_opts:
-                print_msg("starting iteration #%s ..." % self.iter_idx, log=self.log, silent=self.silent)
+                print_msg(f"starting iteration {self.iter_idx + 1}/{self.iter_cnt} ...", log=self.log,
+                          silent=self.silent)
                 self.log.info("Current iteration index: %s", self.iter_idx)
 
             # pop first element from all iterative easyconfig parameters as next value to use
@@ -2429,7 +2433,7 @@ def det_iter_cnt(self):
 
         # we need to take into account that builddependencies is always a list
         # we're only iterating over it if it's a list of lists
-        builddeps = self.cfg['builddependencies']
+        builddeps = self.cfg.get_ref('builddependencies')
         if all(isinstance(x, list) for x in builddeps):
             iter_opt_counts.append(len(builddeps))
 
@@ -2830,8 +2834,6 @@ def patch_step(self, beginpath=None, patches=None):
             self.log.info("Applying patch %s" % patch['name'])
             trace_msg("applying patch %s" % patch['name'])
 
-            # patch source at specified index (first source if not specified)
-            srcind = patch.get('source', 0)
             # if patch level is specified, use that (otherwise let apply_patch derive patch level)
             level = patch.get('level', None)
             # determine suffix of source path to apply patch in (if any)
@@ -2840,16 +2842,14 @@ def patch_step(self, beginpath=None, patches=None):
             copy_patch = 'copy' in patch and 'sourcepath' not in patch
             options = patch.get('opts', None)  # Extra options for patch command
 
-            self.log.debug("Source index: %s; patch level: %s; source path suffix: %s; copy patch: %s; options: %s",
-                           srcind, level, srcpathsuffix, copy_patch, options)
+            self.log.debug("Patch level: %s; source path suffix: %s; copy patch: %s; options: %s",
+                           level, srcpathsuffix, copy_patch, options)
 
             if beginpath is None:
-                try:
-                    beginpath = self.src[srcind]['finalpath']
-                    self.log.debug("Determine begin path for patch %s: %s" % (patch['name'], beginpath))
-                except IndexError as err:
-                    raise EasyBuildError("Can't apply patch %s to source at index %s of list %s: %s",
-                                         patch['name'], srcind, self.src, err)
+                if not self.src:
+                    raise EasyBuildError("Can't apply patch %s to source if no sources are given", patch['name'])
+                beginpath = self.src[0]['finalpath']
+                self.log.debug("Determined begin path for patch %s: %s" % (patch['name'], beginpath))
             else:
                 self.log.debug("Using specified begin path for patch %s: %s" % (patch['name'], beginpath))
 
@@ -4662,18 +4662,19 @@ def run_step(self, step, step_methods):
         run_hook(step, self.hooks, pre_step_hook=True, args=[self])
 
         for step_method in step_methods:
+            # step_method is a lambda function that takes an EasyBlock instance as an argument,
+            # and returns the actual method
+            current_method = step_method(self)
             # Remove leading underscore from e.g. "_test_step"
-            method_name = '_'.join(step_method.__code__.co_names).lstrip('_')
+            method_name = current_method.__name__.lstrip('_')
             self.log.info("Running method %s part of step %s", method_name, step)
 
             if self.dry_run:
                 self.dry_run_msg("[%s method]", method_name)
 
                 # if an known possible error occurs, just report it and continue
                 try:
-                    # step_method is a lambda function that takes an EasyBlock instance as an argument,
-                    # and returns the actual method, so use () to execute it
-                    step_method(self)()
+                    current_method()
                 except Exception as err:
                     if build_option('extended_dry_run_ignore_errors'):
                         dry_run_warning("ignoring error %s" % err, silent=self.silent)
@@ -4682,9 +4683,7 @@ def run_step(self, step, step_methods):
                         raise
                 self.dry_run_msg('')
             else:
-                # step_method is a lambda function that takes an EasyBlock instance as an argument,
-                # and returns the actual method, so use () to execute it
-                step_method(self)()
+                current_method()
 
         run_hook(step, self.hooks, post_step_hook=True, args=[self])
 
@@ -4794,7 +4793,7 @@ def run_all_steps(self, run_test_cases):
         if self.cfg['stop'] == 'cfg':
             return True
 
-        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.det_iter_cnt())
+        steps = self.get_steps(run_test_cases=run_test_cases, iteration_count=self.iter_cnt)
 
         # figure out how many steps will actually be run (not be skipped)
         step_cnt = 0
@@ -4927,7 +4926,7 @@ def copy_build_dirs_logs_failed_install(application_log, silent, app, easyconfig
             msg = f"Build directory of failed installation copied to {build_dirs_path}"
 
             def operation(src, dest):
-                copy_dir(src, dest, dirs_exist_ok=True)
+                copy_dir(src, dest, dirs_exist_ok=True, symlinks=True)
 
             operation_args.append((operation, [app.builddir], build_dirs_path, msg))
 
@@ -5001,6 +5000,8 @@ def build_and_install_one(ecdict, init_env):
         _log.debug("Skip set to %s" % skip)
         app.cfg['skip'] = skip
 
+    hooks = load_hooks(build_option('hooks'))
+
     # build easyconfig
     error_msg = '(no error)'
     exit_code = None
@@ -5022,6 +5023,8 @@ def build_and_install_one(ecdict, init_env):
             else:
                 enabled_write_permissions = False
 
+        run_hook(EASYBLOCK, hooks, pre_step_hook=True, args=[app])
+
         result = app.run_all_steps(run_test_cases=run_test_cases)
 
         if not dry_run:
@@ -5033,7 +5036,9 @@ def build_and_install_one(ecdict, init_env):
             except EasyBuildError as err:
                 _log.warning("Failed to create build environment dump for easyconfig %s: %s", reprod_spec, err)
 
-            # also add any extension easyblocks used during the build for reproducibility
+            # also add any component/extension easyblocks used during the build for reproducibility
+            if hasattr(app, 'comp_instances'):
+                copy_easyblocks_for_reprod([comp for cfg, comp in app.comp_instances], reprod_dir)
             if app.ext_instances:
                 copy_easyblocks_for_reprod(app.ext_instances, reprod_dir)
             # If not already done remove the granted write permissions if we did so
@@ -5120,8 +5125,12 @@ def ensure_writable_log_dir(log_dir):
                     block = det_full_ec_version(app.cfg) + ".block"
                     repo.add_easyconfig(ecdict['original_spec'], app.name, block, buildstats, currentbuildstats)
                 repo.add_easyconfig(spec, app.name, det_full_ec_version(app.cfg), buildstats, currentbuildstats)
-                for patch in app.patches:
-                    repo.add_patch(patch['path'], app.name)
+                patches = app.patches
+                for ext in app.exts:
+                    patches += ext.get('patches', [])
+                for patch in patches:
+                    if 'path' in patch:
+                        repo.add_patch(patch['path'], app.name)
                 repo.commit("Built %s" % app.full_mod_name)
                 del repo
             except EasyBuildError as err:
@@ -5143,10 +5152,14 @@ def ensure_writable_log_dir(log_dir):
                 _log.debug("Copied easyconfig file %s to %s", spec, newspec)
 
                 # copy patches
-                for patch in app.patches:
-                    target = os.path.join(new_log_dir, os.path.basename(patch['path']))
-                    copy_file(patch['path'], target)
-                    _log.debug("Copied patch %s to %s", patch['path'], target)
+                patches = app.patches
+                for ext in app.exts:
+                    patches += ext.get('patches', [])
+                for patch in patches:
+                    if 'path' in patch:
+                        target = os.path.join(new_log_dir, os.path.basename(patch['path']))
+                        copy_file(patch['path'], target)
+                        _log.debug("Copied patch %s to %s", patch['path'], target)
 
                 if build_option('read_only_installdir') and not app.cfg['stop']:
                     # take away user write permissions (again)
@@ -5200,6 +5213,8 @@ def ensure_writable_log_dir(log_dir):
     if not success:
         copy_build_dirs_logs_failed_install(application_log, silent, app, ecdict['ec'])
 
+    run_hook(EASYBLOCK, hooks, post_step_hook=True, args=[app])
+
     del app
 
     return (success, application_log, error_msg, exit_code)

easybuild/framework/easyconfig/easyconfig.py

@@ -1968,17 +1968,22 @@ def asdict(self):
                 res[key] = value
         return res
 
-    def get_cuda_cc_template_value(self, key):
+    def get_cuda_cc_template_value(self, key, required=True):
         """
         Get template value based on --cuda-compute-capabilities EasyBuild configuration option
         and cuda_compute_capabilities easyconfig parameter.
         Returns user-friendly error message in case neither are defined,
         or if an unknown key is used.
+
+        :param required: If False and the key is not found, return an empty string instead of raising an error.
         """
         if key.startswith('cuda_') and any(x == key for x in TEMPLATE_NAMES_DYNAMIC):
             try:
                 return self.template_values[key]
             except KeyError:
+                if not required:
+                    self.log.debug(f'Key {key} not found in template values, returning empty value')
+                    return ''
                 error_msg = "Template value '%s' is not defined!\n"
                 error_msg += "Make sure that either the --cuda-compute-capabilities EasyBuild configuration "
                 error_msg += "option is set, or that the cuda_compute_capabilities easyconfig parameter is defined."

easybuild/scripts/findPythonDeps.py

@@ -85,8 +85,7 @@ def can_run(cmd, *arguments):
 def run_shell_cmd(arguments, action_desc, capture_stderr=True, **kwargs):
     """Run the command and return the return code and output"""
     extra_args = kwargs or {}
-    if sys.version_info[0] >= 3:
-        extra_args['universal_newlines'] = True
+    extra_args['universal_newlines'] = True
     stderr = subprocess.STDOUT if capture_stderr else subprocess.PIPE
     p = subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=stderr, **extra_args)
     out, err = p.communicate()