diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 1e6f235282..3b9ac1604a 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -56,6 +56,7 @@ import time import traceback from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager from datetime import datetime from string import ascii_letters from textwrap import indent @@ -1261,9 +1262,6 @@ def make_devel_module(self, create_in_builddir=False): self.log.info("Making devel module...") - # load fake module - fake_mod_data = self.load_fake_module(purge=True) - header = self.module_generator.MODULE_SHEBANG if header: header += '\n' @@ -1307,9 +1305,6 @@ def make_devel_module(self, create_in_builddir=False): txt = ''.join([header] + load_lines + env_lines) write_file(filename, txt) - # cleanup: unload fake module, remove fake module dir - self.clean_up_fake_module(fake_mod_data) - def make_module_deppaths(self): """ Add specific 'module use' actions to module file, in order to find @@ -1639,7 +1634,7 @@ def make_module_group_check(self): return txt - def make_module_req(self, fake=False): + def make_module_req(self): """ Generate the environment-variables required to run the module. """ @@ -1680,11 +1675,10 @@ def make_module_req(self, fake=False): mod_lines.append(self.module_generator.comment(note)) for env_var, search_paths in env_var_requirements.items(): - if self.dry_run or fake: + if self.dry_run: # Don't expand globs or do any filtering for dry run mod_req_paths = search_paths - if self.dry_run: - self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}") + self.dry_run_msg(f" ${env_var}:{', '.join(mod_req_paths)}") else: mod_req_paths = [ expanded_path for unexpanded_path in search_paths @@ -1881,7 +1875,6 @@ def load_fake_module(self, purge=False, extra_modules=None, verbose=False): # load fake module self.modules_tool.prepend_module_path(os.path.join(fake_mod_path, self.mod_subdir), priority=10000) self.load_module(purge=purge, extra_modules=extra_modules, verbose=verbose) - return (fake_mod_path, env) def clean_up_fake_module(self, fake_mod_data): @@ -1965,10 +1958,11 @@ def skip_extensions(self): if not exts_filter or len(exts_filter) == 0: raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig") - if build_option('parallel_extensions_install'): - self.skip_extensions_parallel(exts_filter) - else: - self.skip_extensions_sequential(exts_filter) + with self.fake_module_environment(): + if build_option('parallel_extensions_install'): + self.skip_extensions_parallel(exts_filter) + else: + self.skip_extensions_sequential(exts_filter) def skip_extensions_sequential(self, exts_filter): """ @@ -2096,31 +2090,27 @@ def install_extensions_sequential(self, install=True): msg = "\n* installing extension %s %s using '%s' easyblock\n" % tup self.dry_run_msg(msg) - self.log.debug("List of loaded modules: %s", self.modules_tool.list()) - - # prepare toolchain build environment, but only when not doing a dry run - # since in that case the build environment is the same as for the parent if self.dry_run: self.dry_run_msg("defining build environment based on toolchain (options) and dependencies...") - else: - # don't reload modules for toolchain, there is no need since they will be loaded already; - # the (fake) module for the parent software gets loaded before installing extensions - ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs, - rpath_include_dirs=self.rpath_include_dirs, - rpath_wrappers_dir=self.rpath_wrappers_dir) # actual installation of the extension - if install: - try: - ext.install_extension_substep("pre_install_extension") - with self.module_generator.start_module_creation(): - txt = ext.install_extension_substep("install_extension") - if txt: - self.module_extra_extensions += txt - ext.install_extension_substep("post_install_extension") - finally: - if not self.dry_run: + if install and not self.dry_run: + with self.fake_module_environment(with_build_deps=True): + self.log.debug("List of loaded modules: %s", self.modules_tool.list()) + # don't reload modules for toolchain, there is no need + # since they will be loaded already by the fake module + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs, + rpath_include_dirs=self.rpath_include_dirs, + rpath_wrappers_dir=self.rpath_wrappers_dir) + try: + ext.install_extension_substep("pre_install_extension") + with self.module_generator.start_module_creation(): + txt = ext.install_extension_substep("install_extension") + if txt: + self.module_extra_extensions += txt + ext.install_extension_substep("post_install_extension") + finally: ext_duration = datetime.now() - start_time if ext_duration.total_seconds() >= 1: print_msg("\t... (took %s)", time2str(ext_duration), log=self.log, silent=self.silent) @@ -2267,17 +2257,18 @@ def update_exts_progress_bar_helper(running_exts, progress_size): tup = (ext.name, ext.version or '') print_msg("starting installation of extension %s %s..." % tup, silent=self.silent, log=self.log) - # don't reload modules for toolchain, there is no need since they will be loaded already; - # the (fake) module for the parent software gets loaded before installing extensions - ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, - rpath_filter_dirs=self.rpath_filter_dirs, - rpath_include_dirs=self.rpath_include_dirs, - rpath_wrappers_dir=self.rpath_wrappers_dir) - if install: - ext.install_extension_substep("pre_install_extension") - ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool) - running_exts.append(ext) - self.log.info(f"Started installation of extension {ext.name} in the background...") + if install and not self.dry_run: + with self.fake_module_environment(with_build_deps=True): + # don't reload modules for toolchain, there is no + # need since they will be loaded by the fake module + ext.toolchain.prepare(onlymod=self.cfg['onlytcmod'], silent=True, loadmod=False, + rpath_filter_dirs=self.rpath_filter_dirs, + rpath_include_dirs=self.rpath_include_dirs, + rpath_wrappers_dir=self.rpath_wrappers_dir) + ext.install_extension_substep("pre_install_extension") + ext.async_cmd_task = ext.install_extension_substep("install_extension_async", thread_pool) + running_exts.append(ext) + self.log.info(f"Started installation of extension {ext.name} in the background...") update_exts_progress_bar_helper(running_exts, 0) # print progress info after every iteration (unless that info is already shown via progress bar) @@ -2301,6 +2292,25 @@ def start_dir(self): """Start directory in build directory""" return self.cfg['start_dir'] + @contextmanager + def fake_module_environment(self, extra_modules=None, with_build_deps=False): + """ + Load/Unload fake module + """ + fake_mod_data = None + + if with_build_deps: + # load modules for build dependencies as extra modules + extra_modules = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)] + + fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules) + + yield + + # cleanup (unload fake module, remove fake module dir) + if fake_mod_data: + self.clean_up_fake_module(fake_mod_data) + def guess_start_dir(self): """ Return the directory where to start the whole configure/make/make install cycle from @@ -3108,14 +3118,9 @@ def extensions_step(self, fetch=False, install=True): self.log.debug("No extensions in exts_list") return - # load fake module - fake_mod_data = None - if install and not self.dry_run: - - # load modules for build dependencies as extra modules - build_dep_mods = [dep['short_mod_name'] for dep in self.cfg.dependencies(build_only=True)] - - fake_mod_data = self.load_fake_module(purge=True, extra_modules=build_dep_mods) + # we really need a default class + if not self.cfg['exts_defaultclass'] and install: + raise EasyBuildError("ERROR: No default extension class set for %s", self.name) start_progress_bar(PROGRESS_BAR_EXTENSIONS, len(self.cfg.get_ref('exts_list'))) @@ -3131,11 +3136,6 @@ def extensions_step(self, fetch=False, install=True): if install: self.log.info("Installing extensions") - # we really need a default class - if not self.cfg['exts_defaultclass'] and fake_mod_data: - self.clean_up_fake_module(fake_mod_data) - raise EasyBuildError("ERROR: No default extension class set for %s", self.name) - self.init_ext_instances() if self.skip: @@ -3143,10 +3143,6 @@ def extensions_step(self, fetch=False, install=True): self.install_all_extensions(install=install) - # cleanup (unload fake module, remove fake module dir) - if fake_mod_data: - self.clean_up_fake_module(fake_mod_data) - stop_progress_bar(PROGRESS_BAR_EXTENSIONS, visible=False) def package_step(self): @@ -4056,7 +4052,7 @@ def make_module_step(self, fake=False): txt += self.make_module_deppaths() txt += self.make_module_dep() txt += self.make_module_extend_modpath() - txt += self.make_module_req(fake=fake) + txt += self.make_module_req() txt += self.make_module_extra() txt += self.make_module_footer() @@ -4100,13 +4096,14 @@ def make_module_step(self, fake=False): self.module_generator.create_symlinks(mod_symlink_paths, fake=fake) if ActiveMNS().mns.det_make_devel_module() and not fake and build_option('generate_devel_module'): - try: - self.make_devel_module() - except EasyBuildError as error: - if build_option('module_only') or self.cfg['module_only']: - self.log.info("Using --module-only so can recover from error: %s", error) - else: - raise error + with self.fake_module_environment(): + try: + self.make_devel_module() + except EasyBuildError as error: + if build_option('module_only') or self.cfg['module_only']: + self.log.info("Using --module-only so can recover from error: %s", error) + else: + raise error else: self.log.info("Skipping devel module...") diff --git a/easybuild/framework/extensioneasyblock.py b/easybuild/framework/extensioneasyblock.py index b85b97265a..9fb7f1563c 100644 --- a/easybuild/framework/extensioneasyblock.py +++ b/easybuild/framework/extensioneasyblock.py @@ -176,29 +176,22 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands # make sure Extension sanity check step is run once, by using a single empty list of extra modules lists_of_extra_modules = [[]] - for extra_modules in lists_of_extra_modules: - - fake_mod_data = None - - # only load fake module + extra modules for stand-alone installations (not for extensions), - # since for extension the necessary modules should already be loaded at this point; - # take into account that module may already be loaded earlier in sanity check - if not (self.sanity_check_module_loaded or self.is_extension or self.dry_run): - # load fake module - fake_mod_data = self.load_fake_module(purge=True, extra_modules=extra_modules) - - if extra_modules: - info_msg = "Running extension sanity check with extra modules: %s" % ', '.join(extra_modules) - self.log.info(info_msg) - trace_msg(info_msg) - - # perform extension sanity check + # only load fake module + extra modules for stand-alone installations (not for extensions), + # since for extension the necessary modules should already be loaded at this point; + # take into account that module may already be loaded earlier in sanity check + if not (self.sanity_check_module_loaded or self.is_extension or self.dry_run): + for extra_modules in lists_of_extra_modules: + with self.fake_module_environment(extra_modules=extra_modules): + if extra_modules: + info_msg = f"Running extension sanity check with extra modules: {', '.join(extra_modules)}" + self.log.info(info_msg) + trace_msg(info_msg) + # perform sanity check for stand-alone extension + (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self) + else: + # perform single sanity check for extension (sanity_check_ok, fail_msg) = Extension.sanity_check_step(self) - if fake_mod_data: - # unload fake module and clean up - self.clean_up_fake_module(fake_mod_data) - if custom_paths or custom_commands or not self.is_extension: super().sanity_check_step(custom_paths=custom_paths, custom_commands=custom_commands, diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 0cbefe3387..df06fb0ab0 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -1318,7 +1318,6 @@ def test_extensions_step_deprecations(self): 'description = "test easyconfig"', 'toolchain = SYSTEM', 'exts_defaultclass = "DummyExtension"', - 'exts_list = ["ext1"]', 'exts_list = [', ' "dummy_ext",', ' ("custom_ext", "0.0", {"easyblock": "CustomDummyExtension"}),', @@ -1410,7 +1409,7 @@ def test_extension_source_tmpl(self): "toolchain = SYSTEM", "exts_list = [", " ('bar', '0.0', {", - " 'source_tmpl': [SOURCE_TAR_GZ],", + " 'source_tmpl': [SOURCE_TAR_GZ],", " }),", "]", ]) @@ -1484,6 +1483,46 @@ def test_skip_extensions_step(self): eb.close_log() os.remove(eb.logfile) + def test_extension_fake_modules(self): + """ + Test that extensions relying on installation files from previous extensions work + Search paths of fake module should update for each extension and resolve any globs + """ + self.contents = cleandoc(""" + easyblock = 'ConfigureMake' + name = 'toy' + version = '0.0' + homepage = 'https://example.com' + description = 'test' + toolchain = SYSTEM + exts_list = [ + ('bar', '0.0', { + 'postinstallcmds': [ + 'mkdir -p %(installdir)s/custom_bin', + 'touch %(installdir)s/custom_bin/bar.sh', + 'chmod +x %(installdir)s/custom_bin/bar.sh', + ], + }), + ('barbar', '0.0', { + 'postinstallcmds': ['bar.sh'], + }), + ] + exts_defaultclass = "DummyExtension" + modextrapaths = {'PATH': 'custom*'} + """) + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.builddir = config.build_path() + eb.installdir = config.install_path() + + self.mock_stdout(True) + eb.extensions_step(fetch=True) + stdout = self.get_stdout() + self.mock_stdout(False) + + pattern = r">> running shell command:\n\s+bar.sh(\n\s+\[.*\]){3}\n\s+>> command completed: exit 0" + self.assertTrue(re.search(pattern, stdout, re.M)) + def test_make_module_step(self): """Test the make_module_step""" @@ -2483,6 +2522,7 @@ def test_extensions_sanity_check(self): exts_list = toy_ec['exts_list'] exts_list[-1][2]['exts_filter'] = ("thisshouldfail", '') toy_ec['exts_list'] = exts_list + toy_ec['exts_defaultclass'] = 'DummyExtension' eb = EB_toy(toy_ec) eb.silent = True @@ -2497,6 +2537,7 @@ def test_extensions_sanity_check(self): # sanity check commands are checked after checking sanity check paths, so this should work toy_ec = EasyConfig(toy_ec_fn) toy_ec.update('sanity_check_commands', [("%(installdir)s/bin/toy && rm %(installdir)s/bin/toy", '')]) + toy_ec['exts_defaultclass'] = 'DummyExtension' eb = EB_toy(toy_ec) eb.silent = True with self.mocked_stdout_stderr(): diff --git a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb index 8d5b7b7f99..d84ab4ad0b 100644 --- a/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb +++ b/test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-test.eb @@ -27,6 +27,7 @@ local_bar_buildopts = " && gcc bar.c -o anotherbar && " # used to check whether $TOY_LIBS_PATH is defined even when 'lib' subdirectory doesn't exist yet local_bar_buildopts += 'echo "TOY_EXAMPLES=$TOY_EXAMPLES" > %(installdir)s/toy_libs_path.txt' +exts_defaultclass = 'DummyExtension' exts_list = [ 'ulimit', # extension that is part of "standard library" ('bar', '0.0', { @@ -59,7 +60,7 @@ sanity_check_paths = { 'dirs': [], } -modextrapaths = {'TOY_EXAMPLES': 'examples'} +modextravars = {'TOY_EXAMPLES': 'examples'} postinstallcmds = ["echo TOY > %(installdir)s/README"] diff --git a/test/framework/options.py b/test/framework/options.py index bd4f4918c5..5f18900d63 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -6731,6 +6731,7 @@ def test_sanity_check_only(self): test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + "exts_defaultclass = 'DummyExtension'", "exts_list = [", " ('barbar', '0.0', {", " 'start_dir': 'src',", diff --git a/test/framework/toy_build.py b/test/framework/toy_build.py index 5f02ea2f1e..a02c633e73 100644 --- a/test/framework/toy_build.py +++ b/test/framework/toy_build.py @@ -1224,7 +1224,7 @@ def test_toy_advanced(self): toy_libs_path = os.path.join(toy_installdir, 'toy_libs_path.txt') self.assertTrue(os.path.exists(toy_libs_path)) txt = read_file(toy_libs_path) - regex = re.compile('^TOY_EXAMPLES=.*/examples$') + regex = re.compile('^TOY_EXAMPLES=examples$') self.assertTrue(regex.match(txt), f"Pattern '{regex.pattern}' should match in: {txt}") def test_toy_advanced_filter_deps(self): @@ -1357,6 +1357,7 @@ def test_toy_extension_patches_postinstallcmds(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "buildopts": " && ls -l test.txt",', @@ -1405,6 +1406,7 @@ def test_toy_extension_sources(self): # test use of single-element list in 'sources' with just the filename test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "sources": %s,' % bar_sources_spec, @@ -1432,6 +1434,7 @@ def test_toy_extension_sources(self): test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1447,6 +1450,7 @@ def test_toy_extension_sources(self): # check that checksums are picked up and verified test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1470,6 +1474,7 @@ def test_toy_extension_sources(self): # test again with correct checksum for bar-0.0.tar.gz, but faulty checksum for patch file test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1493,6 +1498,7 @@ def test_toy_extension_sources(self): # test again with correct checksums test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', ' "source_urls": ["file://%s"],' % test_source_path, @@ -1518,6 +1524,7 @@ def test_toy_extension_extract_cmd(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = '\n'.join([ toy_ec_txt, + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("bar", "0.0", {', # deliberately incorrect custom extract command, just to verify that it's picked up @@ -1556,6 +1563,7 @@ def test_toy_extension_sources_git_config(self): test_ec_txt = '\n'.join([ toy_ec_txt, 'prebuildopts = "echo \\\"%s\\\" > %s && ",' % (ext_code, ext_cfile), + 'exts_defaultclass = "DummyExtension"', 'exts_list = [', ' ("exts-git", "0.0", {', ' "buildopts": "&& ls -l %s %s",' % (ext_tarball, ext_tarfile), @@ -1924,6 +1932,7 @@ def test_module_only_extensions(self): test_ec_txt += '\n' + '\n'.join([ "sanity_check_commands = ['barbar', 'toy']", "sanity_check_paths = {'files': ['bin/barbar', 'bin/toy'], 'dirs': ['bin']}", + "exts_defaultclass = 'DummyExtension'", "exts_list = [", " ('barbar', '0.0', {", " 'start_dir': 'src',", @@ -1993,6 +2002,7 @@ def test_toy_exts_parallel(self): test_ec = os.path.join(self.test_prefix, 'test.eb') test_ec_txt = read_file(toy_ec) test_ec_txt += '\n' + '\n'.join([ + "exts_defaultclass = 'DummyExtension'", "exts_list = [", " ('ls'),", " ('bar', '0.0'),", @@ -2373,6 +2383,7 @@ def test_reproducibility_ext_easyblocks(self): ec1 = os.path.join(self.test_prefix, 'toy1.eb') ec1_txt = '\n'.join([ toy_ec_txt, + "exts_defaultclass = 'DummyExtension'", "exts_list = [('barbar', '1.2', {'start_dir': 'src'})]", "", ]) @@ -3220,7 +3231,10 @@ def test_toy_build_hooks(self): """Test use of --hooks.""" toy_ec = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') test_ec = os.path.join(self.test_prefix, 'test.eb') - test_ec_txt = read_file(toy_ec) + "\nexts_list = [('bar', '0.0'), ('toy', '0.0')]" + test_ec_txt = read_file(toy_ec) + '\n'.join([ + "exts_list = [('bar', '0.0'), ('toy', '0.0')]", + "exts_defaultclass = 'DummyExtension'", + ]) write_file(test_ec, test_ec_txt) hooks_file = os.path.join(self.test_prefix, 'my_hooks.py') @@ -3328,14 +3342,15 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs): ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! in post-install hook for toy v0.0 bin, lib - in module-write hook hook for {mod_name} toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy toy 0.0 ['%(name)s-%(version)s.tar.gz'] echo toy + in module-write hook hook for {mod_name} installing of extension bar is done! + in module-write hook hook for {mod_name} pre_run_shell_cmd_hook triggered for ' gcc toy.c -o toy ' ' gcc toy.c -o toy && copy_toy_file toy copy_of_toy' command failed (exit code 127), but I fixed it! installing of extension toy is done! @@ -3364,6 +3379,7 @@ def test_toy_multi_deps(self): test_ec = os.path.join(self.test_prefix, 'test.eb') # also inject (minimal) list of extensions to test iterative installation of extensions + test_ec_txt += "\nexts_defaultclass = 'DummyExtension'" test_ec_txt += "\nexts_list = [('barbar', '1.2', {'start_dir': 'src'})]" test_ec_txt += "\nmulti_deps = {'GCC': ['4.6.3', '7.3.0-2.30']}" @@ -3472,6 +3488,7 @@ def check_toy_load(depends_on=False): loaded_mod_names = [x['mod_name'] for x in self.modtool.list()] self.assertNotIn('toy/0.0', loaded_mod_names) self.assertIn('GCC/7.3.0-2.30', loaded_mod_names) + self.modtool.unload(['GCC/7.3.0-2.30']) else: # just undo self.modtool.unload(['toy/0.0', 'GCC/7.3.0-2.30'])