From cffc2b8a9548c3f6b1fa5518be7a30428119cc51 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 25 Sep 2024 00:29:06 +0200 Subject: [PATCH 01/15] add option module-search-path-headers to control how modules set search paths to header files --- easybuild/framework/easyconfig/default.py | 7 ++++--- easybuild/tools/config.py | 12 ++++++++++++ easybuild/tools/options.py | 4 ++++ 3 files changed, 20 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 4b2c7a0a1d..fb942109e3 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -208,10 +208,11 @@ 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' '(implies recursive unloading of modules) [DEPRECATED]', MODULES], + 'module_search_path_headers': [False, "Environment variable set by modules on load with search paths " + "to header files", MODULES], 'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module " - "(True/False to hard enable/disable; None implies honoring " - "the --recursive-module-unload EasyBuild configuration setting", - MODULES], + "(True/False to hard enable/disable; None implies honoring the " + "--recursive-module-unload EasyBuild configuration setting", MODULES], # MODULES documentation easyconfig parameters # (docurls is part of MANDATORY) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index ecce1a7f0d..0c793a71da 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -182,6 +182,17 @@ EBPYTHONPREFIXES = 'EBPYTHONPREFIXES' PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES] +# modes to handle header search paths in environment of modules +MOD_SEARCH_PATH_HEADERS_NONE = "none" +MOD_SEARCH_PATH_HEADERS_CPATH = "CPATH" +MOD_SEARCH_PATH_HEADERS_INCLUDE = "INCLUDE_PATHS" +MOD_SEARCH_PATH_HEADERS = { + MOD_SEARCH_PATH_HEADERS_NONE: [], + MOD_SEARCH_PATH_HEADERS_CPATH: ["CPATH"], + MOD_SEARCH_PATH_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], +} +DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH + class Singleton(ABCMeta): """Serves as metaclass for classes that should implement the Singleton pattern. @@ -307,6 +318,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'logtostdout', 'minimal_toolchains', 'module_only', + 'module_search_path_headers', 'package', 'parallel_extensions_install', 'read_only_installdir', diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 9c4ad3c734..de415501f6 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -69,6 +69,7 @@ from easybuild.tools.config import DEFAULT_JOB_EB_CMD, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL +from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import DEFAULT_PKG_TYPE, DEFAULT_PNS, DEFAULT_PREFIX, DEFAULT_EXTRA_SOURCE_URLS from easybuild.tools.config import DEFAULT_REPOSITORY, DEFAULT_WAIT_ON_LOCK_INTERVAL, DEFAULT_WAIT_ON_LOCK_LIMIT from easybuild.tools.config import DEFAULT_PR_TARGET_ACCOUNT, DEFAULT_FILTER_RPATH_SANITY_LIBS @@ -615,6 +616,9 @@ def config_options(self): 'module-extensions': ("Include 'extensions' statement in generated module file (Lua syntax only)", None, 'store_true', True), 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), + 'module-search-path-headers': ("Environment variable set by modules on load with search paths " + "to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS, + [*MOD_SEARCH_PATH_HEADERS]), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), 'moduleclasses': (("Extend supported module classes " From 3159de03fe6115af33e4992a747ceada53a5f115 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 27 Jan 2025 13:39:17 +0100 Subject: [PATCH 02/15] integrate --module-search-path-headers into ModuleLoadEnvironment --- easybuild/framework/easyblock.py | 8 +++- easybuild/tools/config.py | 13 +++--- easybuild/tools/modules.py | 69 +++++++++++++++++++++++++++++--- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index b8007e7cd9..709db30702 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -220,7 +220,13 @@ def __init__(self, ec, logfile=None): self.modules_header = read_file(modules_header_path) # environment variables on module load - self.module_load_environment = ModuleLoadEnvironment() + # apply --module-search-path-headers: easyconfig parameter has precedence + mod_load_cpp_headers = build_option('module_search_path_headers') + cfg_cpp_headers = self.cfg['module_search_path_headers'] + if cfg_cpp_headers is not False: + mod_load_cpp_headers = cfg_cpp_headers + + self.module_load_environment = ModuleLoadEnvironment(cpp_headers=mod_load_cpp_headers) # determine install subdirectory, based on module name self.install_subdir = None diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 0c793a71da..b69085ac7c 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -182,16 +182,13 @@ EBPYTHONPREFIXES = 'EBPYTHONPREFIXES' PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES] -# modes to handle header search paths in environment of modules -MOD_SEARCH_PATH_HEADERS_NONE = "none" -MOD_SEARCH_PATH_HEADERS_CPATH = "CPATH" -MOD_SEARCH_PATH_HEADERS_INCLUDE = "INCLUDE_PATHS" +# options to handle header search paths in environment of modules MOD_SEARCH_PATH_HEADERS = { - MOD_SEARCH_PATH_HEADERS_NONE: [], - MOD_SEARCH_PATH_HEADERS_CPATH: ["CPATH"], - MOD_SEARCH_PATH_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], + "none": [], + "cpath": ["CPATH"], + "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], } -DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH +DEFAULT_MOD_SEARCH_PATH_HEADERS = "cpath" class Singleton(ABCMeta): diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 695133f7f1..496c6b966c 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -46,7 +46,8 @@ from easybuild.base import fancylogger from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning -from easybuild.tools.config import ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE, LOADED_MODULES_ACTIONS, PURGE +from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE +from easybuild.tools.config import LOADED_MODULES_ACTIONS, MOD_SEARCH_PATH_HEADERS, PURGE from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars @@ -246,16 +247,18 @@ def is_path(self): class ModuleLoadEnvironment: """Changes to environment variables that should be made when environment module is loaded""" - def __init__(self): + def __init__(self, cpp_headers=None): """ Initialize default environment definition Paths are relative to root of installation directory + + :cpp_headers: string defining MOD_SEARCH_PATH_HEADERS setting """ + self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')] self.CLASSPATH = ['*.jar'] self.CMAKE_LIBRARY_PATH = ['lib64'] # only needed for installations with standalone lib64 self.CMAKE_PREFIX_PATH = [''] - self.CPATH = SEARCH_PATH_HEADER_DIRS self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS] self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS @@ -264,14 +267,39 @@ def __init__(self): self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']] self.XDG_DATA_DIRS = ['share'] + # handle search paths to C/C++ headers + self._cpp_headers_opt = DEFAULT_MOD_SEARCH_PATH_HEADERS + if cpp_headers is not None and cpp_headers is not False: + self._cpp_headers_opt = str(cpp_headers) + if self._cpp_headers_opt not in MOD_SEARCH_PATH_HEADERS: + raise EasyBuildError( + f"Unknown value selected for option module-search-path-headers: {self._cpp_headers_opt}. " + f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}" + ) + self.set_cpp_headers(SEARCH_PATH_HEADER_DIRS) + def __setattr__(self, name, value): """ Specific restrictions for ModuleLoadEnvironment attributes: + - public attributes are instances of ModuleEnvironmentVariable with uppercase names + - private attributes are allowed with any name + """ + if name.startswith('_'): + # do not control protected/private attributes + return super().__setattr__(name, value) + + return self.__set_module_environment_variable(name, value) + + def __set_module_environment_variable(self, name, value): + """ + Specific restrictions for ModuleEnvironmentVariable attributes: - attribute names are uppercase - - attributes are instances of ModuleEnvironmentVariable + - dictionaries are unpacked into arguments of ModuleEnvironmentVariable + - controls variables with special types (e.g. PATH, LD_LIBRARY_PATH) """ if name != name.upper(): raise EasyBuildError(f"Names of ModuleLoadEnvironment attributes must be uppercase, got '{name}'") + try: (contents, kwargs) = value except ValueError: @@ -286,9 +314,15 @@ def __setattr__(self, name, value): return super().__setattr__(name, ModuleEnvironmentVariable(contents, **kwargs)) + @property + def vars(self): + """Return list of public ModuleEnvironmentVariable""" + + return [envar for envar in self.__dict__ if not str(envar).startswith('_')] + def __iter__(self): """Make the class iterable""" - yield from self.__dict__ + yield from self.vars def items(self): """ @@ -296,7 +330,7 @@ def items(self): - key = attribute name - value = its "contents" attribute """ - for attr in self.__dict__: + for attr in self.vars: yield attr, getattr(self, attr) def update(self, new_env): @@ -325,6 +359,29 @@ def environ(self): mapping.update({envar_name: str(envar_contents)}) return mapping + @property + def name_cpp_headers(self): + """ + Return list of environment variable names holding search paths to CPP headers + According to option --module-search-path-headers + """ + return MOD_SEARCH_PATH_HEADERS[self._cpp_headers_opt] + + @property + def cpp_headers(self): + """ + Return dict with search path variables for C/C++ headers + According to option --module-search-path-headers + """ + return {envar_name: getattr(self, envar_name) for envar_name in self.name_cpp_headers} + + def set_cpp_headers(self, new_headers): + """ + Set search paths variables for C/C++ headers + According to option --module-search-path-headers + """ + for envar_name in self.name_cpp_headers: + setattr(self, envar_name, new_headers) class ModulesTool(object): """An abstract interface to a tool that deals with modules.""" From 1fcf567540171cf0d78b650b1e9b6b240da4045a Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 27 Jan 2025 17:34:09 +0100 Subject: [PATCH 03/15] add unit tests for cpp headers functionality in ModuleLoadEnvironment --- test/framework/modules.py | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/test/framework/modules.py b/test/framework/modules.py index e1f910db62..959e95e820 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1706,8 +1706,10 @@ def test_module_load_environment(self): self.assertEqual(mod_load_env.TEST_VARTYPE.type, mod.ModEnvVarType.PATH) self.assertRaises(TypeError, setattr, mod_load_env, 'TEST_UNKNONW', (test_contents, {'unkown_param': True})) - # test retrieving environment + # test retrieval of environment + # use copy of public attributes as reference ref_load_env = mod_load_env.__dict__.copy() + ref_load_env = {envar: value for envar, value in ref_load_env.items() if not envar.startswith('_')} self.assertCountEqual(list(mod_load_env), ref_load_env.keys()) ref_load_env_item_list = list(ref_load_env.items()) @@ -1738,6 +1740,41 @@ def test_module_load_environment(self): self.assertTrue(hasattr(mod_load_env, 'TEST_STR')) self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path']) + # test functionality related to --module-search-path-headers + default_search_path_headers = 'cpath' + self.assertEqual(mod_load_env._cpp_headers_opt, default_search_path_headers) + mod_load_env = mod.ModuleLoadEnvironment(default_search_path_headers) + self.assertEqual(mod_load_env._cpp_headers_opt, default_search_path_headers) + self.assertEqual(mod_load_env.name_cpp_headers, ['CPATH']) + repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} + self.assertDictEqual(repr_mod_load_env, {'CPATH': 'include'}) + mod_load_env.set_cpp_headers('new_include') + repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} + self.assertDictEqual(repr_mod_load_env, {'CPATH': 'new_include'}) + mod_load_env.set_cpp_headers(["new_include_1", "new_include_2"]) + repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} + self.assertDictEqual(repr_mod_load_env, {'CPATH': 'new_include_1:new_include_2'}) + + mod_load_env = mod.ModuleLoadEnvironment(cpp_headers='include_paths') + self.assertNotEqual(mod_load_env._cpp_headers_opt, default_search_path_headers) + ref_include_vars = ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'] + self.assertEqual(mod_load_env.name_cpp_headers, ref_include_vars) + repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} + ref_cpp_headers = {key: 'include' for key in ref_include_vars} + self.assertDictEqual(repr_mod_load_env, ref_cpp_headers) + mod_load_env.set_cpp_headers('new_include') + repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} + ref_cpp_headers = {key: 'new_include' for key in ref_include_vars} + self.assertDictEqual(repr_mod_load_env, ref_cpp_headers) + + mod_load_env = mod.ModuleLoadEnvironment(cpp_headers='none') + self.assertEqual(mod_load_env.name_cpp_headers, []) + self.assertEqual(mod_load_env.cpp_headers, {}) + mod_load_env.set_cpp_headers('new_include') + self.assertEqual(mod_load_env.cpp_headers, {}) + + error_pattern = "Unknown value selected for option module-search-path-headers" + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, cpp_headers='nonexistent') def suite(): """ returns all the testcases in this module """ From a63bd4a45b53067e01f039597aad946d765702bd Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Wed, 25 Sep 2024 10:38:16 +0200 Subject: [PATCH 04/15] add unit test for option module-search-path-headers --- test/framework/easyblock.py | 94 +++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 51360ae634..532622de62 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -609,6 +609,100 @@ def test_make_module_req(self): eb.close_log() os.remove(eb.logfile) + def test_module_search_path_headers(self): + """Test functionality of module-search-path-headers option""" + sp_headers_mode = { + "none": [], + "cpath": ["CPATH"], + "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], + } + + self.contents = '\n'.join([ + 'easyblock = "ConfigureMake"', + 'name = "pi"', + 'version = "3.14"', + 'homepage = "http://example.com"', + 'description = "test easyconfig"', + 'toolchain = SYSTEM', + ]) + self.writeEC() + + for build_opt, sp_headers in sp_headers_mode.items(): + init_config(build_options={"module_search_path_headers": build_opt, "silent": True}) + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + try: + os.makedirs(os.path.join(eb.installdir, 'include')) + write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file') + except FileExistsError: + pass + + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() + + if not sp_headers: + # none option adds nothing to module file + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$" + self.assertFalse(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$' + self.assertFalse(re.search(lua_ref_pattern, guess, re.M)) + else: + for env_var in sp_headers: + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$" + self.assertTrue(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$' + self.assertTrue(re.search(lua_ref_pattern, guess, re.M)) + + # test with easyconfig parameter + for ec_param, sp_headers in sp_headers_mode.items(): + self.contents += f'\nmodule_search_path_headers = "{ec_param}"' + self.writeEC() + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + try: + os.makedirs(os.path.join(eb.installdir, 'include')) + write_file(os.path.join(eb.installdir, 'include', 'header.h'), 'dummy header file') + except FileExistsError: + pass + + for build_opt in sp_headers_mode: + init_config(build_options={"module_search_path_headers": build_opt, "silent": True}) + with eb.module_generator.start_module_creation(): + guess = eb.make_module_req() + if not sp_headers: + # none option adds nothing to module file + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = r"^prepend-path\s+CPATH\s+\$root/include$" + self.assertFalse(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = r'^prepend_path\("CPATH", pathJoin\(root, "include"\)\)$' + self.assertFalse(re.search(lua_ref_pattern, guess, re.M)) + else: + for env_var in sp_headers: + if get_module_syntax() == 'Tcl': + tcl_ref_pattern = rf"^prepend-path\s+{env_var}\s+\$root/include$" + self.assertTrue(re.search(tcl_ref_pattern, guess, re.M)) + elif get_module_syntax() == 'Lua': + lua_ref_pattern = rf'^prepend_path\("{env_var}", pathJoin\(root, "include"\)\)$' + self.assertTrue(re.search(lua_ref_pattern, guess, re.M)) + + # test wrong easyconfig parameter + self.contents += '\nmodule_search_path_headers = "WRONG_OPT"' + self.writeEC() + ec =EasyConfig(self.eb_file) + + error_pattern = "Unknown value selected for option module-search-path-headers" + with eb.module_generator.start_module_creation(): + self.assertErrorRegex(EasyBuildError, error_pattern, EasyBlock, ec) + + # cleanup + eb.close_log() + os.remove(eb.logfile) + def test_make_module_extra(self): """Test for make_module_extra.""" init_config(build_options={'silent': True}) From beb4aa5373e4c532d9f5a9b76927bf64587bddc3 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 27 Jan 2025 18:11:47 +0100 Subject: [PATCH 05/15] make test_make_module_req aware of cpp_headers in ModuleLoadEnvironment --- test/framework/easyblock.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 532622de62..9856e354b2 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -532,7 +532,7 @@ def test_make_module_req(self): for env_var in default_mod_load_vars: delattr(eb.module_load_environment, env_var) - self.assertEqual(len(vars(eb.module_load_environment)), 0) + self.assertEqual(len(eb.module_load_environment.vars), 0) # check for behavior when a string value is used as value of module_load_environment eb.module_load_environment.PATH = 'bin' @@ -693,7 +693,7 @@ def test_module_search_path_headers(self): # test wrong easyconfig parameter self.contents += '\nmodule_search_path_headers = "WRONG_OPT"' self.writeEC() - ec =EasyConfig(self.eb_file) + ec = EasyConfig(self.eb_file) error_pattern = "Unknown value selected for option module-search-path-headers" with eb.module_generator.start_module_creation(): From 13d86115ef3439faf65328b97861735984bc0109 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 27 Jan 2025 18:47:19 +0100 Subject: [PATCH 06/15] fix codestyle in tools.modules --- easybuild/tools/modules.py | 1 + test/framework/modules.py | 1 + 2 files changed, 2 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 496c6b966c..d9fa8a2516 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -383,6 +383,7 @@ def set_cpp_headers(self, new_headers): for envar_name in self.name_cpp_headers: setattr(self, envar_name, new_headers) + class ModulesTool(object): """An abstract interface to a tool that deals with modules.""" # name of this modules tool (used in log/warning/error messages) diff --git a/test/framework/modules.py b/test/framework/modules.py index 959e95e820..688fa5cba7 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1776,6 +1776,7 @@ def test_module_load_environment(self): error_pattern = "Unknown value selected for option module-search-path-headers" self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, cpp_headers='nonexistent') + def suite(): """ returns all the testcases in this module """ return TestLoaderFiltered().loadTestsFromTestCase(ModulesTest, sys.argv[1:]) From 9ea0e6ec2046ad44fc081117027ec76dc17d7f16 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 13:13:55 +0100 Subject: [PATCH 07/15] replace cpp_headers methods from ModuleLoadEnvironment with a generic aliases feature --- easybuild/tools/modules.py | 90 +++++++++++++++++++++++++------------- test/framework/modules.py | 78 ++++++++++++++++++--------------- 2 files changed, 103 insertions(+), 65 deletions(-) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index b5a332b2d5..ec4f6bb013 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -46,8 +46,7 @@ from easybuild.base import fancylogger from easybuild.tools import LooseVersion from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning -from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE -from easybuild.tools.config import LOADED_MODULES_ACTIONS, MOD_SEARCH_PATH_HEADERS, PURGE +from easybuild.tools.config import ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE, LOADED_MODULES_ACTIONS, PURGE from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET from easybuild.tools.config import build_option, get_modules_tool, install_path from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars @@ -242,15 +241,33 @@ def is_path(self): class ModuleLoadEnvironment: - """Changes to environment variables that should be made when environment module is loaded""" + """ + Changes to environment variables that should be made when environment module is loaded. + - Environment variables are defined as ModuleEnvironmentVariables instances + with attribute name equal to environment variable name. + - Aliases are arbitrary names that serve to apply changes to lists of + environment variables + - Only environment variables attributes are public. Other attributes like + aliases are private. + """ - def __init__(self, cpp_headers=None): + def __init__(self, aliases=None): """ Initialize default environment definition Paths are relative to root of installation directory - :cpp_headers: string defining MOD_SEARCH_PATH_HEADERS setting + :aliases: dict defining environment variables aliases """ + self._aliases = {} + if aliases is not None: + try: + for alias_name, alias_vars in aliases.items(): + self.update_alias(alias_name, alias_vars) + except AttributeError as err: + raise EasyBuildError( + "Wrong format for aliases defitions passed to ModuleLoadEnvironment. " + f"Expected a dictionary but got: {type(aliases)}." + ) from err self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')] self.CLASSPATH = ['*.jar'] @@ -264,16 +281,11 @@ def __init__(self, cpp_headers=None): self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']] self.XDG_DATA_DIRS = ['share'] - # handle search paths to C/C++ headers - self._cpp_headers_opt = DEFAULT_MOD_SEARCH_PATH_HEADERS - if cpp_headers is not None and cpp_headers is not False: - self._cpp_headers_opt = str(cpp_headers) - if self._cpp_headers_opt not in MOD_SEARCH_PATH_HEADERS: - raise EasyBuildError( - f"Unknown value selected for option module-search-path-headers: {self._cpp_headers_opt}. " - f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}" - ) - self.set_cpp_headers(SEARCH_PATH_HEADER_DIRS) + # environment variables with known aliases + # e.g. search paths to C/C++ headers + if 'HEADERS' in self._aliases: + for envar_name in self._aliases['HEADERS']: + setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS) def __setattr__(self, name, value): """ @@ -353,29 +365,47 @@ def environ(self): """ return {envar_name: str(envar_contents) for envar_name, envar_contents in self.items()} - @property - def name_cpp_headers(self): + def alias(self, alias): """ - Return list of environment variable names holding search paths to CPP headers - According to option --module-search-path-headers + Return iterator to search path variables for given alias """ - return MOD_SEARCH_PATH_HEADERS[self._cpp_headers_opt] + try: + yield from [getattr(self, envar) for envar in self._aliases[alias]] + except KeyError as err: + raise EasyBuildError(f"Unknown search path alias: {alias}") from err + except AttributeError as err: + raise EasyBuildError(f"Missing environment variable in '{alias} alias") from err - @property - def cpp_headers(self): + def alias_vars(self, alias): """ - Return dict with search path variables for C/C++ headers - According to option --module-search-path-headers + Return list of environment variable names aliased by given alias """ - return {envar_name: getattr(self, envar_name) for envar_name in self.name_cpp_headers} + try: + return self._aliases[alias] + except KeyError as err: + raise EasyBuildError(f"Unknown search path alias: {alias}") from err - def set_cpp_headers(self, new_headers): + def update_alias(self, alias, value): """ - Set search paths variables for C/C++ headers - According to option --module-search-path-headers + Update existing or non-existing alias with given search paths variables """ - for envar_name in self.name_cpp_headers: - setattr(self, envar_name, new_headers) + if isinstance(value, str): + value = [value] + + try: + self._aliases[alias] = [str(envar) for envar in value] + except TypeError as err: + raise TypeError("ModuleLoadEnvironment aliases must be a list of strings") from err + + def set_alias_vars(self, alias, value): + """ + Set value of search paths variables for given alias + """ + try: + for envar_name in self._aliases[alias]: + setattr(self, envar_name, value) + except KeyError as err: + raise EasyBuildError(f"Unknown search path alias: {alias}") from err class ModulesTool(object): diff --git a/test/framework/modules.py b/test/framework/modules.py index 688fa5cba7..d10a5a298c 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1740,41 +1740,49 @@ def test_module_load_environment(self): self.assertTrue(hasattr(mod_load_env, 'TEST_STR')) self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path']) - # test functionality related to --module-search-path-headers - default_search_path_headers = 'cpath' - self.assertEqual(mod_load_env._cpp_headers_opt, default_search_path_headers) - mod_load_env = mod.ModuleLoadEnvironment(default_search_path_headers) - self.assertEqual(mod_load_env._cpp_headers_opt, default_search_path_headers) - self.assertEqual(mod_load_env.name_cpp_headers, ['CPATH']) - repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} - self.assertDictEqual(repr_mod_load_env, {'CPATH': 'include'}) - mod_load_env.set_cpp_headers('new_include') - repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} - self.assertDictEqual(repr_mod_load_env, {'CPATH': 'new_include'}) - mod_load_env.set_cpp_headers(["new_include_1", "new_include_2"]) - repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} - self.assertDictEqual(repr_mod_load_env, {'CPATH': 'new_include_1:new_include_2'}) - - mod_load_env = mod.ModuleLoadEnvironment(cpp_headers='include_paths') - self.assertNotEqual(mod_load_env._cpp_headers_opt, default_search_path_headers) - ref_include_vars = ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'] - self.assertEqual(mod_load_env.name_cpp_headers, ref_include_vars) - repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} - ref_cpp_headers = {key: 'include' for key in ref_include_vars} - self.assertDictEqual(repr_mod_load_env, ref_cpp_headers) - mod_load_env.set_cpp_headers('new_include') - repr_mod_load_env = {k: str(v) for k, v in mod_load_env.cpp_headers.items()} - ref_cpp_headers = {key: 'new_include' for key in ref_include_vars} - self.assertDictEqual(repr_mod_load_env, ref_cpp_headers) - - mod_load_env = mod.ModuleLoadEnvironment(cpp_headers='none') - self.assertEqual(mod_load_env.name_cpp_headers, []) - self.assertEqual(mod_load_env.cpp_headers, {}) - mod_load_env.set_cpp_headers('new_include') - self.assertEqual(mod_load_env.cpp_headers, {}) - - error_pattern = "Unknown value selected for option module-search-path-headers" - self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, cpp_headers='nonexistent') + # test aliases + aliases = { + 'ALIAS1': ['ALIAS_VAR11', 'ALIAS_VAR12'], + 'ALIAS2': ['ALIAS_VAR21'], + } + alias_load_env = mod.ModuleLoadEnvironment(aliases=aliases) + self.assertEqual(alias_load_env._aliases, aliases) + self.assertEqual(sorted(alias_load_env.alias_vars('ALIAS1')), ['ALIAS_VAR11', 'ALIAS_VAR12']) + self.assertEqual(alias_load_env.alias_vars('ALIAS2'), ['ALIAS_VAR21']) + # set a known alias + alias_load_env.set_alias_vars('ALIAS1', 'alias1_path') + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR11')) + self.assertEqual(alias_load_env.ALIAS_VAR11.contents, ['alias1_path']) + self.assertEqual(alias_load_env.ALIAS_VAR11.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR12')) + self.assertEqual(alias_load_env.ALIAS_VAR12.contents, ['alias1_path']) + self.assertEqual(alias_load_env.ALIAS_VAR12.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertFalse(hasattr(alias_load_env, 'ALIAS_VAR21')) + for envar in alias_load_env.alias('ALIAS1'): + self.assertEqual(envar.contents, ['alias1_path']) + self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES) + # set a second known alias + alias_load_env.set_alias_vars('ALIAS2', 'alias2_path') + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR11')) + self.assertEqual(alias_load_env.ALIAS_VAR11.contents, ['alias1_path']) + self.assertEqual(alias_load_env.ALIAS_VAR11.type, mod.ModEnvVarType.PATH_WITH_FILES) + self.assertTrue(hasattr(alias_load_env, 'ALIAS_VAR21')) + self.assertEqual(alias_load_env.ALIAS_VAR21.contents, ['alias2_path']) + self.assertEqual(alias_load_env.ALIAS_VAR21.type, mod.ModEnvVarType.PATH_WITH_FILES) + # add a new alias + alias_load_env.update_alias('ALIAS3', 'ALIAS_VAR31') + self.assertEqual(alias_load_env.alias_vars('ALIAS3'), ['ALIAS_VAR31']) + alias_load_env.update_alias('ALIAS3', ['ALIAS_VAR31', 'ALIAS_VAR32']) + self.assertEqual(sorted(alias_load_env.alias_vars('ALIAS3')), ['ALIAS_VAR31', 'ALIAS_VAR32']) + alias_load_env.set_alias_vars('ALIAS3', 'alias3_path') + for envar in alias_load_env.alias('ALIAS3'): + self.assertEqual(envar.contents, ['alias3_path']) + self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES) + + error_pattern = "Wrong format for aliases defitions passed to ModuleLoadEnvironment" + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=False) + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases='wrong') + self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=['some', 'list']) def suite(): From 4cede72fb79ce6b14d01be85a9f08d309cee9493 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 13:14:32 +0100 Subject: [PATCH 08/15] define aliases in ModuleLoadEnvironment for --module-search-path-headers option --- easybuild/framework/easyblock.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index bbadbd5a4b..99e7c39a48 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -75,10 +75,10 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_MOD_SEARCH_PATH_HEADERS from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES -from easybuild.tools.config import PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS +from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths from easybuild.tools.environment import restore_env, sanitize_env @@ -220,13 +220,23 @@ def __init__(self, ec, logfile=None): self.modules_header = read_file(modules_header_path) # environment variables on module load + mod_load_aliases = {} # apply --module-search-path-headers: easyconfig parameter has precedence - mod_load_cpp_headers = build_option('module_search_path_headers') - cfg_cpp_headers = self.cfg['module_search_path_headers'] - if cfg_cpp_headers is not False: - mod_load_cpp_headers = cfg_cpp_headers + mod_load_cpp_headers = self.cfg['module_search_path_headers'] + if mod_load_cpp_headers is False: + mod_load_cpp_headers = build_option('module_search_path_headers') + if mod_load_cpp_headers is False or mod_load_cpp_headers is None: + mod_load_cpp_headers = DEFAULT_MOD_SEARCH_PATH_HEADERS - self.module_load_environment = ModuleLoadEnvironment(cpp_headers=mod_load_cpp_headers) + try: + mod_load_aliases['HEADERS'] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers] + except KeyError as err: + raise EasyBuildError( + f"Unknown value selected for option module-search-path-headers: {mod_load_cpp_headers}. " + f"Choose one of: {', '.join(MOD_SEARCH_PATH_HEADERS)}" + ) from err + + self.module_load_environment = ModuleLoadEnvironment(aliases=mod_load_aliases) # determine install subdirectory, based on module name self.install_subdir = None From dca5a52d65a40871f43124405adcf9397039b3c5 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Mon, 10 Feb 2025 18:28:34 +0100 Subject: [PATCH 09/15] test appending of paths to ModuleLoadEnvironment aliases to test_module_load_environment --- test/framework/modules.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/framework/modules.py b/test/framework/modules.py index d10a5a298c..528d10c6a5 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1778,6 +1778,12 @@ def test_module_load_environment(self): for envar in alias_load_env.alias('ALIAS3'): self.assertEqual(envar.contents, ['alias3_path']) self.assertEqual(envar.type, mod.ModEnvVarType.PATH_WITH_FILES) + # append path to existing alias + for envar in alias_load_env.alias('ALIAS3'): + envar.append('new_path') + self.assertEqual(sorted(envar.contents), ['alias3_path', 'new_path']) + self.assertEqual(alias_load_env.ALIAS_VAR31.contents, ['alias3_path', 'new_path']) + self.assertEqual(alias_load_env.ALIAS_VAR32.contents, ['alias3_path', 'new_path']) error_pattern = "Wrong format for aliases defitions passed to ModuleLoadEnvironment" self.assertErrorRegex(EasyBuildError, error_pattern, mod.ModuleLoadEnvironment, aliases=False) From 8cf9189c49ba0a71fb2b2852a011a86d785ea4a3 Mon Sep 17 00:00:00 2001 From: Alex Domingo Date: Tue, 11 Feb 2025 01:45:49 +0100 Subject: [PATCH 10/15] add remove method to ModuleLoadEnvironment --- easybuild/tools/modules.py | 8 ++++++++ test/framework/modules.py | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 4b1cb70430..9cff5e1aec 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -350,6 +350,14 @@ def update(self, new_env): except AttributeError as err: raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err + def remove(self, var_name): + """ + Remove ModuleEnvironmentVariable attribute from instance + Silently goes through if attribute is already missing + """ + if var_name in self.vars: + delattr(self, var_name) + @property def as_dict(self): """ diff --git a/test/framework/modules.py b/test/framework/modules.py index d1d692e4d3..dac136f148 100644 --- a/test/framework/modules.py +++ b/test/framework/modules.py @@ -1742,6 +1742,11 @@ def test_module_load_environment(self): self.assertTrue(hasattr(mod_load_env, 'TEST_STR')) self.assertEqual(mod_load_env.TEST_STR.contents, ['some/path']) + # test removal of envars + mod_load_env.remove('TEST_VARTYPE') + self.assertFalse(hasattr(mod_load_env, 'TEST_VARTYPE')) + mod_load_env.remove('NONEXISTENT') + # test aliases aliases = { 'ALIAS1': ['ALIAS_VAR11', 'ALIAS_VAR12'], From 476c5b07dbbee21d561ce0200162fa652c2da02e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2025 12:47:56 +0100 Subject: [PATCH 11/15] introduce more constants for MOD_SEARCH_PATH_HEADERS* --- easybuild/tools/config.py | 11 +++++++---- easybuild/tools/options.py | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b69085ac7c..7b9000c2f2 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -183,12 +183,15 @@ PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES] # options to handle header search paths in environment of modules +MOD_SEARCH_PATH_HEADERS_CPATH = 'cpath' +MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS = 'include_paths' +MOD_SEARCH_PATH_HEADERS_NONE = 'none' MOD_SEARCH_PATH_HEADERS = { - "none": [], - "cpath": ["CPATH"], - "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], + MOD_SEARCH_PATH_HEADERS_CPATH: ['CPATH'], + MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS: ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'], + MOD_SEARCH_PATH_HEADERS_NONE: [], } -DEFAULT_MOD_SEARCH_PATH_HEADERS = "cpath" +DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH class Singleton(ABCMeta): diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index de415501f6..70671ce20f 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -618,7 +618,7 @@ def config_options(self): 'module-naming-scheme': ("Module naming scheme to use", None, 'store', DEFAULT_MNS), 'module-search-path-headers': ("Environment variable set by modules on load with search paths " "to header files", 'choice', 'store', DEFAULT_MOD_SEARCH_PATH_HEADERS, - [*MOD_SEARCH_PATH_HEADERS]), + sorted(MOD_SEARCH_PATH_HEADERS.keys())), 'module-syntax': ("Syntax to be used for module files", 'choice', 'store', DEFAULT_MODULE_SYNTAX, sorted(avail_module_generators().keys())), 'moduleclasses': (("Extend supported module classes " From ca64b965eb0238fc9073ea1a975e0c06741b7394 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2025 12:58:06 +0100 Subject: [PATCH 12/15] use 'cpath' as default for module_search_path_headers easyconfig parameter --- easybuild/framework/easyconfig/default.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index fb942109e3..78354d3d92 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -37,7 +37,7 @@ """ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import MODULECLASS_BASE +from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MODULECLASS_BASE _log = fancylogger.getLogger('easyconfig.default', fname=False) @@ -208,8 +208,8 @@ 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' '(implies recursive unloading of modules) [DEPRECATED]', MODULES], - 'module_search_path_headers': [False, "Environment variable set by modules on load with search paths " - "to header files", MODULES], + 'module_search_path_headers': [DEFAULT_MOD_SEARCH_PATH_HEADERS, "Environment variable set by modules on load " + "with search paths to header files", MODULES], 'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module " "(True/False to hard enable/disable; None implies honoring the " "--recursive-module-unload EasyBuild configuration setting", MODULES], From f8afe45956a09ea2b7b85872f15fe69f8c1be0f9 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2025 13:00:41 +0100 Subject: [PATCH 13/15] simplify logic to determine mod_load_cpp_headers --- easybuild/framework/easyblock.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 8e82c3e356..c422e8d7cf 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -223,10 +223,8 @@ def __init__(self, ec, logfile=None): # environment variables on module load mod_load_aliases = {} # apply --module-search-path-headers: easyconfig parameter has precedence - mod_load_cpp_headers = self.cfg['module_search_path_headers'] - if mod_load_cpp_headers is False: - mod_load_cpp_headers = build_option('module_search_path_headers') - if mod_load_cpp_headers is False or mod_load_cpp_headers is None: + mod_load_cpp_headers = self.cfg['module_search_path_headers'] or build_option('module_search_path_headers') + if mod_load_cpp_headers is None: mod_load_cpp_headers = DEFAULT_MOD_SEARCH_PATH_HEADERS try: From f908b8d22274549df6bbb2709913762ca3ef3d8b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2025 13:03:33 +0100 Subject: [PATCH 14/15] add constant for 'HEADERS' magic string in ModuleLoadEnvironment --- easybuild/framework/easyblock.py | 4 ++-- easybuild/tools/modules.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index c422e8d7cf..92b0dfceca 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -100,7 +100,7 @@ from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version from easybuild.tools.modules import ROOT_ENV_VAR_NAME_PREFIX, VERSION_ENV_VAR_NAME_PREFIX, DEVEL_ENV_VAR_NAME_PREFIX -from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment +from easybuild.tools.modules import Lmod, ModEnvVarType, ModuleLoadEnvironment, MODULE_LOAD_ENV_HEADERS from easybuild.tools.modules import curr_module_paths, invalidate_module_caches_for, get_software_root from easybuild.tools.modules import get_software_root_env_var_name, get_software_version_env_var_name from easybuild.tools.output import PROGRESS_BAR_DOWNLOAD_ALL, PROGRESS_BAR_EASYCONFIG, PROGRESS_BAR_EXTENSIONS @@ -228,7 +228,7 @@ def __init__(self, ec, logfile=None): mod_load_cpp_headers = DEFAULT_MOD_SEARCH_PATH_HEADERS try: - mod_load_aliases['HEADERS'] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers] + mod_load_aliases[MODULE_LOAD_ENV_HEADERS] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers] except KeyError as err: raise EasyBuildError( f"Unknown value selected for option module-search-path-headers: {mod_load_cpp_headers}. " diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 9cff5e1aec..cba8810e74 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -57,6 +57,8 @@ from easybuild.tools.utilities import get_subclasses, nub +MODULE_LOAD_ENV_HEADERS = 'HEADERS' + # software root/version environment variable name prefixes ROOT_ENV_VAR_NAME_PREFIX = "EBROOT" VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION" @@ -283,9 +285,8 @@ def __init__(self, aliases=None): # environment variables with known aliases # e.g. search paths to C/C++ headers - if 'HEADERS' in self._aliases: - for envar_name in self._aliases['HEADERS']: - setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS) + for envar_name in self._aliases.get(MODULE_LOAD_ENV_HEADERS, []): + setattr(self, envar_name, SEARCH_PATH_HEADER_DIRS) def __setattr__(self, name, value): """ From 10525cda000678808ef75f4e7a01c94080606978 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 12 Feb 2025 13:35:57 +0100 Subject: [PATCH 15/15] fix default for module_search_path_headers build option, remove 'none' as valid value for --module-search-path-headers --- easybuild/framework/easyblock.py | 4 +--- easybuild/framework/easyconfig/default.py | 6 +++--- easybuild/tools/config.py | 6 +++--- test/framework/easyblock.py | 5 ++--- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 92b0dfceca..b0fc9ff51c 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -75,7 +75,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning -from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_MOD_SEARCH_PATH_HEADERS +from easybuild.tools.config import CHECKSUM_PRIORITY_JSON, DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import EASYBUILD_SOURCES_URL, EBPYTHONPREFIXES # noqa from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import MOD_SEARCH_PATH_HEADERS, PYTHONPATH, SEARCH_PATH_BIN_DIRS, SEARCH_PATH_LIB_DIRS @@ -224,8 +224,6 @@ def __init__(self, ec, logfile=None): mod_load_aliases = {} # apply --module-search-path-headers: easyconfig parameter has precedence mod_load_cpp_headers = self.cfg['module_search_path_headers'] or build_option('module_search_path_headers') - if mod_load_cpp_headers is None: - mod_load_cpp_headers = DEFAULT_MOD_SEARCH_PATH_HEADERS try: mod_load_aliases[MODULE_LOAD_ENV_HEADERS] = MOD_SEARCH_PATH_HEADERS[mod_load_cpp_headers] diff --git a/easybuild/framework/easyconfig/default.py b/easybuild/framework/easyconfig/default.py index 78354d3d92..bca46c3856 100644 --- a/easybuild/framework/easyconfig/default.py +++ b/easybuild/framework/easyconfig/default.py @@ -37,7 +37,7 @@ """ from easybuild.base import fancylogger from easybuild.tools.build_log import EasyBuildError -from easybuild.tools.config import DEFAULT_MOD_SEARCH_PATH_HEADERS, MODULECLASS_BASE +from easybuild.tools.config import MODULECLASS_BASE _log = fancylogger.getLogger('easyconfig.default', fname=False) @@ -208,8 +208,8 @@ 'moduleloadnoconflict': [False, "Don't check for conflicts, unload other versions instead ", MODULES], 'module_depends_on': [None, 'Use depends_on (Lmod 7.6.1+) for dependencies in generated module ' '(implies recursive unloading of modules) [DEPRECATED]', MODULES], - 'module_search_path_headers': [DEFAULT_MOD_SEARCH_PATH_HEADERS, "Environment variable set by modules on load " - "with search paths to header files", MODULES], + 'module_search_path_headers': [None, "Environment variable set by modules on load " + "with search paths to header files (if None, use $CPATH)", MODULES], 'recursive_module_unload': [None, "Recursive unload of all dependencies when unloading module " "(True/False to hard enable/disable; None implies honoring the " "--recursive-module-unload EasyBuild configuration setting", MODULES], diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index 7b9000c2f2..3503d5c2f5 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -185,11 +185,9 @@ # options to handle header search paths in environment of modules MOD_SEARCH_PATH_HEADERS_CPATH = 'cpath' MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS = 'include_paths' -MOD_SEARCH_PATH_HEADERS_NONE = 'none' MOD_SEARCH_PATH_HEADERS = { MOD_SEARCH_PATH_HEADERS_CPATH: ['CPATH'], MOD_SEARCH_PATH_HEADERS_INCLUDE_PATHS: ['C_INCLUDE_PATH', 'CPLUS_INCLUDE_PATH', 'OBJC_INCLUDE_PATH'], - MOD_SEARCH_PATH_HEADERS_NONE: [], } DEFAULT_MOD_SEARCH_PATH_HEADERS = MOD_SEARCH_PATH_HEADERS_CPATH @@ -318,7 +316,6 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'logtostdout', 'minimal_toolchains', 'module_only', - 'module_search_path_headers', 'package', 'parallel_extensions_install', 'read_only_installdir', @@ -401,6 +398,9 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): DEFAULT_MINIMAL_BUILD_ENV: [ 'minimal_build_env', ], + DEFAULT_MOD_SEARCH_PATH_HEADERS: [ + 'module_search_path_headers', + ], DEFAULT_PKG_RELEASE: [ 'package_release', ], diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index d745be8ec3..97d8f31b93 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -671,7 +671,6 @@ def test_make_module_req(self): def test_module_search_path_headers(self): """Test functionality of module-search-path-headers option""" sp_headers_mode = { - "none": [], "cpath": ["CPATH"], "include_paths": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"], } @@ -687,7 +686,7 @@ def test_module_search_path_headers(self): self.writeEC() for build_opt, sp_headers in sp_headers_mode.items(): - init_config(build_options={"module_search_path_headers": build_opt, "silent": True}) + update_build_option('module_search_path_headers', build_opt) eb = EasyBlock(EasyConfig(self.eb_file)) eb.installdir = config.install_path() try: @@ -729,7 +728,7 @@ def test_module_search_path_headers(self): pass for build_opt in sp_headers_mode: - init_config(build_options={"module_search_path_headers": build_opt, "silent": True}) + update_build_option('module_search_path_headers', build_opt) with eb.module_generator.start_module_creation(): guess = eb.make_module_req() if not sp_headers: