diff --git a/easybuild/framework/easyblock.py b/easybuild/framework/easyblock.py index 5f2b5ec819..78b4cdcc1b 100644 --- a/easybuild/framework/easyblock.py +++ b/easybuild/framework/easyblock.py @@ -66,6 +66,7 @@ from easybuild.tools.build_details import get_build_stats from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs from easybuild.tools.build_log import print_error, print_msg, print_warning +from easybuild.tools.config import DEFAULT_ENVVAR_USERS_MODULES from easybuild.tools.config import FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_PATCHES, FORCE_DOWNLOAD_SOURCES from easybuild.tools.config import build_option, build_path, get_log_filename, get_repository, get_repositorypath from easybuild.tools.config import install_path, log_path, package_path, source_paths @@ -1351,10 +1352,16 @@ def make_module_extend_modpath(self): # add user-specific module path; use statement will be guarded so no need to create the directories user_modpath = build_option('subdir_user_modules') if user_modpath: + user_envvars = build_option('envvars_user_modules') or [DEFAULT_ENVVAR_USERS_MODULES] user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg) self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts) - txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'), - guarded=True, user_modpath=user_modpath) + for user_envvar in user_envvars: + self.log.debug("Requested environment variable $%s to host additional branch for modules", + user_envvar) + default_value = user_envvar + "_NOT_DEFINED" + getenv_txt = self.module_generator.getenv_cmd(user_envvar, default=default_value) + txt += self.module_generator.use(user_modpath_exts, prefix=getenv_txt, + guarded=True, user_modpath=user_modpath) else: self.log.debug("Not including module path extensions, as specified.") return txt diff --git a/easybuild/tools/config.py b/easybuild/tools/config.py index b97186f3c5..66ae496b38 100644 --- a/easybuild/tools/config.py +++ b/easybuild/tools/config.py @@ -79,6 +79,7 @@ DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY DEFAULT_BRANCH = 'develop' +DEFAULT_ENVVAR_USERS_MODULES = 'HOME' DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds) DEFAULT_JOB_BACKEND = 'GC3Pie' DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log") @@ -173,6 +174,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX): 'download_timeout', 'dump_test_report', 'easyblock', + 'envvars_user_modules', 'extra_modules', 'filter_deps', 'filter_env_vars', diff --git a/easybuild/tools/module_generator.py b/easybuild/tools/module_generator.py index c031f14d83..53a6c25916 100644 --- a/easybuild/tools/module_generator.py +++ b/easybuild/tools/module_generator.py @@ -394,7 +394,7 @@ def get_description(self, conflict=True): """ raise NotImplementedError - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ @@ -776,11 +776,19 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - return '$::env(%s)' % envvar + if default is None: + cmd = '$::env(%s)' % envvar + else: + values = { + 'default': default, + 'envvar': '::env(%s)' % envvar, + } + cmd = '[if { [info exists %(envvar)s] } { concat $%(envvar)s } else { concat "%(default)s" } ]' % values + return cmd def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): """ @@ -1196,11 +1204,15 @@ def get_description(self, conflict=True): return txt - def getenv_cmd(self, envvar): + def getenv_cmd(self, envvar, default=None): """ Return module-syntax specific code to get value of specific environment variable. """ - return 'os.getenv("%s")' % envvar + if default is None: + cmd = 'os.getenv("%s")' % envvar + else: + cmd = 'os.getenv("%s") or "%s"' % (envvar, default) + return cmd def load_module(self, mod_name, recursive_unload=False, depends_on=False, unload_modules=None, multi_dep_mods=None): """ diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index a363cde325..e46be243cc 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -60,7 +60,8 @@ from easybuild.tools.build_log import DEVEL_LOG_LEVEL, EasyBuildError from easybuild.tools.build_log import init_logging, log_start, print_msg, print_warning, raise_easybuilderror from easybuild.tools.config import CONT_IMAGE_FORMATS, CONT_TYPES, DEFAULT_CONT_TYPE, DEFAULT_ALLOW_LOADED_MODULES -from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_FORCE_DOWNLOAD, DEFAULT_INDEX_MAX_AGE +from easybuild.tools.config import DEFAULT_BRANCH, DEFAULT_ENVVAR_USERS_MODULES, DEFAULT_FORCE_DOWNLOAD +from easybuild.tools.config import DEFAULT_INDEX_MAX_AGE from easybuild.tools.config import DEFAULT_JOB_BACKEND, DEFAULT_LOGFILE_FORMAT, DEFAULT_MAX_FAIL_RATIO_PERMS from easybuild.tools.config import DEFAULT_MINIMAL_BUILD_ENV, DEFAULT_MNS, DEFAULT_MODULE_SYNTAX, DEFAULT_MODULES_TOOL from easybuild.tools.config import DEFAULT_MODULECLASSES, DEFAULT_PATH_SUBDIRS, DEFAULT_PKG_RELEASE, DEFAULT_PKG_TOOL @@ -489,6 +490,9 @@ def config_options(self): 'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')), 'containerpath': ("Location where container recipe & image will be stored", None, 'store', mk_full_default_path('containerpath')), + 'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific " + "modules will be installed relative to", 'strlist', 'store', + [DEFAULT_ENVVAR_USERS_MODULES]), 'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata " "for external modules (INI format)", 'strlist', 'store', None), 'hooks': ("Location of Python module with hook implementations", 'str', 'store', None), @@ -547,7 +551,8 @@ def config_options(self): 'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']), 'subdir-software': ("Installpath subdir for software", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']), - 'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None), + 'subdir-user-modules': ("Base path of user-specific modules relative to --envvar-user-modules", + None, 'store', None), 'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS), # this one is sort of an exception, it's something jobscripts can set, # has no real meaning for regular eb usage diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index 7e6911269b..7d308d287d 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -248,6 +248,9 @@ def test_fake_module_load(self): def test_make_module_extend_modpath(self): """Test for make_module_extend_modpath""" + + module_syntax = get_module_syntax() + self.contents = '\n'.join([ 'easyblock = "ConfigureMake"', 'name = "pi"', @@ -263,7 +266,7 @@ def test_make_module_extend_modpath(self): # no $MODULEPATH extensions for default module naming scheme (EasyBuildMNS) self.assertEqual(eb.make_module_extend_modpath(), '') - usermodsdir = 'my/own/modules' + usermodsdir = 'my_own_modules' modclasses = ['compiler', 'tools'] os.environ['EASYBUILD_MODULE_NAMING_SCHEME'] = 'CategorizedHMNS' build_options = { @@ -276,9 +279,10 @@ def test_make_module_extend_modpath(self): eb.installdir = config.install_path() txt = eb.make_module_extend_modpath() - if get_module_syntax() == 'Tcl': + if module_syntax == 'Tcl': regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] - home = r'\$::env\(HOME\)' + home = r'\[if { \[info exists ::env\(HOME\)\] } { concat \$::env\(HOME\) } ' + home += r'else { concat "HOME_NOT_DEFINED" } \]' fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -286,9 +290,9 @@ def test_make_module_extend_modpath(self): # no per-moduleclass extension for user modules r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir), ]) - elif get_module_syntax() == 'Lua': + elif module_syntax == 'Lua': regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c for c in modclasses] - home = r'os.getenv\("HOME"\)' + home = r'os.getenv\("HOME"\) or "HOME_NOT_DEFINED"' pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir regexs.extend([ # extension for user modules is guarded @@ -297,11 +301,113 @@ def test_make_module_extend_modpath(self): r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir), ]) else: - self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax()) + self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + for regex in regexs: regex = re.compile(regex, re.M) self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + # Repeat this but using an alternate envvars (instead of $HOME) + list_of_envvars = ['SITE_INSTALLS', 'USER_INSTALLS'] + + build_options = { + 'envvars_user_modules': list_of_envvars, + 'subdir_user_modules': usermodsdir, + 'valid_module_classes': modclasses, + 'suffix_modules_path': 'funky', + } + init_config(build_options=build_options) + eb = EasyBlock(EasyConfig(self.eb_file)) + eb.installdir = config.install_path() + + txt = eb.make_module_extend_modpath() + for envvar in list_of_envvars: + if module_syntax == 'Tcl': + regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses] + module_envvar = r'\[if \{ \[info exists ::env\(%s\)\] \} ' % envvar + module_envvar += r'\{ concat \$::env\(%s\) \} ' % envvar + module_envvar += r'else { concat "%s" } \]' % (envvar + '_NOT_DEFINED') + fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (module_envvar, fj_usermodsdir), + # no per-moduleclass extension for user modules + r'^\s+module use \[ file join %s \[ %s \] \]$' % (module_envvar, fj_usermodsdir), + ]) + elif module_syntax == 'Lua': + regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c + for c in modclasses] + module_envvar = r'os.getenv\("%s"\) or "%s"' % (envvar, envvar + "_NOT_DEFINED") + pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir + regexs.extend([ + # extension for user modules is guarded + r'if isDir\(pathJoin\(%s, %s\)\) then' % (module_envvar, pj_usermodsdir), + # no per-moduleclass extension for user modules + r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (module_envvar, pj_usermodsdir), + ]) + else: + self.assertTrue(False, "Unknown module syntax: %s" % module_syntax) + + for regex in regexs: + regex = re.compile(regex, re.M) + self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt)) + os.unsetenv(envvar) + + # Check behaviour when directories do and do not exist + usermodsdir_extension = os.path.join(usermodsdir, "funky", "Compiler/pi/3.14") + site_install_path = os.path.join(config.install_path(), 'site') + site_modules = os.path.join(site_install_path, usermodsdir_extension) + user_install_path = os.path.join(config.install_path(), 'user') + user_modules = os.path.join(user_install_path, usermodsdir_extension) + + # make a modules directory so that we can create our module files + temp_module_file_dir = os.path.join(site_install_path, usermodsdir, "temp_module_files") + mkdir(temp_module_file_dir, parents=True) + + # write out a module file + if module_syntax == 'Tcl': + module_file = os.path.join(temp_module_file_dir, "mytest") + module_txt = "#%Module\n" + txt + elif module_syntax == 'Lua': + module_file = os.path.join(temp_module_file_dir, "mytest.lua") + module_txt = txt + write_file(module_file, module_txt) + + # Set MODULEPATH and check the effect of `module load` + os.environ['MODULEPATH'] = temp_module_file_dir + + # Let's switch to a dir where the paths we will use exist to make sure they can + # not be accidentally picked up if the variable is not defined but the paths exist + # relative to the current directory + cwd = os.getcwd() + mkdir(os.path.join(config.install_path(), "existing_dir", usermodsdir_extension), parents=True) + change_dir(os.path.join(config.install_path(), "existing_dir")) + self.modtool.run_module('load', 'mytest') + self.assertFalse(usermodsdir_extension in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + change_dir(cwd) + + # Now define our environment variables + os.environ['SITE_INSTALLS'] = site_install_path + os.environ['USER_INSTALLS'] = user_install_path + + # Check MODULEPATH when neither directories exist + self.modtool.run_module('load', 'mytest') + self.assertFalse(site_modules in os.environ['MODULEPATH']) + self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + # Now create the directory for site modules + mkdir(site_modules, parents=True) + self.modtool.run_module('load', 'mytest') + self.assertTrue(os.environ['MODULEPATH'].startswith(site_modules)) + self.assertFalse(user_modules in os.environ['MODULEPATH']) + self.modtool.run_module('unload', 'mytest') + # Now create the directory for user modules + mkdir(user_modules, parents=True) + self.modtool.run_module('load', 'mytest') + self.assertTrue(os.environ['MODULEPATH'].startswith(user_modules + ":" + site_modules)) + self.modtool.run_module('unload', 'mytest') + def test_make_module_req(self): """Testcase for make_module_req""" self.contents = '\n'.join([ diff --git a/test/framework/module_generator.py b/test/framework/module_generator.py index 4a8cea46f5..00ee2cd8ae 100644 --- a/test/framework/module_generator.py +++ b/test/framework/module_generator.py @@ -853,13 +853,44 @@ def test_env(self): def test_getenv_cmd(self): """Test getting value of environment variable.""" + + test_mod_file = os.path.join(self.test_prefix, 'test', '1.2.3') + if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: + # can't have $LMOD_QUIET set when testing with Tcl syntax, + # otherwise we won't get the output produced by the test module file... + if 'LMOD_QUIET' in os.environ: + del os.environ['LMOD_QUIET'] + self.assertEqual('$::env(HOSTNAME)', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('$::env(HOME)', self.modgen.getenv_cmd('HOME')) + + expected = '[if { [info exists ::env(TEST)] } { concat $::env(TEST) } else { concat "foobar" } ]' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + + write_file(test_mod_file, '#%%Module\nputs stderr %s' % getenv_txt) else: self.assertEqual('os.getenv("HOSTNAME")', self.modgen.getenv_cmd('HOSTNAME')) self.assertEqual('os.getenv("HOME")', self.modgen.getenv_cmd('HOME')) + expected = 'os.getenv("TEST") or "foobar"' + getenv_txt = self.modgen.getenv_cmd('TEST', default='foobar') + self.assertEqual(getenv_txt, expected) + + test_mod_file += '.lua' + write_file(test_mod_file, "io.stderr:write(%s)" % getenv_txt) + + # only test loading of test module in Lua syntax when using Lmod + if isinstance(self.modtool, Lmod) or not test_mod_file.endswith('.lua'): + self.modtool.use(self.test_prefix) + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'foobar') + + os.environ['TEST'] = 'test_value_that_is_not_foobar' + out = self.modtool.run_module('load', 'test/1.2.3', return_stderr=True) + self.assertEqual(out.strip(), 'test_value_that_is_not_foobar') + def test_alias(self): """Test setting of alias in modulefiles.""" if self.MODULE_GENERATOR_CLASS == ModuleGeneratorTcl: