Skip to content

Allow use of alternate envvar(s) to $HOME for user modules #3558

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Feb 18, 2021
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
cc0a77e
Allow use of alternate envvar to HOME for user modules
Feb 2, 2021
0ac6d01
Allow use of a list of environment variables
Feb 3, 2021
ab957ab
Appease hound
Feb 3, 2021
8655e2d
Update easybuild/framework/easyblock.py
Feb 15, 2021
ba2ee41
Merge branch 'develop' into alt-envvar-user-mod
Feb 16, 2021
86d38b7
Merge branch 'alt-envvar-user-mod' of github.com:ocaisa/easybuild-fra…
Feb 16, 2021
97c0374
Address comments
Feb 16, 2021
595fb68
Appease hound
Feb 16, 2021
086fc41
Allow use of non-existent environment variables (defaulting to "" for…
Feb 16, 2021
b57f177
Add additional tests and keep old behaviour as much as possible
Feb 16, 2021
57501e3
Replace "safe" kwarg with possibility to define default value when gr…
Feb 17, 2021
d1f29d4
Replace "safe" kwarg with possibility to define default value when gr…
Feb 17, 2021
2f89410
Add additional test to make sure any matching path in the CWD is not …
Feb 17, 2021
b67ce23
Appease hound
Feb 17, 2021
a36f81a
Replace warning with debug message when for environment variables use…
Feb 17, 2021
34db258
Be more careful with test case
Feb 17, 2021
7d63bf6
Make sure environment variables are not defined for initial module lo…
Feb 17, 2021
9e53ee6
Clean up a little
Feb 17, 2021
b8ca530
cleanup in ModuleGenerator.getenv_cmd implementations + enhance tests…
boegel Feb 18, 2021
d542e43
minor style cleanups
boegel Feb 18, 2021
0133222
only load test module in Lua syntax to check getenv_cmd syntax when u…
boegel Feb 18, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions easybuild/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -1345,10 +1345,16 @@ def make_module_extend_modpath(self):
# add user-specific module path; use statement will be guarded so no need to create the directories
user_modpath = build_option('subdir_user_modules')
if user_modpath:
user_envvars = build_option('envvars_user_modules') or ['HOME']
user_modpath_exts = ActiveMNS().det_user_modpath_extensions(self.cfg)
self.log.debug("Including user module path extensions returned by naming scheme: %s", user_modpath_exts)
txt += self.module_generator.use(user_modpath_exts, prefix=self.module_generator.getenv_cmd('HOME'),
guarded=True, user_modpath=user_modpath)
for user_envvar in user_envvars:
if not os.getenv(user_envvar):
raise EasyBuildError("Requested environment variable %s as an additional branch for user"
"modules does not exist in current environment", user_envvar)
txt += self.module_generator.use(user_modpath_exts,
prefix=self.module_generator.getenv_cmd(user_envvar), guarded=True,
user_modpath=user_modpath)
else:
self.log.debug("Not including module path extensions, as specified.")
return txt
Expand Down
2 changes: 2 additions & 0 deletions easybuild/tools/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
DEFAULT_CONT_TYPE = CONT_TYPE_SINGULARITY

DEFAULT_BRANCH = 'develop'
DEFAULT_ENVVAR_USERS_MODULES = 'HOME'
DEFAULT_INDEX_MAX_AGE = 7 * 24 * 60 * 60 # 1 week (in seconds)
DEFAULT_JOB_BACKEND = 'GC3Pie'
DEFAULT_LOGFILE_FORMAT = ("easybuild", "easybuild-%(name)s-%(version)s-%(date)s.%(time)s.log")
Expand Down Expand Up @@ -173,6 +174,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'download_timeout',
'dump_test_report',
'easyblock',
'envvars_user_modules',
'extra_modules',
'filter_deps',
'filter_env_vars',
Expand Down
6 changes: 5 additions & 1 deletion easybuild/tools/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,9 @@ def config_options(self):
'buildpath': ("Temporary build path", None, 'store', mk_full_default_path('buildpath')),
'containerpath': ("Location where container recipe & image will be stored", None, 'store',
mk_full_default_path('containerpath')),
'envvars-user-modules': ("List of environment variables that hold the base paths for which user-specific "
"modules will be installed relative to", 'strlist', 'store',
['HOME']),
'external-modules-metadata': ("List of (glob patterns for) paths to files specifying metadata "
"for external modules (INI format)", 'strlist', 'store', None),
'hooks': ("Location of Python module with hook implementations", 'str', 'store', None),
Expand Down Expand Up @@ -547,7 +550,8 @@ def config_options(self):
'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']),
'subdir-software': ("Installpath subdir for software",
None, 'store', DEFAULT_PATH_SUBDIRS['subdir_software']),
'subdir-user-modules': ("Base path of user-specific modules relative to their $HOME", None, 'store', None),
'subdir-user-modules': ("Base path of user-specific modules relative to --envvar-user-modules (which "
"defaults to $HOME)", None, 'store', None),
'suffix-modules-path': ("Suffix for module files install path", None, 'store', GENERAL_CLASS),
# this one is sort of an exception, it's something jobscripts can set,
# has no real meaning for regular eb usage
Expand Down
45 changes: 45 additions & 0 deletions test/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,51 @@ def test_make_module_extend_modpath(self):
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))

# Repeat this but using an alternate envvars (instead of $HOME)
list_of_envvars = ['TURKEY', 'HAM']
os.environ['TURKEY'] = "1"
os.environ['HAM'] = "1"

build_options = {
'envvars_user_modules': list_of_envvars,
'subdir_user_modules': usermodsdir,
'valid_module_classes': modclasses,
'suffix_modules_path': 'funky',
}
init_config(build_options=build_options)
eb = EasyBlock(EasyConfig(self.eb_file))
eb.installdir = config.install_path()

txt = eb.make_module_extend_modpath()
for envvar in list_of_envvars:
if get_module_syntax() == 'Tcl':
regexs = [r'^module use ".*/modules/funky/Compiler/pi/3.14/%s"$' % c for c in modclasses]
home = r'\$::env\(%s\)' % envvar
fj_usermodsdir = 'file join "%s" "funky" "Compiler/pi/3.14"' % usermodsdir
regexs.extend([
# extension for user modules is guarded
r'if { \[ file isdirectory \[ file join %s \[ %s \] \] \] } {$' % (home, fj_usermodsdir),
# no per-moduleclass extension for user modules
r'^\s+module use \[ file join %s \[ %s \] \]$' % (home, fj_usermodsdir),
])
elif get_module_syntax() == 'Lua':
regexs = [r'^prepend_path\("MODULEPATH", ".*/modules/funky/Compiler/pi/3.14/%s"\)$' % c
for c in modclasses]
home = r'os.getenv\("%s"\)' % envvar
pj_usermodsdir = r'pathJoin\("%s", "funky", "Compiler/pi/3.14"\)' % usermodsdir
regexs.extend([
# extension for user modules is guarded
r'if isDir\(pathJoin\(%s, %s\)\) then' % (home, pj_usermodsdir),
# no per-moduleclass extension for user modules
r'\s+prepend_path\("MODULEPATH", pathJoin\(%s, %s\)\)' % (home, pj_usermodsdir),
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
for regex in regexs:
regex = re.compile(regex, re.M)
self.assertTrue(regex.search(txt), "Pattern '%s' found in: %s" % (regex.pattern, txt))
os.unsetenv(envvar)

def test_make_module_req(self):
"""Testcase for make_module_req"""
self.contents = '\n'.join([
Expand Down