Skip to content

avoid running expensive 'module use' command when using Lmod as modules tool, update $MODULEPATH directly instead #3557

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 5, 2021
Merged

avoid running expensive 'module use' command when using Lmod as modules tool, update $MODULEPATH directly instead #3557

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
73055d0
Avoid module use in LMod if possible to allow faster execution
Flamefire Feb 2, 2021
832010c
Merge branch 'develop' into avoid_module_use
boegel Apr 5, 2021
5293218
extend test for 'module use', also test 'module unuse'
boegel Apr 5, 2021
a45763f
also load module when testing use/unuse
boegel Apr 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion easybuild/tools/modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1428,7 +1428,20 @@ def use(self, path, priority=None):
if priority:
self.run_module(['use', '--priority', str(priority), path])
else:
self.run_module(['use', path])
# LMod allows modifying MODULEPATH directly. So do that to avoid the costly module use
# unless priorities are in use already
if os.environ.get('__LMOD_Priority_MODULEPATH'):
self.run_module(['use', path])
else:
cur_mod_path = os.environ.get('MODULEPATH')
if cur_mod_path is None:
new_mod_path = path
else:
new_mod_path = [path] + [p for p in cur_mod_path.split(':') if p != path]
new_mod_path = ':'.join(new_mod_path)
self.log.debug('Changing MODULEPATH from %s to %s' %
('<unset>' if cur_mod_path is None else cur_mod_path, new_mod_path))
os.environ['MODULEPATH'] = new_mod_path

def prepend_module_path(self, path, set_mod_paths=True, priority=None):
"""
Expand Down
57 changes: 54 additions & 3 deletions test/framework/modules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1142,24 +1142,75 @@ def test_module_caches(self):
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
self.assertEqual(mod.MODULE_SHOW_CACHE, {})

def test_module_use(self):
"""Test 'module use'."""
def test_module_use_unuse(self):
"""Test 'module use' and 'module unuse'."""
test_dir1 = os.path.join(self.test_prefix, 'one')
test_dir2 = os.path.join(self.test_prefix, 'two')
test_dir3 = os.path.join(self.test_prefix, 'three')

for subdir in ('one', 'two', 'three'):
modtxt = '\n'.join([
'#%Module',
"setenv TEST123 %s" % subdir,
])
write_file(os.path.join(self.test_prefix, subdir, 'test'), modtxt)

self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', ''))
self.modtool.use(test_dir1)
self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir1))
self.modtool.use(test_dir2)
self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir2))
self.modtool.use(test_dir3)
self.assertTrue(os.environ.get('MODULEPATH', '').startswith('%s:' % test_dir3))

# make sure the right test module is loaded
self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'three')
self.modtool.unload(['test'])

self.modtool.unuse(test_dir3)
self.assertFalse(test_dir3 in os.environ.get('MODULEPATH', ''))

self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'two')
self.modtool.unload(['test'])

self.modtool.unuse(test_dir2)
self.assertFalse(test_dir2 in os.environ.get('MODULEPATH', ''))

self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'one')
self.modtool.unload(['test'])

self.modtool.unuse(test_dir1)
self.assertFalse(test_dir1 in os.environ.get('MODULEPATH', ''))

# also test use with high priority
self.modtool.use(test_dir2, priority=10000)
self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir2))

self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'two')
self.modtool.unload(['test'])

# check whether prepend with priority actually works (only for Lmod)
if isinstance(self.modtool, Lmod):
self.modtool.use(test_dir1, priority=100)
self.modtool.use(test_dir3)
self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:' % (test_dir2, test_dir3)))
self.assertTrue(os.environ['MODULEPATH'].startswith('%s:%s:%s:' % (test_dir2, test_dir1, test_dir3)))
self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'two')
self.modtool.unload(['test'])

self.modtool.unuse(test_dir2)
self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'one')
self.modtool.unload(['test'])

self.modtool.unuse(test_dir1)
self.modtool.load(['test'])
self.assertEqual(os.getenv('TEST123'), 'three')
self.modtool.unload(['test'])

def test_module_use_bash(self):
"""Test whether effect of 'module use' is preserved when a new bash session is started."""
Expand Down