Skip to content

Commit dec669c

Browse files
akesandgrenakesandgren
authored
Merge pull request #3637 from Flamefire/deprecateCreatingDirInUse
Deprecate adding a non-existing path to $MODULEPATH
2 parents caef231 + cf60507 commit dec669c

File tree

7 files changed

+155
-32
lines changed

7 files changed

+155
-32
lines changed

‎easybuild/framework/easyblock.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2170,7 +2170,7 @@ def prepare_step(self, start_dir=True, load_tc_deps_modules=True):
21702170
curr_modpaths = curr_module_paths()
21712171
for init_modpath in init_modpaths:
21722172
full_mod_path = os.path.join(self.installdir_mod, init_modpath)
2173-
if full_mod_path not in curr_modpaths:
2173+
if os.path.exists(full_mod_path) and full_mod_path not in curr_modpaths:
21742174
self.modules_tool.prepend_module_path(full_mod_path)
21752175

21762176
# prepare toolchain: load toolchain module and dependencies, set up build environment

‎easybuild/tools/filetools.py

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,24 @@ def det_common_path_prefix(paths):
537537
return None
538538

539539

540+
def normalize_path(path):
541+
"""Normalize path removing empty and dot components.
542+
543+
Similar to os.path.normpath but does not resolve '..' which may return a wrong path when symlinks are used
544+
"""
545+
# In POSIX 3 or more leading slashes are equivalent to 1
546+
if path.startswith(os.path.sep):
547+
if path.startswith(os.path.sep * 2) and not path.startswith(os.path.sep * 3):
548+
start_slashes = os.path.sep * 2
549+
else:
550+
start_slashes = os.path.sep
551+
else:
552+
start_slashes = ''
553+
554+
filtered_comps = (comp for comp in path.split(os.path.sep) if comp and comp != '.')
555+
return start_slashes + os.path.sep.join(filtered_comps)
556+
557+
540558
def is_alt_pypi_url(url):
541559
"""Determine whether specified URL is already an alternate PyPI URL, i.e. whether it contains a hash."""
542560
# example: .../packages/5b/03/e135b19fadeb9b1ccb45eac9f60ca2dc3afe72d099f6bd84e03cb131f9bf/easybuild-2.7.0.tar.gz

‎easybuild/tools/modules.py

Lines changed: 45 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS
4747
from easybuild.tools.config import build_option, get_modules_tool, install_path
4848
from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars
49-
from easybuild.tools.filetools import convert_name, mkdir, path_matches, read_file, which, write_file
49+
from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
5050
from easybuild.tools.module_naming_scheme.mns import DEVEL_MODULE_SUFFIX
5151
from easybuild.tools.py2vs3 import subprocess_popen_text
5252
from easybuild.tools.run import run_cmd
@@ -362,8 +362,12 @@ def use(self, path, priority=None):
362362
self.log.info("Ignoring specified priority '%s' when running 'module use %s' (Lmod-specific)",
363363
priority, path)
364364

365-
# make sure path exists before we add it
366-
mkdir(path, parents=True)
365+
if not path:
366+
raise EasyBuildError("Cannot add empty path to $MODULEPATH")
367+
if not os.path.exists(path):
368+
self.log.deprecated("Path '%s' for module.use should exist" % path, '5.0')
369+
# make sure path exists before we add it
370+
mkdir(path, parents=True)
367371
self.run_module(['use', path])
368372

369373
def unuse(self, path):
@@ -377,7 +381,8 @@ def add_module_path(self, path, set_mod_paths=True):
377381
:param path: path to add to $MODULEPATH via 'use'
378382
:param set_mod_paths: (re)set self.mod_paths
379383
"""
380-
if path not in curr_module_paths():
384+
path = normalize_path(path)
385+
if path not in curr_module_paths(normalize=True):
381386
# add module path via 'module use' and make sure self.mod_paths is synced
382387
self.use(path)
383388
if set_mod_paths:
@@ -391,8 +396,14 @@ def remove_module_path(self, path, set_mod_paths=True):
391396
:param set_mod_paths: (re)set self.mod_paths
392397
"""
393398
# remove module path via 'module unuse' and make sure self.mod_paths is synced
394-
if path in curr_module_paths():
395-
self.unuse(path)
399+
path = normalize_path(path)
400+
try:
401+
# Unuse the path that is actually present in the environment
402+
module_path = next(p for p in curr_module_paths() if normalize_path(p) == path)
403+
except StopIteration:
404+
pass
405+
else:
406+
self.unuse(module_path)
396407

397408
if set_mod_paths:
398409
self.set_mod_paths()
@@ -431,6 +442,7 @@ def check_module_path(self):
431442
eb_modpath = os.path.join(install_path(typ='modules'), build_option('suffix_modules_path'))
432443

433444
# make sure EasyBuild module path is in 1st place
445+
mkdir(eb_modpath, parents=True)
434446
self.prepend_module_path(eb_modpath)
435447
self.log.info("Prepended list of module paths with path used by EasyBuild: %s" % eb_modpath)
436448

@@ -665,7 +677,8 @@ def load(self, modules, mod_paths=None, purge=False, init_env=None, allow_reload
665677
# extend $MODULEPATH if needed
666678
for mod_path in mod_paths:
667679
full_mod_path = os.path.join(install_path('mod'), build_option('suffix_modules_path'), mod_path)
668-
self.prepend_module_path(full_mod_path)
680+
if os.path.exists(full_mod_path):
681+
self.prepend_module_path(full_mod_path)
669682

670683
loaded_modules = self.loaded_modules()
671684
for mod in modules:
@@ -1286,8 +1299,14 @@ def remove_module_path(self, path, set_mod_paths=True):
12861299
# remove module path via 'module use' and make sure self.mod_paths is synced
12871300
# modulecmd.tcl keeps track of how often a path was added via 'module use',
12881301
# so we need to check to make sure it's really removed
1289-
while path in curr_module_paths():
1290-
self.unuse(path)
1302+
path = normalize_path(path)
1303+
while True:
1304+
try:
1305+
# Unuse the path that is actually present in the environment
1306+
module_path = next(p for p in curr_module_paths() if normalize_path(p) == path)
1307+
except StopIteration:
1308+
break
1309+
self.unuse(module_path)
12911310
if set_mod_paths:
12921311
self.set_mod_paths()
12931312

@@ -1422,8 +1441,12 @@ def use(self, path, priority=None):
14221441
:param path: path to add to $MODULEPATH
14231442
:param priority: priority for this path in $MODULEPATH (Lmod-specific)
14241443
"""
1425-
# make sure path exists before we add it
1426-
mkdir(path, parents=True)
1444+
if not path:
1445+
raise EasyBuildError("Cannot add empty path to $MODULEPATH")
1446+
if not os.path.exists(path):
1447+
self.log.deprecated("Path '%s' for module.use should exist" % path, '5.0')
1448+
# make sure path exists before we add it
1449+
mkdir(path, parents=True)
14271450

14281451
if priority:
14291452
self.run_module(['use', '--priority', str(priority), path])
@@ -1433,11 +1456,12 @@ def use(self, path, priority=None):
14331456
if os.environ.get('__LMOD_Priority_MODULEPATH'):
14341457
self.run_module(['use', path])
14351458
else:
1459+
path = normalize_path(path)
14361460
cur_mod_path = os.environ.get('MODULEPATH')
14371461
if cur_mod_path is None:
14381462
new_mod_path = path
14391463
else:
1440-
new_mod_path = [path] + [p for p in cur_mod_path.split(':') if p != path]
1464+
new_mod_path = [path] + [p for p in cur_mod_path.split(':') if normalize_path(p) != path]
14411465
new_mod_path = ':'.join(new_mod_path)
14421466
self.log.debug('Changing MODULEPATH from %s to %s' %
14431467
('<unset>' if cur_mod_path is None else cur_mod_path, new_mod_path))
@@ -1453,7 +1477,8 @@ def unuse(self, path):
14531477
self.log.debug('Changing MODULEPATH from %s to <unset>' % cur_mod_path)
14541478
del os.environ['MODULEPATH']
14551479
else:
1456-
new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if p != path)
1480+
path = normalize_path(path)
1481+
new_mod_path = ':'.join(p for p in cur_mod_path.split(':') if normalize_path(p) != path)
14571482
if new_mod_path != cur_mod_path:
14581483
self.log.debug('Changing MODULEPATH from %s to %s' % (cur_mod_path, new_mod_path))
14591484
os.environ['MODULEPATH'] = new_mod_path
@@ -1603,12 +1628,17 @@ def get_software_version(name):
16031628
return version
16041629

16051630

1606-
def curr_module_paths():
1631+
def curr_module_paths(normalize=False):
16071632
"""
16081633
Return a list of current module paths.
1634+
1635+
:param normalize: Normalize the paths
16091636
"""
16101637
# avoid empty or nonexistent paths, which don't make any sense
1611-
return [p for p in os.environ.get('MODULEPATH', '').split(':') if p and os.path.exists(p)]
1638+
module_paths = (p for p in os.environ.get('MODULEPATH', '').split(':') if p and os.path.exists(p))
1639+
if normalize:
1640+
module_paths = (normalize_path(p) for p in module_paths)
1641+
return list(module_paths)
16121642

16131643

16141644
def mk_module_path(paths):

‎easybuild/tools/toolchain/toolchain.py

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -636,7 +636,9 @@ def _load_toolchain_module(self, silent=False):
636636
if self.init_modpaths:
637637
mod_path_suffix = build_option('suffix_modules_path')
638638
for modpath in self.init_modpaths:
639-
self.modules_tool.prepend_module_path(os.path.join(install_path('mod'), mod_path_suffix, modpath))
639+
modpath = os.path.join(install_path('mod'), mod_path_suffix, modpath)
640+
if os.path.exists(modpath):
641+
self.modules_tool.prepend_module_path(modpath)
640642

641643
# load modules for all dependencies
642644
self.log.debug("Loading module for toolchain: %s", tc_mod)

‎test/framework/easyconfig.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1666,7 +1666,6 @@ def test_external_dependencies(self):
16661666
# by adding a couple of matching module files with some useful data in them
16671667
# (use Tcl syntax, so it works with all varieties of module tools)
16681668
mod_dir = os.path.join(self.test_prefix, 'modules')
1669-
self.modtool.use(mod_dir)
16701669

16711670
pi_mod_txt = '\n'.join([
16721671
"#%Module",
@@ -1691,6 +1690,8 @@ def test_external_dependencies(self):
16911690
])
16921691
write_file(os.path.join(mod_dir, 'foobar/2.3.4'), foobar_mod_txt)
16931692

1693+
self.modtool.use(mod_dir)
1694+
16941695
ec = EasyConfig(toy_ec)
16951696
deps = ec.dependencies()
16961697

‎test/framework/filetools.py

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,21 @@ def test_common_path_prefix(self):
357357
self.assertEqual(ft.det_common_path_prefix(['foo']), None)
358358
self.assertEqual(ft.det_common_path_prefix([]), None)
359359

360+
def test_normalize_path(self):
361+
"""Test normalize_path"""
362+
self.assertEqual(ft.normalize_path(''), '')
363+
self.assertEqual(ft.normalize_path('/'), '/')
364+
self.assertEqual(ft.normalize_path('//'), '//')
365+
self.assertEqual(ft.normalize_path('///'), '/')
366+
self.assertEqual(ft.normalize_path('/foo/bar/baz'), '/foo/bar/baz')
367+
self.assertEqual(ft.normalize_path('/foo//bar/././baz/'), '/foo/bar/baz')
368+
self.assertEqual(ft.normalize_path('foo//bar/././baz/'), 'foo/bar/baz')
369+
self.assertEqual(ft.normalize_path('//foo//bar/././baz/'), '//foo/bar/baz')
370+
self.assertEqual(ft.normalize_path('///foo//bar/././baz/'), '/foo/bar/baz')
371+
self.assertEqual(ft.normalize_path('////foo//bar/././baz/'), '/foo/bar/baz')
372+
self.assertEqual(ft.normalize_path('/././foo//bar/././baz/'), '/foo/bar/baz')
373+
self.assertEqual(ft.normalize_path('//././foo//bar/././baz/'), '//foo/bar/baz')
374+
360375
def test_download_file(self):
361376
"""Test download_file function."""
362377
fn = 'toy-0.0.tar.gz'

‎test/framework/modules.py

Lines changed: 71 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -531,14 +531,15 @@ def test_check_module_path(self):
531531
os.environ['MODULEPATH'] = test2
532532
modtool.check_module_path()
533533
self.assertEqual(modtool.mod_paths, [mod_install_dir, test1, test2])
534-
self.assertEqual(os.environ['MODULEPATH'], mod_install_dir + ':' + test1 + ':' + test2)
534+
self.assertEqual(os.environ['MODULEPATH'], os.pathsep.join([mod_install_dir, test1, test2]))
535535

536536
# check behaviour if non-existing directories are included in $MODULEPATH
537537
os.environ['MODULEPATH'] = '%s:/does/not/exist:%s' % (test3, test2)
538538
modtool.check_module_path()
539539
# non-existing dir is filtered from mod_paths, but stays in $MODULEPATH
540540
self.assertEqual(modtool.mod_paths, [mod_install_dir, test1, test3, test2])
541-
self.assertEqual(os.environ['MODULEPATH'], ':'.join([mod_install_dir, test1, test3, '/does/not/exist', test2]))
541+
self.assertEqual(os.environ['MODULEPATH'],
542+
os.pathsep.join([mod_install_dir, test1, test3, '/does/not/exist', test2]))
542543

543544
def test_check_module_path_hmns(self):
544545
"""Test behaviour of check_module_path with HierarchicalMNS."""
@@ -1090,8 +1091,9 @@ def test_module_caches(self):
10901091
"""Test module caches and invalidate_module_caches_for function."""
10911092
self.assertEqual(mod.MODULE_AVAIL_CACHE, {})
10921093

1093-
# purposely extending $MODULEPATH with non-existing path, should be handled fine
1094+
# purposely extending $MODULEPATH with an empty path, should be handled fine
10941095
nonpath = os.path.join(self.test_prefix, 'nosuchfileordirectory')
1096+
mkdir(nonpath)
10951097
self.modtool.use(nonpath)
10961098
modulepaths = [p for p in os.environ.get('MODULEPATH', '').split(os.pathsep) if p]
10971099
self.assertTrue(any([os.path.samefile(nonpath, mp) for mp in modulepaths]))
@@ -1164,6 +1166,11 @@ def test_module_use_unuse(self):
11641166
self.modtool.use(test_dir3)
11651167
self.assertTrue(os.environ['MODULEPATH'].startswith('%s:' % test_dir3))
11661168

1169+
# Adding an empty modulepath is not possible
1170+
modulepath = os.environ.get('MODULEPATH', '')
1171+
self.assertErrorRegex(EasyBuildError, "Cannot add empty path", self.modtool.use, '')
1172+
self.assertEqual(os.environ.get('MODULEPATH', ''), modulepath)
1173+
11671174
# make sure the right test module is loaded
11681175
self.modtool.load(['test'])
11691176
self.assertEqual(os.getenv('TEST123'), 'three')
@@ -1224,17 +1231,67 @@ def test_module_use_unuse(self):
12241231
self.assertFalse('MODULEPATH' in os.environ)
12251232
os.environ['MODULEPATH'] = old_module_path # Restore
12261233

1227-
# Using an empty path still works (technically) (Lmod only, ignored by Tcl)
1228-
old_module_path = os.environ['MODULEPATH']
1229-
self.modtool.use('')
1230-
self.assertEqual(os.environ['MODULEPATH'], ':' + old_module_path)
1231-
self.modtool.unuse('')
1232-
self.assertEqual(os.environ['MODULEPATH'], old_module_path)
1233-
# Even works when the whole path is empty
1234-
os.environ['MODULEPATH'] = ''
1235-
self.modtool.unuse('')
1236-
self.assertFalse('MODULEPATH' in os.environ)
1237-
os.environ['MODULEPATH'] = old_module_path # Restore
1234+
def test_add_and_remove_module_path(self):
1235+
"""Test add_module_path and whether remove_module_path undoes changes of add_module_path"""
1236+
test_dir1 = tempfile.mkdtemp(suffix="_dir1")
1237+
test_dir2 = tempfile.mkdtemp(suffix="_dir2")
1238+
old_module_path = os.environ.get('MODULEPATH')
1239+
del os.environ['MODULEPATH']
1240+
self.modtool.add_module_path(test_dir1)
1241+
self.assertEqual(os.environ['MODULEPATH'], test_dir1)
1242+
self.modtool.add_module_path(test_dir2)
1243+
test_dir_2_and_1 = os.pathsep.join([test_dir2, test_dir1])
1244+
self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1)
1245+
# Adding the same path does not change the path
1246+
self.modtool.add_module_path(test_dir1)
1247+
self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1)
1248+
self.modtool.add_module_path(test_dir2)
1249+
self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1)
1250+
# Even when a (meaningless) slash is added
1251+
# This occurs when using an empty modules directory name
1252+
self.modtool.add_module_path(os.path.join(test_dir1, ''))
1253+
self.assertEqual(os.environ['MODULEPATH'], test_dir_2_and_1)
1254+
1255+
# Similar tests for remove_module_path
1256+
self.modtool.remove_module_path(test_dir2)
1257+
self.assertEqual(os.environ['MODULEPATH'], test_dir1)
1258+
# Same again -> no-op
1259+
self.modtool.remove_module_path(test_dir2)
1260+
self.assertEqual(os.environ['MODULEPATH'], test_dir1)
1261+
# And with empty last part
1262+
self.modtool.remove_module_path(os.path.join(test_dir1, ''))
1263+
self.assertEqual(os.environ.get('MODULEPATH', ''), '')
1264+
1265+
# And with some more trickery
1266+
# Lmod seems to remove empty paths: /foo//bar/. -> /foo/bar
1267+
# Environment-Modules 4.x seems to resolve relative paths: /foo/../foo -> /foo
1268+
# Hence we can only check the real paths
1269+
def get_resolved_module_path():
1270+
return os.pathsep.join(os.path.realpath(p) for p in os.environ['MODULEPATH'].split(os.pathsep))
1271+
1272+
test_dir1_relative = os.path.join(test_dir1, '..', os.path.basename(test_dir1))
1273+
test_dir2_dot = os.path.join(os.path.dirname(test_dir2), '.', os.path.basename(test_dir2))
1274+
self.modtool.add_module_path(test_dir1_relative)
1275+
self.assertEqual(get_resolved_module_path(), test_dir1)
1276+
# Adding the same path, but in a different form may be possible, but may also be ignored, e.g. in EnvModules
1277+
self.modtool.add_module_path(test_dir1)
1278+
if get_resolved_module_path() != test_dir1:
1279+
self.assertEqual(get_resolved_module_path(), os.pathsep.join([test_dir1, test_dir1]))
1280+
self.modtool.remove_module_path(test_dir1)
1281+
self.assertEqual(get_resolved_module_path(), test_dir1)
1282+
self.modtool.add_module_path(test_dir2_dot)
1283+
self.assertEqual(get_resolved_module_path(), test_dir_2_and_1)
1284+
self.modtool.remove_module_path(test_dir2_dot)
1285+
self.assertEqual(get_resolved_module_path(), test_dir1)
1286+
# Force adding such a dot path which can be removed with either variant
1287+
os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative])
1288+
self.modtool.remove_module_path(test_dir2_dot)
1289+
self.assertEqual(get_resolved_module_path(), test_dir1)
1290+
os.environ['MODULEPATH'] = os.pathsep.join([test_dir2_dot, test_dir1_relative])
1291+
self.modtool.remove_module_path(test_dir2)
1292+
self.assertEqual(get_resolved_module_path(), test_dir1)
1293+
1294+
os.environ['MODULEPATH'] = old_module_path # Restore
12381295

12391296
def test_module_use_bash(self):
12401297
"""Test whether effect of 'module use' is preserved when a new bash session is started."""

0 commit comments

Comments
 (0)