Merge pull request #4653 from lexming/cpath-mod

refactor generation of required environment variables in module files + deprecate `make_module_req_guess` method in `EasyBlock` class

Author: boegel

easybuild/framework/easyblock.py

@@ -89,6 +89,7 @@ def __init__(self, *args, **kwargs):
             self.installdir = self.master.installdir
             self.modules_tool = self.master.modules_tool
             self.module_generator = self.master.module_generator
+            self.module_load_environment = self.master.module_load_environment
             self.robot_path = self.master.robot_path
             self.is_extension = True
             self.unpack_options = None

easybuild/framework/extensioneasyblock.py

@@ -168,13 +168,16 @@
 LOCAL_VAR_NAMING_CHECK_WARN = WARN
 LOCAL_VAR_NAMING_CHECKS = [LOCAL_VAR_NAMING_CHECK_ERROR, LOCAL_VAR_NAMING_CHECK_LOG, LOCAL_VAR_NAMING_CHECK_WARN]
 
-
 OUTPUT_STYLE_AUTO = 'auto'
 OUTPUT_STYLE_BASIC = 'basic'
 OUTPUT_STYLE_NO_COLOR = 'no_color'
 OUTPUT_STYLE_RICH = 'rich'
 OUTPUT_STYLES = (OUTPUT_STYLE_AUTO, OUTPUT_STYLE_BASIC, OUTPUT_STYLE_NO_COLOR, OUTPUT_STYLE_RICH)
 
+SEARCH_PATH_BIN_DIRS = ['bin']
+SEARCH_PATH_HEADER_DIRS = ['include']
+SEARCH_PATH_LIB_DIRS = ['lib', 'lib64']
+
 PYTHONPATH = 'PYTHONPATH'
 EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
 PYTHON_SEARCH_PATH_TYPES = [PYTHONPATH, EBPYTHONPREFIXES]

easybuild/tools/config.py

@@ -41,12 +41,13 @@
 import os
 import re
 import shlex
+from enum import Enum
 
 from easybuild.base import fancylogger
 from easybuild.tools import LooseVersion
 from easybuild.tools.build_log import EasyBuildError, EasyBuildExit, print_warning
-from easybuild.tools.config import ERROR, IGNORE, PURGE, UNLOAD, UNSET
-from easybuild.tools.config import EBROOT_ENV_VAR_ACTIONS, LOADED_MODULES_ACTIONS
+from easybuild.tools.config import ERROR, EBROOT_ENV_VAR_ACTIONS, IGNORE, LOADED_MODULES_ACTIONS, PURGE
+from easybuild.tools.config import SEARCH_PATH_BIN_DIRS, SEARCH_PATH_HEADER_DIRS, SEARCH_PATH_LIB_DIRS, UNLOAD, UNSET
 from easybuild.tools.config import build_option, get_modules_tool, install_path
 from easybuild.tools.environment import ORIG_OS_ENVIRON, restore_env, setvar, unset_env_vars
 from easybuild.tools.filetools import convert_name, mkdir, normalize_path, path_matches, read_file, which, write_file
@@ -55,6 +56,7 @@
 from easybuild.tools.systemtools import get_shared_lib_ext
 from easybuild.tools.utilities import get_subclasses, nub
 
+
 # software root/version environment variable name prefixes
 ROOT_ENV_VAR_NAME_PREFIX = "EBROOT"
 VERSION_ENV_VAR_NAME_PREFIX = "EBVERSION"
@@ -131,6 +133,199 @@
 _log = fancylogger.getLogger('modules', fname=False)
 
 
+class ModEnvVarType(Enum):
+    """
+    Possible types of ModuleEnvironmentVariable:
+    - STRING: (list of) strings with no further meaning
+    - PATH: (list of) of paths to existing directories or files
+    - PATH_WITH_FILES: (list of) of paths to existing directories containing
+      one or more files
+    - PATH_WITH_TOP_FILES: (list of) of paths to existing directories
+      containing one or more files in its top directory
+    - """
+    STRING, PATH, PATH_WITH_FILES, PATH_WITH_TOP_FILES = range(0, 4)
+
+
+class ModuleEnvironmentVariable:
+    """
+    Environment variable data structure for modules
+    Contents of environment variable is a list of unique strings
+    """
+
+    def __init__(self, contents, var_type=None, delim=os.pathsep):
+        """
+        Initialize new environment variable
+        Actual contents of the environment variable are held in self.contents
+        By default, environment variable is a (list of) paths with files in them
+        Existence of paths and their contents are not checked at init
+        """
+        self.contents = contents
+        self.delim = delim
+
+        if var_type is None:
+            var_type = ModEnvVarType.PATH_WITH_FILES
+        self.type = var_type
+
+        self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
+
+    def __repr__(self):
+        return repr(self.contents)
+
+    def __str__(self):
+        return self.delim.join(self.contents)
+
+    def __iter__(self):
+        return iter(self.contents)
+
+    @property
+    def contents(self):
+        return self._contents
+
+    @contents.setter
+    def contents(self, value):
+        """Enforce that contents is a list of strings"""
+        if isinstance(value, str):
+            value = [value]
+
+        try:
+            str_list = [str(path) for path in value]
+        except TypeError as err:
+            raise TypeError("ModuleEnvironmentVariable.contents must be a list of strings") from err
+
+        self._contents = nub(str_list)  # remove duplicates and keep order
+
+    @property
+    def type(self):
+        return self._type
+
+    @type.setter
+    def type(self, value):
+        """Convert type to VarType"""
+        if isinstance(value, ModEnvVarType):
+            self._type = value
+        else:
+            try:
+                self._type = ModEnvVarType[value]
+            except KeyError as err:
+                raise EasyBuildError(f"Cannot create ModuleEnvironmentVariable with type {value}") from err
+
+    def append(self, item):
+        """Shortcut to append to list of contents"""
+        self.contents += [item]
+
+    def extend(self, item):
+        """Shortcut to extend list of contents"""
+        self.contents += item
+
+    def prepend(self, item):
+        """Shortcut to prepend item to list of contents"""
+        self.contents = [item] + self.contents
+
+    def update(self, item):
+        """Shortcut to replace list of contents with item"""
+        self.contents = item
+
+    def remove(self, *args):
+        """Shortcut to remove items from list of contents"""
+        try:
+            self.contents.remove(*args)
+        except ValueError:
+            # item is not in the list, move along
+            self.log.debug(f"ModuleEnvironmentVariable does not contain item: {' '.join(args)}")
+
+    @property
+    def is_path(self):
+        path_like_types = [
+            ModEnvVarType.PATH,
+            ModEnvVarType.PATH_WITH_FILES,
+            ModEnvVarType.PATH_WITH_TOP_FILES,
+        ]
+        return self.type in path_like_types
+
+
+class ModuleLoadEnvironment:
+    """Changes to environment variables that should be made when environment module is loaded"""
+
+    def __init__(self):
+        """
+        Initialize default environment definition
+        Paths are relative to root of installation directory
+        """
+        self.ACLOCAL_PATH = [os.path.join('share', 'aclocal')]
+        self.CLASSPATH = ['*.jar']
+        self.CMAKE_LIBRARY_PATH = ['lib64']  # only needed for installations with standalone lib64
+        self.CMAKE_PREFIX_PATH = ['']
+        self.CPATH = SEARCH_PATH_HEADER_DIRS
+        self.GI_TYPELIB_PATH = [os.path.join(x, 'girepository-*') for x in SEARCH_PATH_LIB_DIRS]
+        self.LD_LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
+        self.LIBRARY_PATH = SEARCH_PATH_LIB_DIRS
+        self.MANPATH = ['man', os.path.join('share', 'man')]
+        self.PATH = SEARCH_PATH_BIN_DIRS + ['sbin']
+        self.PKG_CONFIG_PATH = [os.path.join(x, 'pkgconfig') for x in SEARCH_PATH_LIB_DIRS + ['share']]
+        self.XDG_DATA_DIRS = ['share']
+
+    def __setattr__(self, name, value):
+        """
+        Specific restrictions for ModuleLoadEnvironment attributes:
+        - attribute names are uppercase
+        - attributes are instances of ModuleEnvironmentVariable
+        """
+        if name != name.upper():
+            raise EasyBuildError(f"Names of ModuleLoadEnvironment attributes must be uppercase, got '{name}'")
+        try:
+            (contents, kwargs) = value
+        except ValueError:
+            contents, kwargs = value, {}
+
+        if not isinstance(kwargs, dict):
+            contents, kwargs = value, {}
+
+        # special variables that require files in their top directories
+        if name in ('LD_LIBRARY_PATH', 'PATH'):
+            kwargs.update({'var_type': ModEnvVarType.PATH_WITH_TOP_FILES})
+
+        return super().__setattr__(name, ModuleEnvironmentVariable(contents, **kwargs))
+
+    def __iter__(self):
+        """Make the class iterable"""
+        yield from self.__dict__
+
+    def items(self):
+        """
+        Return key-value pairs for each attribute that is a ModuleEnvironmentVariable
+        - key = attribute name
+        - value = its "contents" attribute
+        """
+        for attr in self.__dict__:
+            yield attr, getattr(self, attr)
+
+    def update(self, new_env):
+        """Update contents of environment from given dictionary"""
+        try:
+            for envar_name, envar_contents in new_env.items():
+                setattr(self, envar_name, envar_contents)
+        except AttributeError as err:
+            raise EasyBuildError("Cannot update ModuleLoadEnvironment from a non-dict variable") from err
+
+    @property
+    def as_dict(self):
+        """
+        Return dict with mapping of ModuleEnvironmentVariables names with their contents
+        """
+        return dict(self.items())
+
+    @property
+    def environ(self):
+        """
+        Return dict with mapping of ModuleEnvironmentVariables names with their contents
+        Equivalent in shape to os.environ
+        """
+        mapping = {}
+        for envar_name, envar_contents in self.items():
+            mapping.update({envar_name: str(envar_contents)})
+        return mapping
+
+
 class ModulesTool(object):
     """An abstract interface to a tool that deals with modules."""
     # name of this modules tool (used in log/warning/error messages)