Skip to content

Commit 09609b7

Browse files
authored
Merge pull request #4645 from lexming/cpath
add `--search-path-cpp-headers` configuration option to control how EasyBuild sets paths to headers at build time
2 parents fb6ff38 + 485843d commit 09609b7

File tree

11 files changed

+183
-73
lines changed

11 files changed

+183
-73
lines changed

easybuild/toolchains/fcc.py

Lines changed: 6 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -40,39 +40,10 @@ class FCC(FujitsuCompiler):
4040
OPTIONAL = False
4141

4242
# override in order to add an exception for the Fujitsu lang/tcsds module
43-
def _add_dependency_variables(self, names=None, cpp=None, ld=None):
44-
""" Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
45-
names should be a list of strings containing the name of the dependency
43+
def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
4644
"""
47-
cpp_paths = ['include']
48-
ld_paths = ['lib64', 'lib']
49-
50-
if cpp is not None:
51-
for p in cpp:
52-
if p not in cpp_paths:
53-
cpp_paths.append(p)
54-
if ld is not None:
55-
for p in ld:
56-
if p not in ld_paths:
57-
ld_paths.append(p)
58-
59-
if not names:
60-
deps = self.dependencies
61-
else:
62-
deps = [{'name': name} for name in names if name is not None]
63-
64-
# collect software install prefixes for dependencies
65-
roots = []
66-
for dep in deps:
67-
if dep.get('external_module', False):
68-
# for software names provided via external modules, install prefix may be unknown
69-
names = dep['external_module_metadata'].get('name', [])
70-
roots.extend([root for root in self.get_software_root(names) if root is not None])
71-
else:
72-
roots.extend(self.get_software_root(dep['name']))
73-
74-
for root in roots:
75-
# skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
76-
if 'tcsds' not in root:
77-
self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
78-
self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
45+
Append prepocessor paths for given dependency root directory
46+
"""
47+
# skip Fujitsu's 'lang/tcsds' module, including the top level include breaks vectorization in clang mode
48+
if "tcsds" not in dep_root:
49+
super()._add_dependency_cpp_headers(dep_root, extra_dirs=extra_dirs)

easybuild/toolchains/linalg/acml.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,9 @@ def _set_blas_variables(self):
7777
for root in self.get_software_root(self.BLAS_MODULE_NAME):
7878
subdirs = self.ACML_SUBDIRS_MAP[self.COMPILER_FAMILY]
7979
self.BLAS_LIB_DIR = [os.path.join(x, 'lib') for x in subdirs]
80-
self.variables.append_exists('LDFLAGS', root, self.BLAS_LIB_DIR, append_all=True)
80+
self._add_dependency_linker_paths(root, extra_dirs=self.BLAS_LIB_DIR)
8181
incdirs = [os.path.join(x, 'include') for x in subdirs]
82-
self.variables.append_exists('CPPFLAGS', root, incdirs, append_all=True)
82+
self._add_dependency_cpp_headers(root, extra_dirs=incdirs)
8383
except Exception:
8484
raise EasyBuildError("_set_blas_variables: ACML set LDFLAGS/CPPFLAGS unknown entry in ACML_SUBDIRS_MAP "
8585
"with compiler family %s", self.COMPILER_FAMILY)

easybuild/tools/config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
265265
'rpath_filter',
266266
'rpath_override_dirs',
267267
'required_linked_shared_libs',
268+
'search_path_cpp_headers',
268269
'skip',
269270
'software_commit',
270271
'stop',

easybuild/tools/options.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@
102102
from easybuild.tools.run import run_shell_cmd
103103
from easybuild.tools.package.utilities import avail_package_naming_schemes
104104
from easybuild.tools.toolchain.compiler import DEFAULT_OPT_LEVEL, OPTARCH_MAP_CHAR, OPTARCH_SEP, Compiler
105+
from easybuild.tools.toolchain.toolchain import SEARCH_PATH_CPP_HEADERS, DEFAULT_SEARCH_PATH_CPP_HEADERS
105106
from easybuild.tools.toolchain.toolchain import SYSTEM_TOOLCHAIN_NAME
106107
from easybuild.tools.repository.repository import avail_repositories
107108
from easybuild.tools.systemtools import DARWIN, UNKNOWN, check_python_version, get_cpu_architecture, get_cpu_family
@@ -637,6 +638,8 @@ def config_options(self):
637638
"(is passed as list of arguments to create the repository instance). "
638639
"For more info, use --avail-repositories."),
639640
'strlist', 'store', self.default_repositorypath),
641+
'search-path-cpp-headers': ("Search path used at build time for include directories", 'choice',
642+
'store', DEFAULT_SEARCH_PATH_CPP_HEADERS, [*SEARCH_PATH_CPP_HEADERS]),
640643
'sourcepath': ("Path(s) to where sources should be downloaded (string, colon-separated)",
641644
None, 'store', mk_full_default_path('sourcepath')),
642645
'subdir-modules': ("Installpath subdir for modules", None, 'store', DEFAULT_PATH_SUBDIRS['subdir_modules']),

easybuild/tools/toolchain/compiler.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ class Compiler(Toolchain):
9393
'vectorize': (None, "Enable compiler auto-vectorization, default except for noopt and lowopt"),
9494
'packed-linker-options': (False, "Pack the linker options as comma separated list"), # ScaLAPACK mainly
9595
'rpath': (True, "Use RPATH wrappers when --rpath is enabled in EasyBuild configuration"),
96+
'search-path-cpp-headers': (None, "Search path used at build time for include directories"),
9697
'extra_cflags': (None, "Specify extra CFLAGS options."),
9798
'extra_cxxflags': (None, "Specify extra CXXFLAGS options."),
9899
'extra_fflags': (None, "Specify extra FFLAGS options."),

easybuild/tools/toolchain/constants.py

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
* Kenneth Hoste (Ghent University)
3232
"""
3333

34+
from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs, FlagList
35+
from easybuild.tools.toolchain.variables import IncludePaths, LibraryList, LinkLibraryPaths, SearchPaths
3436
from easybuild.tools.variables import AbsPathList
35-
from easybuild.tools.toolchain.variables import CommandFlagList, CommaSharedLibs, CommaStaticLibs
36-
from easybuild.tools.toolchain.variables import FlagList, IncludePaths, LibraryList, LinkLibraryPaths
3737

3838

3939
COMPILER_VARIABLES = [
@@ -65,7 +65,13 @@
6565
('LDFLAGS', 'Flags passed to linker'), # TODO: overridden by command line?
6666
],
6767
IncludePaths: [
68-
('CPPFLAGS', 'Precompiler flags'),
68+
('CPPFLAGS', 'Preprocessor flags'),
69+
],
70+
SearchPaths: [
71+
('CPATH', 'Location of C/C++ header files'),
72+
('C_INCLUDE_PATH', 'Location of C header files'),
73+
('CPLUS_INCLUDE_PATH', 'Location of C++ header files'),
74+
('OBJC_INCLUDE_PATH', 'Location of Objective C header files'),
6975
],
7076
CommandFlagList: COMPILER_VARIABLES,
7177
}

easybuild/tools/toolchain/toolchain.py

Lines changed: 72 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
from easybuild.tools.systemtools import LINUX, get_os_type
7070
from easybuild.tools.toolchain.options import ToolchainOptions
7171
from easybuild.tools.toolchain.toolchainvariables import ToolchainVariables
72-
from easybuild.tools.utilities import nub, trace_msg
72+
from easybuild.tools.utilities import nub, unique_ordered_extend, trace_msg
7373

7474

7575
_log = fancylogger.getLogger('tools.toolchain', fname=False)
@@ -95,6 +95,17 @@
9595
TOOLCHAIN_CAPABILITY_LAPACK_FAMILY,
9696
TOOLCHAIN_CAPABILITY_MPI_FAMILY,
9797
]
98+
# modes to handle CPP header search paths
99+
# see: https://gcc.gnu.org/onlinedocs/cpp/Environment-Variables.html
100+
SEARCH_PATH_CPP_HEADERS_FLAGS = "CPPFLAGS"
101+
SEARCH_PATH_CPP_HEADERS_CPATH = "CPATH"
102+
SEARCH_PATH_CPP_HEADERS_INCLUDE = "INCLUDE_PATHS"
103+
SEARCH_PATH_CPP_HEADERS = {
104+
SEARCH_PATH_CPP_HEADERS_FLAGS: ["CPPFLAGS"],
105+
SEARCH_PATH_CPP_HEADERS_CPATH: ["CPATH"],
106+
SEARCH_PATH_CPP_HEADERS_INCLUDE: ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
107+
}
108+
DEFAULT_SEARCH_PATH_CPP_HEADERS = SEARCH_PATH_CPP_HEADERS_FLAGS
98109

99110

100111
def is_system_toolchain(tc_name):
@@ -850,7 +861,7 @@ def prepare(self, onlymod=None, deps=None, silent=False, loadmod=True,
850861
else:
851862
self.log.debug("prepare: set additional variables onlymod=%s", onlymod)
852863

853-
# add LDFLAGS and CPPFLAGS from dependencies to self.vars
864+
# add linker and preprocessor paths of dependencies to self.vars
854865
self._add_dependency_variables()
855866
self.generate_vars()
856867
self._setenv_variables(onlymod, verbose=not silent)
@@ -1049,39 +1060,71 @@ def handle_sysroot(self):
10491060
setvar('PKG_CONFIG_PATH', os.pathsep.join(pkg_config_path))
10501061

10511062
def _add_dependency_variables(self, names=None, cpp=None, ld=None):
1052-
""" Add LDFLAGS and CPPFLAGS to the self.variables based on the dependencies
1053-
names should be a list of strings containing the name of the dependency
10541063
"""
1055-
cpp_paths = ['include']
1056-
ld_paths = ['lib64', 'lib']
1057-
1058-
if cpp is not None:
1059-
for p in cpp:
1060-
if p not in cpp_paths:
1061-
cpp_paths.append(p)
1062-
if ld is not None:
1063-
for p in ld:
1064-
if p not in ld_paths:
1065-
ld_paths.append(p)
1066-
1067-
if not names:
1068-
deps = self.dependencies
1069-
else:
1070-
deps = [{'name': name} for name in names if name is not None]
1064+
Add linker and preprocessor paths of dependencies to self.variables
1065+
:names: list of strings containing the name of the dependency
1066+
"""
1067+
# collect dependencies
1068+
dependencies = self.dependencies if names is None else [{"name": name} for name in names if name]
10711069

10721070
# collect software install prefixes for dependencies
1073-
roots = []
1074-
for dep in deps:
1075-
if dep.get('external_module', False):
1071+
dependency_roots = []
1072+
for dep in dependencies:
1073+
if dep.get("external_module", False):
10761074
# for software names provided via external modules, install prefix may be unknown
1077-
names = dep['external_module_metadata'].get('name', [])
1078-
roots.extend([root for root in self.get_software_root(names) if root is not None])
1075+
names = dep["external_module_metadata"].get("name", [])
1076+
dependency_roots.extend([root for root in self.get_software_root(names) if root is not None])
10791077
else:
1080-
roots.extend(self.get_software_root(dep['name']))
1078+
dependency_roots.extend(self.get_software_root(dep["name"]))
1079+
1080+
for root in dependency_roots:
1081+
self._add_dependency_cpp_headers(root, extra_dirs=cpp)
1082+
self._add_dependency_linker_paths(root, extra_dirs=ld)
1083+
1084+
def _add_dependency_cpp_headers(self, dep_root, extra_dirs=None):
1085+
"""
1086+
Append prepocessor paths for given dependency root directory
1087+
"""
1088+
if extra_dirs is None:
1089+
extra_dirs = ()
1090+
1091+
header_dirs = ["include"]
1092+
header_dirs = unique_ordered_extend(header_dirs, extra_dirs)
1093+
1094+
# mode of operation is defined by search-path-cpp-headers option
1095+
# toolchain option has precedence over build option
1096+
cpp_headers_mode = DEFAULT_SEARCH_PATH_CPP_HEADERS
1097+
build_opt = build_option("search_path_cpp_headers")
1098+
if self.options.get("search-path-cpp-headers") is not None:
1099+
cpp_headers_mode = self.options.option("search-path-cpp-headers")
1100+
self.log.debug("search-path-cpp-headers set by toolchain option: %s", cpp_headers_mode)
1101+
elif build_opt is not None:
1102+
cpp_headers_mode = build_opt
1103+
self.log.debug("search-path-cpp-headers set by build option: %s", cpp_headers_mode)
1104+
1105+
if cpp_headers_mode not in SEARCH_PATH_CPP_HEADERS:
1106+
raise EasyBuildError(
1107+
"Unknown value selected for option search-path-cpp-headers: %s. Choose one of: %s",
1108+
cpp_headers_mode, ", ".join(SEARCH_PATH_CPP_HEADERS)
1109+
)
1110+
1111+
for env_var in SEARCH_PATH_CPP_HEADERS[cpp_headers_mode]:
1112+
self.log.debug("Adding header paths to toolchain variable '%s': %s", env_var, dep_root)
1113+
self.variables.append_subdirs(env_var, dep_root, subdirs=header_dirs)
1114+
1115+
def _add_dependency_linker_paths(self, dep_root, extra_dirs=None):
1116+
"""
1117+
Append linker paths for given dependency root directory
1118+
"""
1119+
if extra_dirs is None:
1120+
extra_dirs = ()
1121+
1122+
lib_dirs = ["lib64", "lib"]
1123+
lib_dirs = unique_ordered_extend(lib_dirs, extra_dirs)
10811124

1082-
for root in roots:
1083-
self.variables.append_subdirs("CPPFLAGS", root, subdirs=cpp_paths)
1084-
self.variables.append_subdirs("LDFLAGS", root, subdirs=ld_paths)
1125+
env_var = "LDFLAGS"
1126+
self.log.debug("Adding lib paths to toolchain variable '%s': %s", env_var, dep_root)
1127+
self.variables.append_subdirs(env_var, dep_root, subdirs=lib_dirs)
10851128

10861129
def _setenv_variables(self, donotset=None, verbose=True):
10871130
"""Actually set the environment variables"""

easybuild/tools/toolchain/variables.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,11 @@ class LinkLibraryPaths(AbsPathList):
4646
PREFIX = '-L'
4747

4848

49+
class SearchPaths(AbsPathList):
50+
"""Colon-separated list of absolute paths"""
51+
SEPARATOR = ':'
52+
53+
4954
class FlagList(StrList):
5055
"""Flag list"""
5156
PREFIX = "-"

easybuild/tools/utilities.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -223,6 +223,23 @@ def nub(list_):
223223
return [x for x in list_ if x not in seen and not seen_add(x)]
224224

225225

226+
def unique_ordered_extend(base, affix):
227+
"""Extend base list with elements of affix list keeping order and without duplicates"""
228+
if isinstance(affix, str):
229+
# avoid extending with strings, as iterables generate wrong result without error
230+
raise EasyBuildError(f"given affix list is a string: {affix}")
231+
232+
try:
233+
ext_base = base.copy()
234+
ext_base.extend(affix)
235+
except TypeError as err:
236+
raise EasyBuildError(f"given affix list is not iterable: {affix}") from err
237+
except AttributeError as err:
238+
raise EasyBuildError(f"given base cannot be extended: {base}") from err
239+
240+
return nub(ext_base) # remove duplicates
241+
242+
226243
def get_class_for(modulepath, class_name):
227244
"""
228245
Get class for a given Python class name and Python module path.

test/framework/toolchain.py

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,6 +955,49 @@ def test_precision_flags(self):
955955

956956
self.modtool.purge()
957957

958+
def test_search_path_cpp_headers(self):
959+
"""Test functionality behind search-path-cpp-headers option"""
960+
cpp_headers_mode = {
961+
"CPPFLAGS": ["CPPFLAGS"],
962+
"CPATH": ["CPATH"],
963+
"INCLUDE_PATHS": ["C_INCLUDE_PATH", "CPLUS_INCLUDE_PATH", "OBJC_INCLUDE_PATH"],
964+
}
965+
# test without toolchain option
966+
for build_opt in cpp_headers_mode:
967+
init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True})
968+
tc = self.get_toolchain("foss", version="2018a")
969+
with self.mocked_stdout_stderr():
970+
tc.prepare()
971+
for env_var in cpp_headers_mode[build_opt]:
972+
assert_fail_msg = (
973+
f"Variable {env_var} required by search-path-cpp-headers build option '{build_opt}' "
974+
"not found in toolchain environment"
975+
)
976+
self.assertIn(env_var, tc.variables, assert_fail_msg)
977+
self.modtool.purge()
978+
# test with toolchain option
979+
for build_opt in cpp_headers_mode:
980+
init_config(build_options={"search_path_cpp_headers": build_opt, "silent": True})
981+
for tc_opt in cpp_headers_mode:
982+
tc = self.get_toolchain("foss", version="2018a")
983+
tc.set_options({"search-path-cpp-headers": tc_opt})
984+
with self.mocked_stdout_stderr():
985+
tc.prepare()
986+
for env_var in cpp_headers_mode[tc_opt]:
987+
assert_fail_msg = (
988+
f"Variable {env_var} required by search-path-cpp-headers toolchain option '{tc_opt}' "
989+
"not found in toolchain environment"
990+
)
991+
self.assertIn(env_var, tc.variables, assert_fail_msg)
992+
self.modtool.purge()
993+
# test wrong toolchain option
994+
tc = self.get_toolchain("foss", version="2018a")
995+
tc.set_options({"search-path-cpp-headers": "WRONG_MODE"})
996+
with self.mocked_stdout_stderr():
997+
error_pattern = "Unknown value selected for option search-path-cpp-headers"
998+
self.assertErrorRegex(EasyBuildError, error_pattern, tc.prepare)
999+
self.modtool.purge()
1000+
9581001
def test_cgoolf_toolchain(self):
9591002
"""Test for cgoolf toolchain."""
9601003
tc = self.get_toolchain("cgoolf", version="1.1.6")

0 commit comments

Comments
 (0)