Skip to content

add support for using oneAPI versions of 'intel' toolchain components #3665

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
May 17, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 42 additions & 21 deletions easybuild/framework/easyconfig/easyconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@ def cache_aware_func(toolchain, incl_capabilities=False):
return cache_aware_func


def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=False):
def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains, cands, incl_capabilities=False):
"""
Returns unique version for subtoolchain, in tc dict.
If there is no unique version:
Expand All @@ -229,30 +229,46 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains,
optional toolchains or system toolchain without add_system_to_minimal_toolchains.
* in all other cases, raises an exception.
"""
uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])
# init with "skipped"
subtoolchain_version = None

# system toolchain: bottom of the hierarchy
if is_system_toolchain(subtoolchain_name):
add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains')
if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'):
depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains"
_log.deprecated(depr_msg, '5.0')
add_system_to_minimal_toolchains = True

if add_system_to_minimal_toolchains and not incl_capabilities:
subtoolchain_version = ''
elif len(uniq_subtc_versions) == 1:
subtoolchain_version = list(uniq_subtc_versions)[0]
elif len(uniq_subtc_versions) == 0:
if subtoolchain_name not in optional_toolchains:
# ensure we always have a tuple of alternative subtoolchain names, which makes things easier below
if isinstance(subtoolchain_names, string_type):
subtoolchain_names = (subtoolchain_names,)

system_subtoolchain = False

for subtoolchain_name in subtoolchain_names:

uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])

# system toolchain: bottom of the hierarchy
if is_system_toolchain(subtoolchain_name):
add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains')
if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'):
depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains"
_log.deprecated(depr_msg, '5.0')
add_system_to_minimal_toolchains = True

system_subtoolchain = True

if add_system_to_minimal_toolchains and not incl_capabilities:
subtoolchain_version = ''
elif len(uniq_subtc_versions) == 1:
subtoolchain_version = list(uniq_subtc_versions)[0]
elif len(uniq_subtc_versions) > 1:
raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions)))

if subtoolchain_version is not None:
break

if not system_subtoolchain and subtoolchain_version is None:
if not all(n in optional_toolchains for n in subtoolchain_names):
subtoolchain_names = ' or '.join(subtoolchain_names)
# raise error if the subtoolchain considered now is not optional
raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s",
subtoolchain_name, current_tc['name'])
else:
raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions)))
subtoolchain_names, current_tc['name'])

return subtoolchain_version

Expand Down Expand Up @@ -356,11 +372,16 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
cands.append({'name': dep, 'version': current_tc_version})

# only retain candidates that match subtoolchain names
cands = [c for c in cands if c['name'] in subtoolchain_names]
cands = [c for c in cands if any(c['name'] == x or c['name'] in x for x in subtoolchain_names)]

for subtoolchain_name in subtoolchain_names:
subtoolchain_version = det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands,
incl_capabilities=incl_capabilities)

# narrow down alternative subtoolchain names to a single one, based on the selected version
if isinstance(subtoolchain_name, tuple):
subtoolchain_name = [cand['name'] for cand in cands if cand['version'] == subtoolchain_version][0]

# add to hierarchy and move to next
if subtoolchain_version is not None and subtoolchain_name not in visited:
tc = {'name': subtoolchain_name, 'version': subtoolchain_version}
Expand Down
19 changes: 10 additions & 9 deletions easybuild/toolchains/fft/intelfftw.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,16 @@ def _set_fftw_variables(self):
bitsuff = '_lp64'
if self.options.get('i8', None):
bitsuff = '_ilp64'
compsuff = '_intel'
if get_software_root('icc') is None:
if get_software_root('PGI'):
compsuff = '_pgi'
elif get_software_root('GCC'):
compsuff = '_gnu'
else:
error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries."
raise EasyBuildError(error_msg)

if get_software_root('icc') or get_software_root('intel-compilers'):
compsuff = '_intel'
elif get_software_root('PGI'):
compsuff = '_pgi'
elif get_software_root('GCC'):
compsuff = '_gnu'
else:
error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries."
raise EasyBuildError(error_msg)

interface_lib = "fftw3xc%s%s" % (compsuff, picsuff)
fftw_libs = [interface_lib]
Expand Down
68 changes: 58 additions & 10 deletions easybuild/toolchains/iimpi.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,32 +32,80 @@
import re

from easybuild.toolchains.iccifort import IccIfort
from easybuild.toolchains.intel_compilers import IntelCompilersToolchain
from easybuild.toolchains.mpi.intelmpi import IntelMPI


class Iimpi(IccIfort, IntelMPI):
class Iimpi(IccIfort, IntelCompilersToolchain, IntelMPI):
"""
Compiler toolchain with Intel compilers (icc/ifort), Intel MPI.
"""
NAME = 'iimpi'
SUBTOOLCHAIN = IccIfort.NAME
# compiler-only subtoolchain can't be determine statically
# since depends on toolchain version (see below),
# so register both here as possible alternatives (which is taken into account elsewhere)
SUBTOOLCHAIN = [(IntelCompilersToolchain.NAME, IccIfort.NAME)]

def __init__(self, *args, **kwargs):
"""Constructor for Iimpi toolchain class."""

super(Iimpi, self).__init__(*args, **kwargs)

# make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
if re.match('^[0-9]', self.version):
# need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01'
# comparing subversions that include letters causes TypeErrors in Python 3
# 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June)
# (good enough for this purpose)
self.iimpi_ver = self.version.replace('a', '.01').replace('b', '.07')
if LooseVersion(self.iimpi_ver) >= LooseVersion('2020.12'):
self.oneapi_gen = True
self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME
self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME
else:
self.oneapi_gen = False
self.SUBTOOLCHAIN = IccIfort.NAME
self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME
else:
self.iimpi_ver = self.version
self.oneapi_gen = False

def is_deprecated(self):
"""Return whether or not this toolchain is deprecated."""
# need to transform a version like '2016a' with something that is safe to compare with '8.0', '2000', '2016.01'
# comparing subversions that include letters causes TypeErrors in Python 3
# 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
version = self.version.replace('a', '.01').replace('b', '.07')

deprecated = False

# make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
if re.match('^[0-9]', version):
iimpi_ver = LooseVersion(version)
if re.match('^[0-9]', str(self.iimpi_ver)):
loosever = LooseVersion(self.iimpi_ver)
# iimpi toolchains older than iimpi/2016.01 are deprecated
# iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet)
if iimpi_ver < LooseVersion('8.0'):
if loosever < LooseVersion('8.0'):
deprecated = True
elif iimpi_ver > LooseVersion('2000') and iimpi_ver < LooseVersion('2016.01'):
elif loosever > LooseVersion('2000') and loosever < LooseVersion('2016.01'):
deprecated = True

return deprecated

def is_dep_in_toolchain_module(self, *args, **kwargs):
"""Check whether a specific software name is listed as a dependency in the module for this toolchain."""
if self.oneapi_gen:
res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs)
else:
res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs)

return res

def _set_compiler_vars(self):
"""Intel compilers-specific adjustments after setting compiler variables."""
if self.oneapi_gen:
IntelCompilersToolchain._set_compiler_vars(self)
else:
IccIfort._set_compiler_vars(self)

def set_variables(self):
"""Intel compilers-specific adjustments after setting compiler variables."""
if self.oneapi_gen:
IntelCompilersToolchain.set_variables(self)
else:
IccIfort.set_variables(self)
16 changes: 11 additions & 5 deletions easybuild/toolchains/linalg/intelmkl.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
:author: Stijn De Weirdt (Ghent University)
:author: Kenneth Hoste (Ghent University)
"""
import os
from distutils.version import LooseVersion

from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC
Expand Down Expand Up @@ -152,13 +153,18 @@ def _set_blas_variables(self):
raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)",
found_version)
else:
self.BLAS_LIB_DIR = ['mkl/lib/intel64']
if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
self.BLAS_LIB_DIR.append('compiler/lib/intel64')
if ver >= LooseVersion('2021'):
basedir = os.path.join('mkl', found_version)
else:
self.BLAS_LIB_DIR.append('lib/intel64')
basedir = 'mkl'

self.BLAS_LIB_DIR = [os.path.join(basedir, 'lib', 'intel64')]
if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
self.BLAS_LIB_DIR.append(os.path.join('compiler', 'lib', 'intel64'))
elif ver < LooseVersion('2021'):
self.BLAS_LIB_DIR.append(os.path.join('lib', 'intel64'))

self.BLAS_INCLUDE_DIR = ['mkl/include']
self.BLAS_INCLUDE_DIR = [os.path.join(basedir, 'include')]

super(IntelMKL, self)._set_blas_variables()

Expand Down
6 changes: 6 additions & 0 deletions test/framework/easyconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -3337,6 +3337,12 @@ def test_det_subtoolchain_version(self):
for subtoolchain_name in subtoolchains[current_tc['name']]]
self.assertEqual(versions, ['4.9.3', ''])

# test det_subtoolchain_version when two alternatives for subtoolchain are specified
current_tc = {'name': 'gompi', 'version': '2018b'}
cands = [{'name': 'GCC', 'version': '7.3.0-2.30'}]
subtc_version = det_subtoolchain_version(current_tc, ('GCCcore', 'GCC'), optional_toolchains, cands)
self.assertEqual(subtc_version, '7.3.0-2.30')

def test_verify_easyconfig_filename(self):
"""Test verify_easyconfig_filename function"""
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
Expand Down