Merge pull request #3665 from boegel/oneapi

add support for using oneAPI versions of 'intel' toolchain components

Author: migueldiascosta

easybuild/framework/easyconfig/easyconfig.py

@@ -220,7 +220,7 @@ def cache_aware_func(toolchain, incl_capabilities=False):
     return cache_aware_func
 
 
-def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands, incl_capabilities=False):
+def det_subtoolchain_version(current_tc, subtoolchain_names, optional_toolchains, cands, incl_capabilities=False):
     """
     Returns unique version for subtoolchain, in tc dict.
     If there is no unique version:
@@ -229,30 +229,46 @@ def det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains,
       optional toolchains or system toolchain without add_system_to_minimal_toolchains.
     * in all other cases, raises an exception.
     """
-    uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])
     # init with "skipped"
     subtoolchain_version = None
 
-    # system toolchain: bottom of the hierarchy
-    if is_system_toolchain(subtoolchain_name):
-        add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains')
-        if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'):
-            depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains"
-            _log.deprecated(depr_msg, '5.0')
-            add_system_to_minimal_toolchains = True
-
-        if add_system_to_minimal_toolchains and not incl_capabilities:
-            subtoolchain_version = ''
-    elif len(uniq_subtc_versions) == 1:
-        subtoolchain_version = list(uniq_subtc_versions)[0]
-    elif len(uniq_subtc_versions) == 0:
-        if subtoolchain_name not in optional_toolchains:
+    # ensure we always have a tuple of alternative subtoolchain names, which makes things easier below
+    if isinstance(subtoolchain_names, string_type):
+        subtoolchain_names = (subtoolchain_names,)
+
+    system_subtoolchain = False
+
+    for subtoolchain_name in subtoolchain_names:
+
+        uniq_subtc_versions = set([subtc['version'] for subtc in cands if subtc['name'] == subtoolchain_name])
+
+        # system toolchain: bottom of the hierarchy
+        if is_system_toolchain(subtoolchain_name):
+            add_system_to_minimal_toolchains = build_option('add_system_to_minimal_toolchains')
+            if not add_system_to_minimal_toolchains and build_option('add_dummy_to_minimal_toolchains'):
+                depr_msg = "Use --add-system-to-minimal-toolchains instead of --add-dummy-to-minimal-toolchains"
+                _log.deprecated(depr_msg, '5.0')
+                add_system_to_minimal_toolchains = True
+
+            system_subtoolchain = True
+
+            if add_system_to_minimal_toolchains and not incl_capabilities:
+                subtoolchain_version = ''
+        elif len(uniq_subtc_versions) == 1:
+            subtoolchain_version = list(uniq_subtc_versions)[0]
+        elif len(uniq_subtc_versions) > 1:
+            raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
+                                 subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions)))
+
+        if subtoolchain_version is not None:
+            break
+
+    if not system_subtoolchain and subtoolchain_version is None:
+        if not all(n in optional_toolchains for n in subtoolchain_names):
+            subtoolchain_names = ' or '.join(subtoolchain_names)
             # raise error if the subtoolchain considered now is not optional
             raise EasyBuildError("No version found for subtoolchain %s in dependencies of %s",
-                                 subtoolchain_name, current_tc['name'])
-    else:
-        raise EasyBuildError("Multiple versions of %s found in dependencies of toolchain %s: %s",
-                             subtoolchain_name, current_tc['name'], ', '.join(sorted(uniq_subtc_versions)))
+                                 subtoolchain_names, current_tc['name'])
 
     return subtoolchain_version
 
@@ -356,11 +372,16 @@ def get_toolchain_hierarchy(parent_toolchain, incl_capabilities=False):
                     cands.append({'name': dep, 'version': current_tc_version})
 
         # only retain candidates that match subtoolchain names
-        cands = [c for c in cands if c['name'] in subtoolchain_names]
+        cands = [c for c in cands if any(c['name'] == x or c['name'] in x for x in subtoolchain_names)]
 
         for subtoolchain_name in subtoolchain_names:
             subtoolchain_version = det_subtoolchain_version(current_tc, subtoolchain_name, optional_toolchains, cands,
                                                             incl_capabilities=incl_capabilities)
+
+            # narrow down alternative subtoolchain names to a single one, based on the selected version
+            if isinstance(subtoolchain_name, tuple):
+                subtoolchain_name = [cand['name'] for cand in cands if cand['version'] == subtoolchain_version][0]
+
             # add to hierarchy and move to next
             if subtoolchain_version is not None and subtoolchain_name not in visited:
                 tc = {'name': subtoolchain_name, 'version': subtoolchain_version}

easybuild/toolchains/fft/intelfftw.py

@@ -57,15 +57,16 @@ def _set_fftw_variables(self):
         bitsuff = '_lp64'
         if self.options.get('i8', None):
             bitsuff = '_ilp64'
-        compsuff = '_intel'
-        if get_software_root('icc') is None:
-            if get_software_root('PGI'):
-                compsuff = '_pgi'
-            elif get_software_root('GCC'):
-                compsuff = '_gnu'
-            else:
-                error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries."
-                raise EasyBuildError(error_msg)
+
+        if get_software_root('icc') or get_software_root('intel-compilers'):
+            compsuff = '_intel'
+        elif get_software_root('PGI'):
+            compsuff = '_pgi'
+        elif get_software_root('GCC'):
+            compsuff = '_gnu'
+        else:
+            error_msg = "Not using Intel compilers, PGI nor GCC, don't know compiler suffix for FFTW libraries."
+            raise EasyBuildError(error_msg)
 
         interface_lib = "fftw3xc%s%s" % (compsuff, picsuff)
         fftw_libs = [interface_lib]

easybuild/toolchains/iimpi.py

@@ -32,32 +32,80 @@
 import re
 
 from easybuild.toolchains.iccifort import IccIfort
+from easybuild.toolchains.intel_compilers import IntelCompilersToolchain
 from easybuild.toolchains.mpi.intelmpi import IntelMPI
 
 
-class Iimpi(IccIfort, IntelMPI):
+class Iimpi(IccIfort, IntelCompilersToolchain, IntelMPI):
     """
     Compiler toolchain with Intel compilers (icc/ifort), Intel MPI.
     """
     NAME = 'iimpi'
-    SUBTOOLCHAIN = IccIfort.NAME
+    # compiler-only subtoolchain can't be determine statically
+    # since depends on toolchain version (see below),
+    # so register both here as possible alternatives (which is taken into account elsewhere)
+    SUBTOOLCHAIN = [(IntelCompilersToolchain.NAME, IccIfort.NAME)]
+
+    def __init__(self, *args, **kwargs):
+        """Constructor for Iimpi toolchain class."""
+
+        super(Iimpi, self).__init__(*args, **kwargs)
+
+        # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
+        if re.match('^[0-9]', self.version):
+            # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2016.01'
+            # comparing subversions that include letters causes TypeErrors in Python 3
+            # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June)
+            # (good enough for this purpose)
+            self.iimpi_ver = self.version.replace('a', '.01').replace('b', '.07')
+            if LooseVersion(self.iimpi_ver) >= LooseVersion('2020.12'):
+                self.oneapi_gen = True
+                self.SUBTOOLCHAIN = IntelCompilersToolchain.NAME
+                self.COMPILER_MODULE_NAME = IntelCompilersToolchain.COMPILER_MODULE_NAME
+            else:
+                self.oneapi_gen = False
+                self.SUBTOOLCHAIN = IccIfort.NAME
+                self.COMPILER_MODULE_NAME = IccIfort.COMPILER_MODULE_NAME
+        else:
+            self.iimpi_ver = self.version
+            self.oneapi_gen = False
 
     def is_deprecated(self):
         """Return whether or not this toolchain is deprecated."""
-        # need to transform a version like '2016a' with something that is safe to compare with '8.0', '2000', '2016.01'
-        # comparing subversions that include letters causes TypeErrors in Python 3
-        # 'a' is assumed to be equivalent with '.01' (January), and 'b' with '.07' (June) (good enough for this purpose)
-        version = self.version.replace('a', '.01').replace('b', '.07')
 
         deprecated = False
+
         # make sure a non-symbolic version (e.g., 'system') is used before making comparisons using LooseVersion
-        if re.match('^[0-9]', version):
-            iimpi_ver = LooseVersion(version)
+        if re.match('^[0-9]', str(self.iimpi_ver)):
+            loosever = LooseVersion(self.iimpi_ver)
             # iimpi toolchains older than iimpi/2016.01 are deprecated
             # iimpi 8.1.5 is an exception, since it used in intel/2016a (which is not deprecated yet)
-            if iimpi_ver < LooseVersion('8.0'):
+            if loosever < LooseVersion('8.0'):
                 deprecated = True
-            elif iimpi_ver > LooseVersion('2000') and iimpi_ver < LooseVersion('2016.01'):
+            elif loosever > LooseVersion('2000') and loosever < LooseVersion('2016.01'):
                 deprecated = True
 
         return deprecated
+
+    def is_dep_in_toolchain_module(self, *args, **kwargs):
+        """Check whether a specific software name is listed as a dependency in the module for this toolchain."""
+        if self.oneapi_gen:
+            res = IntelCompilersToolchain.is_dep_in_toolchain_module(self, *args, **kwargs)
+        else:
+            res = IccIfort.is_dep_in_toolchain_module(self, *args, **kwargs)
+
+        return res
+
+    def _set_compiler_vars(self):
+        """Intel compilers-specific adjustments after setting compiler variables."""
+        if self.oneapi_gen:
+            IntelCompilersToolchain._set_compiler_vars(self)
+        else:
+            IccIfort._set_compiler_vars(self)
+
+    def set_variables(self):
+        """Intel compilers-specific adjustments after setting compiler variables."""
+        if self.oneapi_gen:
+            IntelCompilersToolchain.set_variables(self)
+        else:
+            IccIfort.set_variables(self)

easybuild/toolchains/linalg/intelmkl.py

@@ -28,6 +28,7 @@
 :author: Stijn De Weirdt (Ghent University)
 :author: Kenneth Hoste (Ghent University)
 """
+import os
 from distutils.version import LooseVersion
 
 from easybuild.toolchains.compiler.gcc import TC_CONSTANT_GCC
@@ -152,13 +153,18 @@ def _set_blas_variables(self):
                 raise EasyBuildError("_set_blas_variables: 32-bit libraries not supported yet for IMKL v%s (> v10.3)",
                                      found_version)
             else:
-                self.BLAS_LIB_DIR = ['mkl/lib/intel64']
-                if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
-                    self.BLAS_LIB_DIR.append('compiler/lib/intel64')
+                if ver >= LooseVersion('2021'):
+                    basedir = os.path.join('mkl', found_version)
                 else:
-                    self.BLAS_LIB_DIR.append('lib/intel64')
+                    basedir = 'mkl'
+
+                self.BLAS_LIB_DIR = [os.path.join(basedir, 'lib', 'intel64')]
+                if ver >= LooseVersion('10.3.4') and ver < LooseVersion('11.1'):
+                    self.BLAS_LIB_DIR.append(os.path.join('compiler', 'lib', 'intel64'))
+                elif ver < LooseVersion('2021'):
+                    self.BLAS_LIB_DIR.append(os.path.join('lib', 'intel64'))
 
-            self.BLAS_INCLUDE_DIR = ['mkl/include']
+            self.BLAS_INCLUDE_DIR = [os.path.join(basedir, 'include')]
 
         super(IntelMKL, self)._set_blas_variables()
 

test/framework/easyconfig.py

@@ -3380,6 +3380,12 @@ def test_det_subtoolchain_version(self):
                     for subtoolchain_name in subtoolchains[current_tc['name']]]
         self.assertEqual(versions, ['4.9.3', ''])
 
+        # test det_subtoolchain_version when two alternatives for subtoolchain are specified
+        current_tc = {'name': 'gompi', 'version': '2018b'}
+        cands = [{'name': 'GCC', 'version': '7.3.0-2.30'}]
+        subtc_version = det_subtoolchain_version(current_tc, ('GCCcore', 'GCC'), optional_toolchains, cands)
+        self.assertEqual(subtc_version, '7.3.0-2.30')
+
     def test_verify_easyconfig_filename(self):
         """Test verify_easyconfig_filename function"""
         test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')