Merge pull request #2852 from boegel/exts_templates

correctly resolve template values used for extensions

Author: akesandgren

easybuild/framework/easyblock.py

@@ -58,6 +58,7 @@
 from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR
 from easybuild.framework.easyconfig.easyconfig import ITERATE_OPTIONS, EasyConfig, ActiveMNS, get_easyblock_class
 from easybuild.framework.easyconfig.easyconfig import get_module_path, letter_dir_for, resolve_template
+from easybuild.framework.easyconfig.templates import template_constant_dict
 from easybuild.framework.easyconfig.format.format import INDENT_4SPACES
 from easybuild.framework.easyconfig.parser import fetch_parameters_from_easyconfig
 from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH
@@ -482,8 +483,6 @@ def fetch_extension_sources(self, skip_checksums=False):
                     # since it may use template values like %(name)s & %(version)s
                     ext_options = copy.deepcopy(self.cfg.get_ref('exts_default_options'))
 
-                    def_src_tmpl = "%(name)s-%(version)s.tar.gz"
-
                     if len(ext) == 3:
                         if isinstance(ext_options, dict):
                             ext_options.update(ext[2])
@@ -498,17 +497,24 @@ def fetch_extension_sources(self, skip_checksums=False):
                         'options': ext_options,
                     }
 
+                    # construct dictionary with template values;
+                    # inherited from parent, except for name/version templates which are specific to this extension
+                    template_values = copy.deepcopy(self.cfg.template_values)
+                    template_values.update(template_constant_dict(ext_src))
+
+                    # resolve templates in extension options
+                    ext_options = resolve_template(ext_options, template_values)
+
                     checksums = ext_options.get('checksums', [])
 
-                    if ext_options.get('source_tmpl', None):
-                        fn = resolve_template(ext_options['source_tmpl'], ext_src)
-                    else:
-                        fn = resolve_template(def_src_tmpl, ext_src)
+                    # use default template for name of source file if none is specified
+                    default_source_tmpl = resolve_template('%(name)s-%(version)s.tar.gz', template_values)
+                    fn = ext_options.get('source_tmpl', default_source_tmpl)
 
                     if ext_options.get('nosource', None):
                         exts_sources.append(ext_src)
                     else:
-                        source_urls = [resolve_template(url, ext_src) for url in ext_options.get('source_urls', [])]
+                        source_urls = ext_options.get('source_urls', [])
                         force_download = build_option('force_download') in [FORCE_DOWNLOAD_ALL, FORCE_DOWNLOAD_SOURCES]
                         src_fn = self.obtain_file(fn, extension=True, urls=source_urls, force_download=force_download)
 

easybuild/framework/extension.py

@@ -36,8 +36,9 @@
 import copy
 import os
 
+from easybuild.framework.easyconfig.easyconfig import resolve_template
+from easybuild.framework.easyconfig.templates import template_constant_dict
 from easybuild.tools.build_log import EasyBuildError
-from easybuild.tools.config import build_option, build_path
 from easybuild.tools.filetools import change_dir
 from easybuild.tools.run import run_cmd
 
@@ -60,17 +61,22 @@ def __init__(self, mself, ext, extra_params=None):
         self.ext = copy.deepcopy(ext)
         self.dry_run = self.master.dry_run
 
-        if not 'name' in self.ext:
+        if 'name' not in self.ext:
             raise EasyBuildError("'name' is missing in supplied class instance 'ext'.")
 
+        name, version = self.ext['name'], self.ext.get('version', None)
+
         # parent sanity check paths/commands are not relevant for extension
         self.cfg['sanity_check_commands'] = []
         self.cfg['sanity_check_paths'] = []
 
+        # construct dict with template values that can be used
+        self.cfg.template_values.update(template_constant_dict({'name': name, 'version': version}))
+
         # list of source/patch files: we use an empty list as default value like in EasyBlock
-        self.src = self.ext.get('src', [])
-        self.patches = self.ext.get('patches', [])
-        self.options = copy.deepcopy(self.ext.get('options', {}))
+        self.src = resolve_template(self.ext.get('src', []), self.cfg.template_values)
+        self.patches = resolve_template(self.ext.get('patches', []), self.cfg.template_values)
+        self.options = resolve_template(copy.deepcopy(self.ext.get('options', {})), self.cfg.template_values)
 
         if extra_params:
             self.cfg.extend_params(extra_params, overwrite=False)
@@ -81,12 +87,12 @@ def __init__(self, mself, ext, extra_params=None):
         # this allows to specify custom easyconfig parameters on a per-extension basis
         for key in self.options:
             if key in self.cfg:
-                self.cfg[key] = self.options[key]
+                self.cfg[key] = resolve_template(self.options[key], self.cfg.template_values)
                 self.log.debug("Customising known easyconfig parameter '%s' for extension %s/%s: %s",
-                               key, self.ext['name'], self.ext['version'], self.cfg[key])
+                               key, name, version, self.cfg[key])
             else:
                 self.log.debug("Skipping unknown custom easyconfig parameter '%s' for extension %s/%s: %s",
-                               key, self.ext['name'], self.ext['version'], self.options[key])
+                               key, name, version, self.options[key])
 
         self.sanity_check_fail_msgs = []
 

test/framework/easyconfig.py

@@ -390,6 +390,73 @@ def test_exts_list(self):
         regex = re.compile('EBEXTSLISTPI.*ext1-1.0,ext2-2.0')
         self.assertTrue(regex.search(modtxt), "Pattern '%s' found in: %s" % (regex.pattern, modtxt))
 
+    def test_extensions_templates(self):
+        """Test whether templates used in exts_list are resolved properly."""
+
+        # put dummy source file in place to avoid download fail
+        toy_tar_gz = os.path.join(self.test_sourcepath, 'toy', 'toy-0.0.tar.gz')
+        copy_file(toy_tar_gz, os.path.join(self.test_prefix, 'toy-0.0-py3-test.tar.gz'))
+        toy_patch_fn = 'toy-0.0_fix-silly-typo-in-printf-statement.patch'
+        toy_patch = os.path.join(self.test_sourcepath, 'toy', toy_patch_fn)
+        copy_file(toy_patch, self.test_prefix)
+
+        os.environ['EASYBUILD_SOURCEPATH'] = self.test_prefix
+        init_config(build_options={'silent': True})
+
+        self.contents = '\n'.join([
+            'easyblock = "ConfigureMake"',
+            'name = "pi"',
+            'version = "3.14"',
+            'versionsuffix = "-test"',
+            'homepage = "http://example.com"',
+            'description = "test easyconfig"',
+            'toolchain = {"name": "dummy", "version": ""}',
+            'dependencies = [("Python", "3.6.6")]',
+            'exts_defaultclass = "EB_Toy"',
+            # bogus, but useful to check whether this get resolved
+            'exts_default_options = {"source_urls": [PYPI_SOURCE]}',
+            'exts_list = [',
+            '   ("toy", "0.0", {',
+            # %(name)s and %(version_major_minor)s should be resolved using name/version of extension (not parent)
+            # %(pymajver)s should get resolved because Python is listed as a (runtime) dep
+            # %(versionsuffix)s should get resolved with value of parent
+            '       "source_tmpl": "%(name)s-%(version_major_minor)s-py%(pymajver)s%(versionsuffix)s.tar.gz",',
+            '       "patches": ["%(name)s-%(version)s_fix-silly-typo-in-printf-statement.patch"],',
+            # use hacky prebuildopts that is picked up by 'EB_Toy' easyblock, to check whether templates are resolved
+            '       "prebuildopts": "gcc -O2 %(name)s.c -o toy-%(version)s && mv toy-%(version)s toy #",',
+            '   }),',
+            ']',
+        ])
+        self.prep()
+        ec = EasyConfig(self.eb_file)
+        eb = EasyBlock(ec)
+        eb.fetch_step()
+
+        # run extensions step to install 'toy' extension
+        eb.extensions_step()
+
+        # check whether template values were resolved correctly in Extension instances that were created/used
+        toy_ext = eb.ext_instances[0]
+        self.assertEqual(os.path.basename(toy_ext.src), 'toy-0.0-py3-test.tar.gz')
+        self.assertEqual(toy_ext.patches, [os.path.join(self.test_prefix, toy_patch_fn)])
+        expected = {
+            'patches': ['toy-0.0_fix-silly-typo-in-printf-statement.patch'],
+            'prebuildopts': 'gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy #',
+            'source_tmpl': 'toy-0.0-py3-test.tar.gz',
+            'source_urls': ['https://pypi.python.org/packages/source/t/toy'],
+        }
+        self.assertEqual(toy_ext.options, expected)
+
+        # also .cfg of Extension instance was updated correctly
+        self.assertEqual(toy_ext.cfg['source_urls'], ['https://pypi.python.org/packages/source/t/toy'])
+        self.assertEqual(toy_ext.cfg['patches'], [toy_patch_fn])
+        self.assertEqual(toy_ext.cfg['prebuildopts'], "gcc -O2 toy.c -o toy-0.0 && mv toy-0.0 toy #")
+
+        # check whether files expected to be installed for 'toy' extension are in place
+        pi_installdir = os.path.join(self.test_installpath, 'software', 'pi', '3.14-test')
+        self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'bin', 'toy')))
+        self.assertTrue(os.path.exists(os.path.join(pi_installdir, 'lib', 'libtoy.a')))
+
     def test_suggestions(self):
         """ If a typo is present, suggestions should be provided (if possible) """
         self.contents = '\n'.join([
@@ -2163,6 +2230,31 @@ def test_template_constant_dict(self):
 
         self.assertEqual(res, expected)
 
+        # also check result of template_constant_dict when dict representing extension is passed
+        ext_dict = {
+            'name': 'foo',
+            'version': '1.2.3',
+            'options': {
+                'source_urls': ['https://example.com'],
+                'source_tmpl': '%(name)s-%(version)s.tar.gz',
+            },
+        }
+        res = template_constant_dict(ext_dict)
+
+        self.assertTrue('arch' in res)
+        arch = res.pop('arch')
+        self.assertTrue(arch_regex.match(arch), "'%s' matches with pattern '%s'" % (arch, arch_regex.pattern))
+
+        expected = {
+            'name': 'foo',
+            'nameletter': 'f',
+            'version': '1.2.3',
+            'version_major': '1',
+            'version_major_minor': '1.2',
+            'version_minor': '2'
+        }
+        self.assertEqual(res, expected)
+
     def test_parse_deps_templates(self):
         """Test whether handling of templates defined by dependencies is done correctly."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')