diff --git a/easybuild/framework/easystack.py b/easybuild/framework/easystack.py index 8607734774..0c85602eaf 100644 --- a/easybuild/framework/easystack.py +++ b/easybuild/framework/easystack.py @@ -29,7 +29,7 @@ :author: Pavel Grochal (Inuits) """ -from easybuild.base import fancylogger +from easybuild.base.fancylogger import getLogger from easybuild.tools.build_log import EasyBuildError from easybuild.tools.filetools import read_file from easybuild.tools.module_naming_scheme.utilities import det_full_ec_version @@ -38,15 +38,13 @@ import yaml except ImportError: pass -_log = fancylogger.getLogger('easystack', fname=False) +_log = getLogger('easystack', fname=True) class EasyStack(object): """One class instance per easystack. General options + list of all SoftwareSpecs instances""" def __init__(self): - self.easybuild_version = None - self.robot = False self.software_list = [] def compose_ec_filenames(self): @@ -56,42 +54,109 @@ def compose_ec_filenames(self): full_ec_version = det_full_ec_version({ 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, 'version': sw.version, - 'versionsuffix': sw.versionsuffix, + 'versionsuffix': sw.versionsuffix or '', }) ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) ec_filenames.append(ec_filename) return ec_filenames - # flags applicable to all sw (i.e. robot) - def get_general_options(self): - """Returns general options (flags applicable to all sw (i.e. --robot))""" - general_options = {} - # TODO add support for general_options - # general_options['robot'] = self.robot - # general_options['easybuild_version'] = self.easybuild_version - return general_options + def print_full_commands(self): + """Creates easybuild string to be run via terminal.""" + easystack_log = "Building from easystack:" + _log.info(easystack_log) + print(easystack_log) + print_only_log = "Printing commands only, since a not-fully-supported keyword has been used:\n" + _log.info(print_only_log) + print(print_only_log) + for sw in self.software_list: + full_ec_version = det_full_ec_version({ + 'toolchain': {'name': sw.toolchain_name, 'version': sw.toolchain_version}, + 'version': sw.version, + 'versionsuffix': sw.versionsuffix or '', + }) + ec_filename = '%s-%s.eb' % (sw.name, full_ec_version) + robot_suffix, force_suffix, dry_run_suffix, parallel_suffix = '', '', '', '' + easybuild_version_suffix, from_pr_suffix = '', '' + if sw.robot is True: + robot_suffix = ' --robot' + if sw.force is True: + force_suffix = ' --force' + if sw.dry_run is True: + dry_run_suffix = ' --dry-run' + if sw.parallel: + parallel_suffix = ' --parallel=%s' % sw.parallel + if sw.easybuild_version: + easybuild_version_suffix = ' --easybuild_version=%s' % sw.easybuild_version + if sw.from_pr: + from_pr_suffix = ' --from-pr=%s' % sw.from_pr + full_command = 'eb %s%s%s%s%s%s%s' % (ec_filename, robot_suffix, force_suffix, + dry_run_suffix, parallel_suffix, + easybuild_version_suffix, from_pr_suffix) + full_command_log = "%s; \n" % full_command + _log.info(full_command_log) + print(full_command_log) + + # at least one of include-labels is required on input for SW to install + def process_include_labels(self, provided_labels): + for sw in self.software_list[:]: + for easystack_include_label in sw.include_labels[:]: + # if a match IS NOT FOUND, sw must be deleted + if provided_labels is False or easystack_include_label not in provided_labels: + self.software_list.remove(sw) + + # do input labels match with any of defined 'exclude-labels' ? if so, such sw wont be installed + def process_exclude_labels(self, provided_labels): + for sw in self.software_list[:]: + for easystack_exclude_labels in sw.exclude_labels[:]: + # if a match IS FOUND, sw must be deleted + if provided_labels is False: + continue + elif easystack_exclude_labels in provided_labels: + self.software_list.remove(sw) class SoftwareSpecs(object): """Contains information about every software that should be installed""" - def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name): + def __init__(self, name, version, versionsuffix, toolchain_version, toolchain_name, easybuild_version, + robot, force, dry_run, parallel, from_pr, include_labels, exclude_labels): self.name = name self.version = version self.toolchain_version = toolchain_version self.toolchain_name = toolchain_name self.versionsuffix = versionsuffix + self.easybuild_version = easybuild_version + self.robot = robot + self.force = force + self.dry_run = dry_run + self.parallel = parallel + self.from_pr = from_pr + self.include_labels = include_labels or [] + self.exclude_labels = exclude_labels or [] + self.check_consistency(self.include_labels, self.exclude_labels) + + # general function that checks any potential inconsistencies + def check_consistency(self, include_labels, exclude_labels): + for include_label in include_labels: + if exclude_labels.count(include_label) != 0: + inconsistent_labels_err = 'One or more software specifications contain inconsistent labels. ' + inconsistent_labels_err += 'Make sure there are no cases of one software having the same label ' + inconsistent_labels_err += 'specified both in include_labels and exclude_labels' + raise EasyBuildError(inconsistent_labels_err) class EasyStackParser(object): """Parser for easystack files (in YAML syntax).""" + @only_if_module_is_available('yaml', pkgname='PyYAML') @staticmethod def parse(filepath): """Parses YAML file and assigns obtained values to SW config instances as well as general config instance""" yaml_txt = read_file(filepath) easystack_raw = yaml.safe_load(yaml_txt) easystack = EasyStack() + # should the resulting commands be only printed or should Easybuild continue building? + print_only = False try: software = easystack_raw["software"] @@ -99,37 +164,66 @@ def parse(filepath): wrong_structure_file = "Not a valid EasyStack YAML file: no 'software' key found" raise EasyBuildError(wrong_structure_file) + # trying to assign easybuild_version/robot/force/dry-run/parallel/from_pr on the uppermost level + # if anything changes at any lower level, these will get overwritten + # assign general easystack attributes + easybuild_version = easystack_raw.get('easybuild_version', False) + robot = easystack_raw.get('robot', False) + force = easystack_raw.get('force', False) + dry_run = easystack_raw.get('dry-run', False) + parallel = easystack_raw.get('parallel', False) + from_pr = easystack_raw.get('from-pr', False) + # assign software-specific easystack attributes for name in software: # ensure we have a string value (YAML parser returns type = dict # if levels under the current attribute are present) name = str(name) + + # checking whether software has any labels defined on topmost level + include_labels = [] + exclude_labels = [] + name_lvl_include_labels = [] + name_lvl_exclude_labels = [] + tmp_include = software[name].get('include-labels', False) + tmp_exclude = software[name].get('exclude-labels', False) + if tmp_include: + name_lvl_include_labels.append(tmp_include) + if tmp_exclude: + name_lvl_exclude_labels.append(tmp_exclude) + try: toolchains = software[name]['toolchains'] except KeyError: raise EasyBuildError("Toolchains for software '%s' are not defined in %s", name, filepath) + for toolchain in toolchains: toolchain = str(toolchain) - - if toolchain == 'SYSTEM': - toolchain_name, toolchain_version = 'system', '' + toolchain_parts = toolchain.split('-', 1) + if len(toolchain_parts) == 2: + toolchain_name, toolchain_version = toolchain_parts + elif len(toolchain_parts) == 1: + toolchain_name, toolchain_version = toolchain, '' else: - toolchain_parts = toolchain.split('-', 1) - if len(toolchain_parts) == 2: - toolchain_name, toolchain_version = toolchain_parts - elif len(toolchain_parts) == 1: - toolchain_name, toolchain_version = toolchain, '' - else: - raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", - name, filepath, toolchain_parts) - + raise EasyBuildError("Incorrect toolchain specification for '%s' in %s, too many parts: %s", + name, filepath, toolchain_parts) + + toolchain_lvl_include_labels = name_lvl_include_labels[:] + toolchain_lvl_exclude_labels = name_lvl_exclude_labels[:] + tmp_include = toolchains[toolchain].get('include-labels', False) + tmp_exclude = toolchains[toolchain].get('exclude-labels', False) + if tmp_include: + toolchain_lvl_include_labels.append(tmp_include) + if tmp_exclude: + toolchain_lvl_exclude_labels.append(tmp_exclude) try: - # if version string containts asterisk or labels, raise error (asterisks not supported) versions = toolchains[toolchain]['versions'] except TypeError as err: wrong_structure_err = "An error occurred when interpreting " - wrong_structure_err += "the data for software %s: %s" % (name, err) + wrong_structure_err += "the easystack for software %s: %s" % (name, err) raise EasyBuildError(wrong_structure_err) + + # if version string containts asterisk or labels, raise error (asterisks not supported) if '*' in str(versions): asterisk_err = "EasyStack specifications of '%s' in %s contain asterisk. " asterisk_err += "Wildcard feature is not supported yet." @@ -140,24 +234,48 @@ def parse(filepath): # Example of yaml structure: # ======================================================================== # versions: + # 2.21: # 2.25: + # robot: True + # include-labels: '225' # 2.23: # versionsuffix: '-R-4.0.0' + # parallel: 12 + # 2.26: + # from_pr: 1234 # ======================================================================== if isinstance(versions, dict): for version in versions: + version_lvl_include_labels = toolchain_lvl_include_labels[:] + version_lvl_exclude_labels = toolchain_lvl_exclude_labels[:] + parallel_for_version = parallel + robot_for_version = robot + force_for_version = force + dry_run_for_version = dry_run + from_pr_for_version = from_pr if versions[version] is not None: version_spec = versions[version] - if 'versionsuffix' in version_spec: - versionsuffix = str(version_spec['versionsuffix']) - else: - versionsuffix = '' - if 'exclude-labels' in str(version_spec) or 'include-labels' in str(version_spec): - lab_err = "EasyStack specifications of '%s' in %s " - lab_err += "contain labels. Labels aren't supported yet." - raise EasyBuildError(lab_err, name, filepath) + versionsuffix = version_spec.get('versionsuffix', False) + robot_for_version = version_spec.get('robot', robot) + force_for_version = version_spec.get('force', force) + dry_run_for_version = version_spec.get('dry-run', dry_run) + parallel_for_version = version_spec.get('parallel', parallel) + from_pr_for_version = version_spec.get('from-pr', from_pr) + + # sub-version-level labels handling + tmp_include = version_spec.get('include-labels', False) + if tmp_include: + version_lvl_include_labels.append(tmp_include) + tmp_exclude = version_spec.get('exclude-labels', False) + if tmp_exclude: + version_lvl_exclude_labels.append(tmp_exclude) else: - versionsuffix = '' + versionsuffix = False + + # dont want to overwrite print_only if it's been already set to True + if print_only is False: + if easybuild_version or robot or force or dry_run or parallel or from_pr: + print_only = True specs = { 'name': name, @@ -165,6 +283,14 @@ def parse(filepath): 'toolchain_version': toolchain_version, 'version': version, 'versionsuffix': versionsuffix, + 'easybuild_version': easybuild_version, + 'robot': robot_for_version, + 'force': force_for_version, + 'dry_run': dry_run_for_version, + 'parallel': parallel_for_version, + 'from_pr': from_pr_for_version, + 'include_labels': version_lvl_include_labels, + 'exclude_labels': version_lvl_exclude_labels, } sw = SoftwareSpecs(**specs) @@ -189,7 +315,7 @@ def parse(filepath): elif isinstance(versions, str): versions_list = str(versions).split() - # format read as float (containing one version only)? + # format read as float (containing one version only without :)? # ======================================================================== # versions: # 2.24 @@ -197,25 +323,37 @@ def parse(filepath): elif isinstance(versions, float): versions_list = [str(versions)] - # if no version is a dictionary, versionsuffix isn't specified - versionsuffix = '' + # if no version is a dictionary, neither versionsuffix, robot, + # force, dry_run, parallel, easybuild_version nor from_pr,are specified on this level + versionsuffix = False + easybuild_version = easybuild_version or False + robot = robot or False + force = force or False + dry_run = dry_run or False + parallel = parallel or False + from_pr = from_pr or False + include_labels = toolchain_lvl_include_labels or False + exclude_labels = toolchain_lvl_exclude_labels or False + + # dont want to overwrite print_only once it's been set to True + if print_only is False: + if easybuild_version or robot or force or dry_run or parallel or from_pr: + print_only = True for version in versions_list: sw = SoftwareSpecs( name=name, version=version, versionsuffix=versionsuffix, - toolchain_name=toolchain_name, toolchain_version=toolchain_version) + toolchain_name=toolchain_name, toolchain_version=toolchain_version, + easybuild_version=easybuild_version, robot=robot, force=force, dry_run=dry_run, + parallel=parallel, from_pr=from_pr, + include_labels=include_labels, exclude_labels=exclude_labels + ) # append newly created class instance to the list in instance of EasyStack class easystack.software_list.append(sw) + return easystack, print_only - # assign general easystack attributes - easystack.easybuild_version = easystack_raw.get('easybuild_version', None) - easystack.robot = easystack_raw.get('robot', False) - - return easystack - -@only_if_module_is_available('yaml', pkgname='PyYAML') -def parse_easystack(filepath): +def parse_easystack(filepath, labels): """Parses through easystack file, returns what EC are to be installed together with their options.""" log_msg = "Support for easybuild-ing from multiple easyconfigs based on " log_msg += "information obtained from provided file (easystack) with build specifications." @@ -223,16 +361,19 @@ def parse_easystack(filepath): _log.info("Building from easystack: '%s'" % filepath) # class instance which contains all info about planned build - easystack = EasyStackParser.parse(filepath) + easystack, print_only = EasyStackParser.parse(filepath) + + easystack.process_include_labels(labels or False) + easystack.process_exclude_labels(labels or False) + if easystack.software_list == []: + raise EasyBuildError('No software to build specified in Easystack file. Did you use correct labels?') easyconfig_names = easystack.compose_ec_filenames() - general_options = easystack.get_general_options() + if print_only: + easystack.print_full_commands() - _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: %s" % ', '.join(sorted(easyconfig_names))) - if len(general_options) != 0: - _log.debug("General options for installation are: \n%s" % str(general_options)) - else: - _log.debug("No general options were specified in easystack") + _log.debug("EasyStack parsed. Proceeding to install these Easyconfigs: \n'%s'" % "',\n'".join(easyconfig_names)) + _log.debug("Number of easyconfigs extracted from Easystack: %s" % len(easyconfig_names)) - return easyconfig_names, general_options + return easyconfig_names, print_only diff --git a/easybuild/main.py b/easybuild/main.py index 3c7429ea78..f584e2d143 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -225,12 +225,15 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) - # if easystack is provided with the command, commands with arguments from it will be executed + # check if user is using easystack if options.easystack: - # TODO add general_options (i.e. robot) to build options - orig_paths, general_options = parse_easystack(options.easystack) - if general_options: - raise EasyBuildError("Specifying general configuration options in easystack file is not supported yet.") + # if user provided labels, we need to pass them to parse_easystack() + if options.labels: + orig_paths, print_only = parse_easystack(options.easystack, options.labels) + else: + orig_paths, print_only = parse_easystack(options.easystack, False) + if print_only is True: + clean_exit(logfile, eb_tmpdir, testing, silent=True) # check whether packaging is supported when it's being used if options.package: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 605ed46ab9..abdeb08020 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -595,6 +595,10 @@ def informative_options(self): 'dep-graph': ("Create dependency graph", None, 'store', None, {'metavar': 'depgraph.'}), 'dump-env-script': ("Dump source script to set up build environment based on toolchain/dependencies", None, 'store_true', False), + 'easystack': ("Path to easystack file in YAML format, specifying details of a software stack", + None, 'store', None, {'metavar': "example.yaml"}), + 'labels': ("Easystack feature. Compares labels with those defined in Easystack file.", + 'strlist', 'store', None, {'metavar': "LAB1,LAB2"}), 'last-log': ("Print location to EasyBuild log file of last (failed) session", None, 'store_true', False), 'list-easyblocks': ("Show list of available easyblocks", 'choice', 'store_or_None', 'simple', ['simple', 'detailed']), @@ -619,8 +623,6 @@ def informative_options(self): 'show-full-config': ("Show current EasyBuild configuration (all settings)", None, 'store_true', False), 'show-system-info': ("Show system information relevant to EasyBuild", None, 'store_true', False), 'terse': ("Terse output (machine-readable)", None, 'store_true', False), - 'easystack': ("Path to easystack file in YAML format, specifying details of a software stack", - None, 'store', None), }) self.log.debug("informative_options: descr %s opts %s" % (descr, opts)) diff --git a/test/framework/easystacks/test_easystack_basic.yaml b/test/framework/easystacks/test_easystack_basic.yaml index 2de5dfd129..4272aaf0fc 100644 --- a/test/framework/easystacks/test_easystack_basic.yaml +++ b/test/framework/easystacks/test_easystack_basic.yaml @@ -5,13 +5,9 @@ software: versions: 2.25: 2.26: - foss: - toolchains: - SYSTEM: - versions: [2018a] toy: toolchains: gompi-2018a: versions: 0.0: - versionsuffix: '-test' + versionsuffix: '-test' \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_basic_install.yaml b/test/framework/easystacks/test_easystack_basic_install.yaml new file mode 100644 index 0000000000..38e36e4115 --- /dev/null +++ b/test/framework/easystacks/test_easystack_basic_install.yaml @@ -0,0 +1,9 @@ +software: + SQLite: + toolchains: + foss-2018a: + versions: + 3.8.10.2: + gompi-2018a: + versions: + 3.8.10.2: diff --git a/test/framework/easystacks/test_easystack_labels.yaml b/test/framework/easystacks/test_easystack_labels.yaml index 51a113523f..a1c1fce380 100644 --- a/test/framework/easystacks/test_easystack_labels.yaml +++ b/test/framework/easystacks/test_easystack_labels.yaml @@ -1,7 +1,31 @@ software: - binutils: + HPL: toolchains: - GCCcore-4.9.3: + CrayCCE-5.1.29: + include-labels: 'craycce' versions: - 3.11: - exclude-labels: arch:aarch64 + 2.1: + imkl: + toolchains: + iimpi-2016.01: + versions: + 11.3.1.150: + exclude-labels: 'no_mpi' + iimpi-2019.08: + versions: + 2019.4.243: + exclude-labels: 'no_mpi' + SQLite: + include-labels: sqlite + toolchains: + foss-2018a: + versions: + 3.8.10.2: + gompi-2018a: + exclude-labels: 'no_mpi' + versions: + 3.8.10.2: + GCC-6.4.0-2.28: + versions: + 3.8.10.2: + include-labels: 'sqlitegcc' \ No newline at end of file diff --git a/test/framework/easystacks/test_easystack_print_only.yaml b/test/framework/easystacks/test_easystack_print_only.yaml new file mode 100644 index 0000000000..7a30620f8f --- /dev/null +++ b/test/framework/easystacks/test_easystack_print_only.yaml @@ -0,0 +1,16 @@ +robot: True +dry-run: False +force: True +software: + SQLite: + toolchains: + foss-2018a: + versions: + 3.8.10.2: + parallel: 12 + gompi-2018a: + versions: + 3.8.10.2: + parallel: 6 + dry-run: False + from-pr: '1234xyz' \ No newline at end of file diff --git a/test/framework/options.py b/test/framework/options.py index 36a4a06f95..3c0a972d1c 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -5994,15 +5994,25 @@ def test_easystack_asterisk(self): self.assertErrorRegex(EasyBuildError, expected_err, parse_easystack, toy_easystack) + # tests whether include and exclude labels are read & interpretted correctly def test_easystack_labels(self): """Test for --easystack when yaml easystack contains exclude-labels / include-labels""" easybuild.tools.build_log.EXPERIMENTAL = True topdir = os.path.dirname(os.path.abspath(__file__)) toy_easystack = os.path.join(topdir, 'easystacks', 'test_easystack_labels.yaml') - error_msg = "EasyStack specifications of 'binutils' in .*/test_easystack_labels.yaml contain labels. " - error_msg += "Labels aren't supported yet." - self.assertErrorRegex(EasyBuildError, error_msg, parse_easystack, toy_easystack) + args = ['--easystack', toy_easystack, '--labels', 'sqlite,no_mpi,sqlitegcc', '--debug', + '--experimental', '--dry-run'] + stdout, err = self.eb_main(args, do_build=True, return_error=True) + patterns = [ + r"[\S\s]*DEBUG EasyStack parsed\. Proceeding to install these Easyconfigs: \n[\S\s]*", + r"[\S\s]*'SQLite-3\.8\.10\.2-foss-2018a\.eb',\n'SQLite-3\.8\.10\.2-GCC-6\.4\.0-2\.28\.eb'[\S\s]*", + r"[\S\s]*Building from easystack:[\S\s]*", + r"[\S\s]*Number of easyconfigs extracted from Easystack: 2[\S\s]*", + ] + for pattern in patterns: + regex = re.compile(pattern) + self.assertTrue(regex.match(stdout) is not None) def suite(): diff --git a/test/framework/sandbox/sources/s/SQLite/sqlite-autoconf-3081002.tar.gz b/test/framework/sandbox/sources/s/SQLite/sqlite-autoconf-3081002.tar.gz new file mode 100644 index 0000000000..841053dd60 Binary files /dev/null and b/test/framework/sandbox/sources/s/SQLite/sqlite-autoconf-3081002.tar.gz differ