From 07f3804bc40080094dcc201c87c9ba0b45893a1a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 8 May 2020 14:39:56 +0200 Subject: [PATCH 1/9] Copy tweaked easyconfigs to specified path --- easybuild/main.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 94ed2ea301..9d8b1b4639 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -320,7 +320,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) - if options.copy_ec or options.fix_deprecated_easyconfigs or options.show_ec: + if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: if len(determined_paths) == 1: @@ -420,6 +420,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] + if options.copy_ec and tweaked_ecs_paths: + tweaked_ecs_in_ordered_ecs = [ec.path for ec in ordered_ecs if + any(tweaked_ecs_path in ec.path for tweaked_ecs_path in tweaked_ecs_paths)] + if tweaked_ecs_in_ordered_ecs: + copy_files(determined_paths, target_path) + print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) + # creating/updating PRs if pr_options: if options.new_pr: From e56bd966d0e686dbda37262c2d5eda1648885c52 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 8 May 2020 16:08:25 +0200 Subject: [PATCH 2/9] Allow dumping of tweaked easyconfigs when using --try-* --- easybuild/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/easybuild/main.py b/easybuild/main.py index 9d8b1b4639..41bedd3e77 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -421,11 +421,13 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): ordered_ecs = [] if options.copy_ec and tweaked_ecs_paths: - tweaked_ecs_in_ordered_ecs = [ec.path for ec in ordered_ecs if - any(tweaked_ecs_path in ec.path for tweaked_ecs_path in tweaked_ecs_paths)] - if tweaked_ecs_in_ordered_ecs: - copy_files(determined_paths, target_path) - print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) + all_specs = [spec['spec'] for spec in + resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)] + tweaked_ecs_in_all_ecs = [path for path in all_specs if + any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)] + if tweaked_ecs_in_all_ecs: + copy_files(tweaked_ecs_in_all_ecs, target_path) + print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) # creating/updating PRs if pr_options: From 485be8a6ebb15fc0a70708fd51f6fe10cbddb02f Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 8 May 2020 16:11:11 +0200 Subject: [PATCH 3/9] Appease hound --- easybuild/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/main.py b/easybuild/main.py index 41bedd3e77..3ba81a83d0 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -424,7 +424,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): all_specs = [spec['spec'] for spec in resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)] tweaked_ecs_in_all_ecs = [path for path in all_specs if - any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)] + any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)] if tweaked_ecs_in_all_ecs: copy_files(tweaked_ecs_in_all_ecs, target_path) print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) From 92f9f6a42ef3b6acc8dd5b0bcfc86f9980eae87e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 12 May 2020 11:53:02 +0200 Subject: [PATCH 4/9] Add test --- test/framework/options.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index e4b878d4bb..3741d93b29 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1946,6 +1946,35 @@ def test_try(self): allargs = args + ['--software-version=1.2.3', '--toolchain=gompi,2018a'] self.assertErrorRegex(EasyBuildError, "version .* not available", self.eb_main, allargs, raise_error=True) + def test_try_with_copy(self): + """Test whether --try options are taken into account.""" + ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') + tweaked_toy_ec = os.path.join(self.test_buildpath, 'toy-0.0-tweaked.eb') + copy_file(os.path.join(ecs_path, 't', 'toy', 'toy-0.0.eb'), tweaked_toy_ec) + f = open(tweaked_toy_ec, 'a') + f.write("easyblock = 'ConfigureMake'") + f.close() + + args = [ + tweaked_toy_ec, + '--sourcepath=%s' % self.test_sourcepath, + '--buildpath=%s' % self.test_buildpath, + '--installpath=%s' % self.test_installpath, + '--dry-run', + '--robot=%s' % ecs_path, + '--copy-ec', + ] + copied_ec = os.path.join(self.test_buildpath, 'my_eb.eb') + self.eb_main(args + [copied_ec], verbose=True, raise_error=True) + self.assertTrue(os.path.exists(copied_ec)) + + tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') + self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], + verbose=True, raise_error=True) + self.assertTrue( + os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) + ) + def test_software_version_ordering(self): """Test whether software versions are correctly ordered when using --software.""" ecs_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs') From 183ee19de706e3188269798c7dd2026e32f78a81 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 12 May 2020 11:55:06 +0200 Subject: [PATCH 5/9] Appease hound --- test/framework/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/framework/options.py b/test/framework/options.py index 3741d93b29..d5727dbda6 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1970,7 +1970,7 @@ def test_try_with_copy(self): tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], - verbose=True, raise_error=True) + verbose=True, raise_error=True) self.assertTrue( os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) ) From 839a7f51d9a9a433c188f5c4aaccb4ec9ccf2fff Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 12 May 2020 13:19:07 +0200 Subject: [PATCH 6/9] Add mocking --- test/framework/options.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/framework/options.py b/test/framework/options.py index d5727dbda6..731dc8c214 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1964,13 +1964,29 @@ def test_try_with_copy(self): '--robot=%s' % ecs_path, '--copy-ec', ] + self.mock_stdout(True) + self.mock_stderr(True) copied_ec = os.path.join(self.test_buildpath, 'my_eb.eb') self.eb_main(args + [copied_ec], verbose=True, raise_error=True) + outtxt = self.get_stdout() + errtxt = self.get_stderr() + self.assertTrue(r'toy-0.0-tweaked.eb copied to ' + copied_ec in outtxt) + self.assertFalse(errtxt) + self.mock_stdout(False) + self.mock_stderr(False) self.assertTrue(os.path.exists(copied_ec)) + self.mock_stdout(True) + self.mock_stderr(True) tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], verbose=True, raise_error=True) + outtxt = self.get_stdout() + errtxt = self.get_stderr() + self.assertTrue(r'1 file(s) copied to ' + tweaked_ecs_dir in outtxt) + self.assertFalse(errtxt) + self.mock_stdout(False) + self.mock_stderr(False) self.assertTrue( os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) ) From 17003cc1ea40e0fc58bbe2e93d606b6618a59401 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 12 May 2020 13:28:23 +0200 Subject: [PATCH 7/9] Don't mock stderr due to #3231 --- test/framework/options.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index 731dc8c214..e3225b9f0f 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1965,28 +1965,28 @@ def test_try_with_copy(self): '--copy-ec', ] self.mock_stdout(True) - self.mock_stderr(True) + # self.mock_stderr(True) copied_ec = os.path.join(self.test_buildpath, 'my_eb.eb') self.eb_main(args + [copied_ec], verbose=True, raise_error=True) outtxt = self.get_stdout() - errtxt = self.get_stderr() + # errtxt = self.get_stderr() self.assertTrue(r'toy-0.0-tweaked.eb copied to ' + copied_ec in outtxt) - self.assertFalse(errtxt) + # self.assertFalse(errtxt) self.mock_stdout(False) - self.mock_stderr(False) + # self.mock_stderr(False) self.assertTrue(os.path.exists(copied_ec)) self.mock_stdout(True) - self.mock_stderr(True) + # self.mock_stderr(True) tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], verbose=True, raise_error=True) outtxt = self.get_stdout() - errtxt = self.get_stderr() + # errtxt = self.get_stderr() self.assertTrue(r'1 file(s) copied to ' + tweaked_ecs_dir in outtxt) - self.assertFalse(errtxt) + # self.assertFalse(errtxt) self.mock_stdout(False) - self.mock_stderr(False) + # self.mock_stderr(False) self.assertTrue( os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) ) From 35645548a28a1e0d3b6cc7d3f1abb9b3de13ab78 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 12 May 2020 16:54:21 +0200 Subject: [PATCH 8/9] Clean the easyconfigs before copying them. --- easybuild/main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/main.py b/easybuild/main.py index 3ba81a83d0..f15c648b73 100644 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -47,6 +47,7 @@ from easybuild.framework.easyblock import build_and_install_one, inject_checksums from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR +from easybuild.framework.easyconfig.easyconfig import clean_up_easyconfigs from easybuild.framework.easyconfig.easyconfig import fix_deprecated_easyconfigs, verify_easyconfig_filename from easybuild.framework.easyconfig.style import cmdline_easyconfigs_style_check from easybuild.framework.easyconfig.tools import categorize_files_by_type, dep_graph @@ -426,6 +427,8 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): tweaked_ecs_in_all_ecs = [path for path in all_specs if any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)] if tweaked_ecs_in_all_ecs: + # Clean them, then copy them + clean_up_easyconfigs(tweaked_ecs_in_all_ecs) copy_files(tweaked_ecs_in_all_ecs, target_path) print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) From f149b98957c996e85cb0021e310b7d0e43b03707 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 19 May 2020 09:56:58 +0200 Subject: [PATCH 9/9] Also check stderr is empty --- test/framework/options.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/framework/options.py b/test/framework/options.py index e3225b9f0f..731dc8c214 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -1965,28 +1965,28 @@ def test_try_with_copy(self): '--copy-ec', ] self.mock_stdout(True) - # self.mock_stderr(True) + self.mock_stderr(True) copied_ec = os.path.join(self.test_buildpath, 'my_eb.eb') self.eb_main(args + [copied_ec], verbose=True, raise_error=True) outtxt = self.get_stdout() - # errtxt = self.get_stderr() + errtxt = self.get_stderr() self.assertTrue(r'toy-0.0-tweaked.eb copied to ' + copied_ec in outtxt) - # self.assertFalse(errtxt) + self.assertFalse(errtxt) self.mock_stdout(False) - # self.mock_stderr(False) + self.mock_stderr(False) self.assertTrue(os.path.exists(copied_ec)) self.mock_stdout(True) - # self.mock_stderr(True) + self.mock_stderr(True) tweaked_ecs_dir = os.path.join(self.test_buildpath, 'my_tweaked_ecs') self.eb_main(args + ['--try-software=foo,1.2.3', '--try-toolchain=gompi,2018a', tweaked_ecs_dir], verbose=True, raise_error=True) outtxt = self.get_stdout() - # errtxt = self.get_stderr() + errtxt = self.get_stderr() self.assertTrue(r'1 file(s) copied to ' + tweaked_ecs_dir in outtxt) - # self.assertFalse(errtxt) + self.assertFalse(errtxt) self.mock_stdout(False) - # self.mock_stderr(False) + self.mock_stderr(False) self.assertTrue( os.path.exists(os.path.join(self.test_buildpath, tweaked_ecs_dir, 'foo-1.2.3-GCC-6.4.0-2.28.eb')) )