Merge pull request #26 from boegel/checksums_external

generate test easyconfig without checksums to check whether checksums.json is picked up in test_checksum_step

Author: mboisson

test/framework/easyblock.py

@@ -2347,11 +2347,30 @@ def test_checksum_step(self):
         self.mock_stderr(False)
         self.disallow_deprecated_behaviour()
 
-        # test without checksums, it should work since they are in checksums.json
-        toy_ec_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-testjson.eb')
-        toy_checksums_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'checksums-test.json')
+        # create test easyconfig from which checksums have been stripped
+        test_ec = os.path.join(self.test_prefix, 'test.eb')
+        ectxt = read_file(toy_ec)
+        regex = re.compile(r"'?checksums'?\s*[=:]\s*\[[^]]+\].*", re.M)
+        ectxt = regex.sub('', ectxt)
+        write_file(test_ec, ectxt)
+
+        ec_json = process_easyconfig(test_ec)[0]
+
+        # make sure that test easyconfig file indeed doesn't contain any checksums (either top-level or for extensions)
+        self.assertEqual(ec_json['ec']['checksums'], [])
+        for ext in ec_json['ec']['exts_list']:
+            if isinstance(ext, string_type):
+                continue
+            elif isinstance(ext, tuple):
+                self.assertEqual(ext[2].get('checksums', []), [])
+            else:
+                self.assertTrue(False, "Incorrect extension type: %s" % type(ext))
+
+        # put checksums.json in place next to easyconfig file being used for the tests
+        toy_checksums_json = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'checksums.json')
         copy_file(toy_checksums_json, os.path.join(self.test_prefix, 'checksums.json'))
-        ec_json = process_easyconfig(toy_ec_json)[0]
+
+        # test without checksums, it should work since they are in checksums.json
         eb_json = get_easyblock_instance(ec_json)
         eb_json.fetch_sources()
         eb_json.checksum_step()
@@ -2365,7 +2384,7 @@ def test_checksum_step(self):
         eb.fetch_sources()
         eb.checksum_step()
 
-        # if we look only at checksums in easyconfig, it should fail
+        # if we look only at (incorrect) checksums in easyconfig, it should fail
         eb = get_easyblock_instance(ec)
         build_options = {
             'checksum_priority': config.CHECKSUM_PRIORITY_EASYCONFIG

test/framework/easyconfig.py

@@ -2933,8 +2933,7 @@ def test_find_related_easyconfigs(self):
         ec['toolchain'] = {'name': 'gompi', 'version': '1.5.16'}
         ec['versionsuffix'] = '-foobar'
         res = [os.path.basename(x) for x in find_related_easyconfigs(test_easyconfigs, ec)]
-        self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-testjson.eb',
-                               'toy-0.0-gompi-2018a-test.eb'])
+        self.assertEqual(res, ['toy-0.0-gompi-2018a.eb', 'toy-0.0-gompi-2018a-test.eb'])
 
         # restore original versionsuffix => matching versionsuffix wins over matching toolchain (name)
         ec['versionsuffix'] = '-deps'

…nfigs/test_ecs/t/toy/checksums-test.json …asyconfigs/test_ecs/t/toy/checksums.jsontest/framework/easyconfigs/test_ecs/t/toy/checksums-test.json renamed to test/framework/easyconfigs/test_ecs/t/toy/checksums.json test/framework/easyconfigs/test_ecs/t/toy/checksums-test.json renamed to test/framework/easyconfigs/test_ecs/t/toy/checksums.json

@@ -2384,7 +2384,7 @@ def test_index_functions(self):
         # test with specified path with and without trailing '/'s
         for path in [test_ecs, test_ecs + '/', test_ecs + '//']:
             index = ft.create_index(path)
-            self.assertEqual(len(index), 91)
+            self.assertEqual(len(index), 90)
 
             expected = [
                 os.path.join('b', 'bzip2', 'bzip2-1.0.6-GCC-4.9.2.eb'),
@@ -2395,7 +2395,7 @@ def test_index_functions(self):
                 self.assertTrue(fn in index)
 
             for fp in index:
-                self.assertTrue(fp.endswith(('.eb', '.json')))
+                self.assertTrue(fp.endswith('.eb') or os.path.basename(fp) == 'checksums.json')
 
         # set up some files to create actual index file for
         ft.copy_dir(os.path.join(test_ecs, 'g'), os.path.join(self.test_prefix, 'g'))

test/framework/easyconfigs/test_ecs/t/toy/toy-0.0-gompi-2018a-testjson.eb

@@ -1105,7 +1105,7 @@ def test_module_naming_scheme(self):
 
         ecs_dir = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'test_ecs')
         ec_files = [os.path.join(subdir, fil) for (subdir, _, files) in os.walk(ecs_dir) for fil in files]
-        # keep only .eb files
+        # keep only easyconfig files (there may be additional files like patches, checksums.json, etc.)
         ec_files = [x for x in ec_files if x.endswith('.eb')]
 
         build_options = {

test/framework/filetools.py

@@ -5903,6 +5903,9 @@ def test_enforce_checksums(self):
         toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
         test_ec = os.path.join(self.test_prefix, 'test.eb')
 
+        # wipe $EASYBUILD_ROBOT_PATHS to avoid that checksums.json for toy is found in test_ecs
+        del os.environ['EASYBUILD_ROBOT_PATHS']
+
         args = [
             test_ec,
             '--stop=source',