Add style tests

.travis.yml

@@ -43,7 +43,7 @@ before_install:
     # pydot (dep for python-graph-dot) 1.2.0 and more recent doesn't work with Python 2.6
     - if [ "x$TRAVIS_PYTHON_VERSION" == 'x2.6' ]; then pip install pydot==1.1.0; else pip install pydot; fi
     # optional Python packages for EasyBuild
-    - pip install autopep8 GC3Pie python-graph-dot python-hglib PyYAML
+    - pip install autopep8 GC3Pie pep8 python-graph-dot python-hglib PyYAML
     # git config is required to make actual git commits (cfr. tests for GitRepository)
     - git config --global user.name "Travis CI"
     - git config --global user.email "[email protected]"

easybuild/framework/easyconfig/style.py

@@ -0,0 +1,126 @@
+##
+# Copyright 2016-2016 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://www.vscentrum.be),
+# Flemish Research Foundation (FWO) (http://www.fwo.be/en)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+# #
+
+"""
+Style tests for easyconfig files using pep8.
+
+:author: Ward Poelmans (Ghent University)
+"""
+
+import re
+from vsc.utils import fancylogger
+
+from easybuild.tools.utilities import only_if_module_is_available
+
+try:
+    import pep8
+except ImportError:
+    pass
+
+_log = fancylogger.getLogger('easyconfig.style', fname=False)
+
+EB_CHECK = '_eb_check_'
+
+
+# any function starting with _eb_check_ (see EB_CHECK variable) will be
+# added to the tests if the test number is added to the select list.
+# The test number is definied as WXXX and EXXX (for warnings and errors)
+# where XXX is a 3 digit number.
+#
+# It should be mentioned in the docstring as a single word.
+# Read the pep8 docs to understand the arguments of these functions:
+# https://pep8.readthedocs.org or more specifically:
+# https://pep8.readthedocs.org/en/latest/developer.html#contribute
+def _eb_check_trailing_whitespace(physical_line, lines, line_number, total_lines):  # pylint:disable=unused-argument
+    """
+    W299
+    Warn about trailing whitespace, except for the description and comments.
+    This differs from the standard trailing whitespace check as that
+    will warn for any trailing whitespace.
+    The arguments are explained at
+    https://pep8.readthedocs.org/en/latest/developer.html#contribute
+    """
+    comment_re = re.compile(r'^\s*#')
+    if comment_re.match(physical_line):
+        return None
+
+    result = pep8.trailing_whitespace(physical_line)
+    if result:
+        result = (result[0], result[1].replace("W291", "W299"))
+
+    # if the warning is about the multiline string of description
+    # we will not issue a warning
+    keys_re = re.compile(r"^(?P<key>[a-z_]+)\s*=\s*")
+
+    # starting from the current line and going to the top,
+    # check that if the first `key = value` that is found, has
+    # key == description, then let the test pass, else return
+    # the result of the general pep8 check.
+    for line in reversed(lines[:line_number]):
+        res = keys_re.match(line)
+        if res and res.group("key") == "description":
+            result = None
+            break
+
+    return result
+
+
+@only_if_module_is_available('pep8')
+def check_easyconfigs_style(easyconfigs, verbose=False):
+    """
+    Check the given list of easyconfigs for style
+    :param: easyconfigs list of file paths to easyconfigs
+    :param: verbose print our statistics and be verbose about the errors and warning
+    :return: the number of warnings and errors
+    """
+    # importing autopep8 changes some pep8 functions.
+    # We reload it to be sure to get the real pep8 functions.
+    reload(pep8)
+
+    # register the extra checks before using pep8:
+    # any function in this module starting with `_eb_check_` will be used.
+    cands = globals()
+    for check_function in sorted([cands[f] for f in cands if callable(cands[f]) and f.startswith(EB_CHECK)]):
+        _log.debug("Adding custom style check %s", check_function)
+        pep8.register_check(check_function)
+
+    pep8style = pep8.StyleGuide(quiet=False, config_file=None)
+    options = pep8style.options
+    # we deviate from standard pep8 and allow 120 chars
+    # on a line: the default of 79 is too narrow.
+    options.max_line_length = 120
+    # we ignore some tests
+    # note that W291 has be replaced by our custom W299
+    options.ignore = (
+        'W291',  # replaced by W299
+    )
+    options.verbose = int(verbose)
+
+    result = pep8style.check_files(easyconfigs)
+
+    if verbose:
+        result.print_statistics()
+
+    return result.total_errors

test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.6.3.eb

@@ -1,29 +1,34 @@
 # should be EB_GCC, but OK for testing purposes
 easyblock = 'EB_toy'
 
-name="GCC"
-version='4.6.3'
+name = "GCC"
+version = '4.6.3'
 
-homepage='http://gcc.gnu.org/'
-description="The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada, as well as libraries for these languages (libstdc++, libgcj,...)."
+homepage = 'http://gcc.gnu.org/'
+description = """The GNU Compiler Collection includes front ends for C, C++, Objective-C, Fortran, Java, and Ada,
+as well as libraries for these languages (libstdc++, libgcj,...)."""
 
-toolchain={'name': 'dummy','version': 'dummy'}
+toolchain = {'name': 'dummy', 'version': 'dummy'}
 
-sources=['%s-%s.tar.gz'%(name.lower(),version),
-         'gmp-5.0.4.tar.bz2',
-         'mpfr-3.0.1.tar.gz',
-         'mpc-0.9.tar.gz',
-        ]
-source_urls=['http://ftpmirror.gnu.org/%(name)s/%(name)s-%(version)s' % {'name':name.lower(), 'version':version}, # GCC auto-resolving HTTP mirror
-            'http://ftpmirror.gnu.org/gmp', # idem for GMP
-            'http://ftpmirror.gnu.org/mpfr', # idem for MPFR
-            'http://www.multiprecision.org/mpc/download', # MPC official
-           ]
+sources = [
+    '%s-%s.tar.gz' % (name.lower(), version),
+    'gmp-5.0.4.tar.bz2',
+    'mpfr-3.0.1.tar.gz',
+    'mpc-0.9.tar.gz',
+]
 
-languages=['c', 'c++', 'fortran', 'lto']
+source_urls = [
+    # GCC auto-resolving HTTP mirror
+    'http://ftpmirror.gnu.org/%(name)s/%(name)s-%(version)s' % {'name': name.lower(), 'version': version},
+    'http://ftpmirror.gnu.org/gmp',  # idem for GMP
+    'http://ftpmirror.gnu.org/mpfr',  # idem for MPFR
+    'http://www.multiprecision.org/mpc/download',  # MPC official
+]
 
-## compiler class
-moduleclass='compiler'
+languages = ['c', 'c++', 'fortran', 'lto']
+
+# compiler class
+moduleclass = 'compiler'
 
 # building GCC sometimes fails if make parallelism is too high, so let's limit it
-maxparallel=4
+maxparallel = 4

test/framework/easyconfigs/test_ecs/g/GCC/GCC-4.7.2.eb

@@ -11,10 +11,10 @@ description = """The GNU Compiler Collection includes front ends for C, C++, Obj
 toolchain = {'name': 'dummy', 'version': 'dummy'}
 
 source_urls = [
-        'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s',  # GCC auto-resolving HTTP mirror
-        'http://ftpmirror.gnu.org/gmp',  # idem for GMP
-        'http://ftpmirror.gnu.org/mpfr',  # idem for MPFR
-        'http://www.multiprecision.org/mpc/download',  # MPC official
+    'http://ftpmirror.gnu.org/%(namelower)s/%(namelower)s-%(version)s',  # GCC auto-resolving HTTP mirror
+    'http://ftpmirror.gnu.org/gmp',  # idem for GMP
+    'http://ftpmirror.gnu.org/mpfr',  # idem for MPFR
+    'http://www.multiprecision.org/mpc/download',  # MPC official
 ]
 sources = [
     SOURCELOWER_TAR_GZ,

test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-GCC-4.6.3.eb

@@ -18,10 +18,10 @@ homepage = "http://www.gzip.org/"
 description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"
 
 # test toolchain specification
-toolchain = {'name': 'GCC','version': '4.6.3'}
+toolchain = {'name': 'GCC', 'version': '4.6.3'}
 
 # source tarball filename
-sources = ['%s-%s.tar.gz'%(name,version)]
+sources = ['%(name)s-%(version)s.tar.gz']
 
 # download location for source files
 source_urls = [GNU_SOURCE]

test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.4-broken.eb

@@ -19,7 +19,7 @@ homepage = "http://www.gzip.org/"
 description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"
 
 # test toolchain specification
-toolchain = {'name':'dummy','version':'dummy'}
+toolchain = {'name': 'dummy', 'version': 'dummy'}
 
 # source tarball filename
 sources = [SOURCE_TAR_GZ]

test/framework/easyconfigs/test_ecs/g/gzip/gzip-1.6-GCC-4.9.2.eb

@@ -13,7 +13,7 @@
 easyblock = 'ConfigureMake'
 
 name = 'gzip'
-version = '1.6'  # FIXME bug: quotes are required here to make sure this is parsed a a string, not a floating point value
+version = '1.6'
 
 homepage = 'http://www.gnu.org/software/gzip/'
 description = "gzip (GNU zip) is a popular data compression program as a replacement for compress"

test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.6.4.eb

@@ -12,7 +12,7 @@ description = """The Portable Hardware Locality (hwloc) software package provide
  information about modern computing hardware so as to exploit it accordingly and efficiently."""
 
 toolchain = {'name': 'GCC', 'version': '4.6.4'}
-  
+
 source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
 sources = [SOURCE_TAR_GZ]
 

test/framework/easyconfigs/test_ecs/h/hwloc/hwloc-1.6.2-GCC-4.7.2.eb

@@ -12,7 +12,7 @@ description = """The Portable Hardware Locality (hwloc) software package provide
  information about modern computing hardware so as to exploit it accordingly and efficiently."""
 
 toolchain = {'name': 'GCC', 'version': '4.7.2'}
-  
+
 source_urls = ['http://www.open-mpi.org/software/hwloc/v%(version_major_minor)s/downloads/']
 sources = [SOURCE_TAR_GZ]
 

test/framework/easyconfigs/test_ecs/i/icc/icc-2016.1.150-GCC-4.9.3-2.25.eb

@@ -19,7 +19,7 @@ versionsuffix = '-GCC-%s-%s' % (gccver, binutilsver)
 
 dependencies = [
     ('GCCcore', gccver),
-#    ('binutils', binutilsver, '', ('GCCcore', gccver)),
+    # ('binutils', binutilsver, '', ('GCCcore', gccver)),
 ]
 
 # full list of components can be obtained from pset/mediaconfig.xml in unpacked sources

test/framework/easyconfigs/test_ecs/i/iccifort/iccifort-2016.1.150-GCC-4.9.3-2.25.eb

@@ -6,7 +6,8 @@ version = '2016.1.150'
 versionsuffix = '-GCC-4.9.3-2.25'
 
 homepage = 'http://software.intel.com/en-us/intel-cluster-toolkit-compiler/'
-description = """Intel Cluster Toolkit Compiler Edition provides Intel C,C++ and fortran compilers, Intel MPI and Intel MKL"""
+description = """Intel Cluster Toolkit Compiler Edition provides Intel C,C++ and fortran compilers,
+Intel MPI and Intel MKL"""
 
 toolchain = {'name': 'dummy', 'version': 'dummy'}
 

test/framework/easyconfigs/test_ecs/i/ifort/ifort-2016.1.150-GCC-4.9.3-2.25.eb

@@ -19,7 +19,7 @@ versionsuffix = '-GCC-%s-%s' % (gccver, binutilsver)
 
 dependencies = [
     ('GCCcore', gccver),
-#    ('binutils', binutilsver, '', ('GCCcore', gccver)),
+    # ('binutils', binutilsver, '', ('GCCcore', gccver)),
 ]
 
 # full list of components can be obtained from pset/mediaconfig.xml in unpacked sources

test/framework/easyconfigs/test_ecs/o/OpenBLAS/OpenBLAS-0.2.6-gompi-1.4.10-LAPACK-3.4.2.eb

@@ -41,7 +41,7 @@ buildopts = 'BINARY=64 ' + threading + ' CC="$CC" FC="$F77"'
 installopts = threading + " PREFIX=%(installdir)s"
 
 # extensive testing can be enabled by uncommenting the line below
-#runtest = 'PATH=.:$PATH lapack-timing'
+# runtest = 'PATH=.:$PATH lapack-timing'
 
 sanity_check_paths = {
     'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h',

test/framework/easyconfigs/test_ecs/o/OpenMPI/OpenMPI-1.6.4-GCC-4.6.4.eb

@@ -28,9 +28,9 @@ else:
 sanity_check_paths = {
     'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] +
              ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90",
-                                                                         "mpi", "ompitrace", "open-pal",
-                                                                         "open-rte", "vt", "vt-hyb",
-                                                                         "vt-mpi", "vt-mpi-unify"]] +
+                                                                    "mpi", "ompitrace", "open-pal",
+                                                                    "open-rte", "vt", "vt-hyb",
+                                                                    "vt-mpi", "vt-mpi-unify"]] +
              ["include/%s.h" % x for x in ["mpi-ext", "mpif-common", "mpif-config", "mpif",
                                            "mpif-mpi-io", "mpi", "mpi_portable_platform"]],
     'dirs': ["include/openmpi/ompi/mpi/cxx"],

test/framework/easyconfigs/test_ecs/o/OpenMPI/OpenMPI-1.6.4-GCC-4.7.2.eb

@@ -28,9 +28,9 @@ else:
 sanity_check_paths = {
     'files': ["bin/%s" % binfile for binfile in ["ompi_info", "opal_wrapper", "orterun"]] +
              ["lib/lib%s.%s" % (libfile, SHLIB_EXT) for libfile in ["mpi_cxx", "mpi_f77", "mpi_f90",
-                                                                         "mpi", "ompitrace", "open-pal",
-                                                                         "open-rte", "vt", "vt-hyb",
-                                                                         "vt-mpi", "vt-mpi-unify"]] +
+                                                                    "mpi", "ompitrace", "open-pal",
+                                                                    "open-rte", "vt", "vt-hyb",
+                                                                    "vt-mpi", "vt-mpi-unify"]] +
              ["include/%s.h" % x for x in ["mpi-ext", "mpif-common", "mpif-config", "mpif",
                                            "mpif-mpi-io", "mpi", "mpi_portable_platform"]],
     'dirs': ["include/openmpi/ompi/mpi/cxx"],

test/framework/easyconfigs/test_ecs/p/Python/Python-2.7.10-ictce-4.1.13.eb

@@ -19,14 +19,14 @@ sources = [SOURCE_TGZ]
 # python needs bzip2 to build the bz2 package
 # commented out for testing to avoid having to add them all - dependencies are tested in other files
 dependencies = [
-#    ('bzip2', '1.0.6'),
-#    ('zlib', '1.2.8'),
-#    ('libreadline', '6.3'),
-#    ('ncurses', '5.9'),
-#    ('SQLite', '3.8.10.2'),
-#    ('Tk', '8.6.4', '-no-X11'),
-#    ('OpenSSL', '1.0.1m'),  # OS dependency should be preferred if the os version is more recent then this version, it's
-#    nice to have an up to date openssl for security reasons
+    # ('bzip2', '1.0.6'),
+    # ('zlib', '1.2.8'),
+    # ('libreadline', '6.3'),
+    # ('ncurses', '5.9'),
+    # ('SQLite', '3.8.10.2'),
+    # ('Tk', '8.6.4', '-no-X11'),
+    # ('OpenSSL', '1.0.1m'),  # OS dependency should be preferred if the os version is more recent then this version,
+    # it's nice to have an up to date openssl for security reasons
 ]
 
 # zlib is only included here for the sake of testing parsing of .yeb easyconfigs!
@@ -47,7 +47,7 @@ exts_list = [
     ['numpy', numpyversion, {
         'source_urls': [['http://sourceforge.net/projects/numpy/files/NumPy/%s' % numpyversion, 'download']],
         'patches': [
-            'numpy-1.8.0-mkl.patch', # % numpyversion,
+            'numpy-1.8.0-mkl.patch',  # % numpyversion,
         ],
     }],
     ['scipy', scipyversion, {
@@ -123,4 +123,3 @@ exts_list = [
 ]
 
 moduleclass = 'lang'
-

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-GCC-4.7.2.eb

@@ -27,8 +27,8 @@ sources = ['sqlite-autoconf-%s.tar.gz' % version_str]
 
 # commented out for testing to avoid having to add them all - dependencies are tested in other files
 dependencies = [
-#    ('libreadline', '6.3'),
-#    ('Tcl', '8.6.4'),
+    # ('libreadline', '6.3'),
+    # ('Tcl', '8.6.4'),
 ]
 
 parallel = 1

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-gompi-1.4.10.eb

@@ -27,8 +27,8 @@ sources = ['sqlite-autoconf-%s.tar.gz' % version_str]
 
 # commented out for testing to avoid having to add them all - dependencies are tested in other files
 dependencies = [
-#    ('libreadline', '6.3'),
-#    ('Tcl', '8.6.4'),
+    # ('libreadline', '6.3'),
+    # ('Tcl', '8.6.4'),
 ]
 
 parallel = 1

test/framework/easyconfigs/test_ecs/s/SQLite/SQLite-3.8.10.2-goolf-1.4.10.eb

@@ -27,8 +27,8 @@ sources = ['sqlite-autoconf-%s.tar.gz' % version_str]
 
 # commented out for testing to avoid having to add them all - dependencies are tested in other files
 dependencies = [
-#    ('libreadline', '6.3'),
-#    ('Tcl', '8.6.4'),
+    # ('libreadline', '6.3'),
+    # ('Tcl', '8.6.4'),
 ]
 
 parallel = 1

test/framework/easyconfigs/test_ecs/s/ScaLAPACK/ScaLAPACK-2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2.eb

@@ -22,7 +22,7 @@ versionsuffix = "-%s-%s%s" % (blaslib, blasver, blassuff)
 
 dependencies = [(blaslib, blasver, blassuff)]
 
-## parallel build tends to fail, so disabling it
+# parallel build tends to fail, so disabling it
 parallel = 1
 
 moduleclass = 'numlib'