ENH: Use flake8 to check for PEP8 violations in doctests

scripts/tests/test_validate_docstrings.py

@@ -319,8 +319,6 @@ def method(self, foo=None, bar=None):
 
         Examples
         --------
-        >>> import numpy as np
-        >>> import pandas as pd
         >>> df = pd.DataFrame(np.ones((3, 3)),
         ...                   columns=('a', 'b', 'c'))
         >>> df.all(1)
@@ -584,6 +582,53 @@ def prefix_pandas(self):
         pass
 
 
+class BadExamples(object):
+
+    def numpy_imported(self):
+        """
+        Examples
+        --------
+        >>> import numpy as np
+        >>> df = pd.DataFrame(np.ones((3, 3)), columns=('a', 'b', 'c'))
+        """
+        pass
+
+    def pandas_imported(self):
+        """
+        Examples
+        --------
+        >>> import pandas as pd
+        >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan])
+        """
+        pass
+
+    def missing_whitespace_around_arithmetic_operator(self):
+        """
+        Examples
+        --------
+        >>> 2+5
+        7
+        """
+        pass
+
+    def indentation_is_not_a_multiple_of_four(self):
+        """
+        Examples
+        --------
+        >>> if 2 + 5:
+        ...   pass
+        """
+        pass
+
+    def missing_whitespace_after_comma(self):
+        """
+        Examples
+        --------
+        >>> df = pd.DataFrame(np.ones((3,3)),columns=('a','b', 'c'))
+        """
+        pass
+
+
 class TestValidator(object):
 
     def _import_path(self, klass=None, func=None):
@@ -703,10 +748,23 @@ def test_bad_generic_functions(self, func):
         # See Also tests
         ('BadSeeAlso', 'prefix_pandas',
          ('pandas.Series.rename in `See Also` section '
-          'does not need `pandas` prefix',))
+          'does not need `pandas` prefix',)),
+        # Examples tests
+        ('BadExamples', 'numpy_imported',
+         ('F811 It\'s assumed that pandas and numpy'
+          ' are imported as pd or np',)),
+        ('BadExamples', 'pandas_imported',
+         ('F811 It\'s assumed that pandas and numpy'
+          ' are imported as pd or np',)),
+        ('BadExamples', 'indentation_is_not_a_multiple_of_four',
+         ('E111 indentation is not a multiple of four',)),
+        ('BadExamples', 'missing_whitespace_around_arithmetic_operator',
+         ('E226 missing whitespace around arithmetic operator',)),
+        ('BadExamples', 'missing_whitespace_after_comma',
+         ('3 E231 missing whitespace after \',\'',)),
     ])
     def test_bad_examples(self, capsys, klass, func, msgs):
-        result = validate_one(self._import_path(klass=klass, func=func))  # noqa:F821
+        result = validate_one(self._import_path(klass=klass, func=func))
         for msg in msgs:
             assert msg in ' '.join(result['errors'])
 

scripts/validate_docstrings.py

@@ -13,6 +13,7 @@
     $ ./validate_docstrings.py
     $ ./validate_docstrings.py pandas.DataFrame.head
 """
+import contextlib
 import os
 import sys
 import json
@@ -24,6 +25,10 @@
 import inspect
 import importlib
 import doctest
+import tempfile
+
+import flake8.main.application
+
 try:
     from io import StringIO
 except ImportError:
@@ -331,6 +336,33 @@ def parameter_mismatches(self):
 
         return errs
 
+    @contextlib.contextmanager
+    def _write_examples_code_to_temp_file(self):
+        """
+        Generate file with source code from examples section.
+        """
+        content = ''.join(('import numpy as np; '
+                           'import pandas as pd  # noqa: F401,E702\n',
+                           *self.examples_source_code))
+        with tempfile.NamedTemporaryFile(mode='w') as file:
+            file.write(content)
+            file.flush()
+            yield file
+
+    def validate_pep8(self):
+        if self.examples:
+            with self._write_examples_code_to_temp_file() as file:
+                application = flake8.main.application.Application()
+                application.initialize(["--quiet"])
+                application.run_checks([file.name])
+                application.report()
+
+            for statistic in application.guide.stats.statistics_for(''):
+                if statistic.message.endswith('from line 1'):
+                    statistic.message = "It's assumed that pandas and numpy" \
+                                        " are imported as pd or np"
+                yield statistic
+
     @property
     def correct_parameters(self):
         return not bool(self.parameter_mismatches)
@@ -490,6 +522,13 @@ def validate_one(func_name):
         for param_err in param_errs:
             errs.append('\t{}'.format(param_err))
 
+    pep8_errs = list(doc.validate_pep8())
+    if pep8_errs:
+        errs.append('Linting issues in doctests:')
+        for err in pep8_errs:
+            errs.append('\t{} {} {}'.format(err.count, err.error_code,
+                                            err.message))
+
     if doc.is_function_or_method:
         if not doc.returns and "return" in doc.method_source:
             errs.append('No Returns section found')