ENH: Use flake8 to check for PEP8 violations in doctests

scripts/tests/test_validate_docstrings.py

@@ -127,6 +127,7 @@ def head1(self, n=5):
 
         Examples
         --------
+        >>> import pandas as pd
         >>> s = pd.Series(['Ant', 'Bear', 'Cow', 'Dog', 'Falcon'])
         >>> s.head()
         0   Ant
@@ -164,6 +165,8 @@ def contains(self, pat, case=True, na=np.nan):
 
         Examples
         --------
+        >>> import numpy as np
+        >>> import pandas as pd
         >>> s = pd.Series(['Antelope', 'Lion', 'Zebra', np.nan])
         >>> s.str.contains(pat='a')
         0    False
@@ -584,6 +587,29 @@ def prefix_pandas(self):
         pass
 
 
+class BadExamples(object):
+
+    def missing_numpy_import(self):
+        """
+        Return whether each value contains `pat`.
+
+        Examples
+        --------
+        >>> import pandas as pd
+        >>> df = pd.DataFrame(np.ones((3, 3)),
+        ...                   columns=('a', 'b', 'c'))
+        >>> df.all(1)
+        0    True
+        1    True
+        2    True
+        dtype: bool
+
+        >>> df.all(bool_only=True)
+        Series([], dtype: bool)
+        """
+        pass
+
+
 class TestValidator(object):
 
     def _import_path(self, klass=None, func=None):
@@ -703,10 +729,13 @@ def test_bad_generic_functions(self, func):
         # See Also tests
         ('BadSeeAlso', 'prefix_pandas',
          ('pandas.Series.rename in `See Also` section '
-          'does not need `pandas` prefix',))
+          'does not need `pandas` prefix',)),
+        # Examples tests
+        ('BadExamples', 'missing_numpy_import',
+         ('1 F821 undefined name \'np\'',))
     ])
     def test_bad_examples(self, capsys, klass, func, msgs):
-        result = validate_one(self._import_path(klass=klass, func=func))  # noqa:F821
+        result = validate_one(self._import_path(klass=klass, func=func))
         for msg in msgs:
             assert msg in ' '.join(result['errors'])
 

scripts/validate_docstrings.py

@@ -24,6 +24,10 @@
 import inspect
 import importlib
 import doctest
+import tempfile
+
+from flake8.main import application as flake8
+
 try:
     from io import StringIO
 except ImportError:
@@ -331,6 +335,26 @@ def parameter_mismatches(self):
 
         return errs
 
+    def validate_pep8(self):
+        create_function = 'def {name}():\n' \
+                          '    """\n{examples}\n    """\n' \
+                          '    pass\n'
+
+        name = self.name.split('.')[-1]
+        lines = (' ' * 4 + line if line else '' for line in self.examples)
+        examples = '\n'.join(lines)
+        with tempfile.NamedTemporaryFile(mode='w') as file:
+            func = create_function.format(name=name, examples=examples)
+            file.write(func)
+            file.flush()
+
+            application = flake8.Application()
+            application.initialize(["--doctests", "--quiet"])
+            application.run_checks([file.name])
+            application.report()
+
+            yield from application.guide.stats.statistics_for('')
+
     @property
     def correct_parameters(self):
         return not bool(self.parameter_mismatches)
@@ -490,6 +514,13 @@ def validate_one(func_name):
         for param_err in param_errs:
             errs.append('\t{}'.format(param_err))
 
+    pep8_errs = [error for error in doc.validate_pep8()]
+    if pep8_errs:
+        errs.append('Errors in doctests')
+        for err in pep8_errs:
+            errs.append('\t{} {} {}'.format(err.count, err.error_code,
+                                            err.message))
+
     if doc.is_function_or_method:
         if not doc.returns and "return" in doc.method_source:
             errs.append('No Returns section found')