Restructure hooks as a sub-structure of mypy.options.Options

Author: chadrik

mypy/fastparse.py

@@ -29,7 +29,6 @@
 from mypy import defaults
 from mypy import experiments
 from mypy import messages
-from mypy import hooks
 from mypy.errors import Errors
 from mypy.options import Options
 
@@ -115,13 +114,13 @@ def parse_type_comment(type_comment: str, line: int, errors: Optional[Errors]) -
         return TypeConverter(errors, line=line).visit(typ.body)
 
 
-def parse_docstring(docstring: str, arg_names: List[str],
+def parse_docstring(options: Options, docstring: str, arg_names: List[str],
                     line: int, errors: Errors) -> Optional[Tuple[List[Type], Type]]:
     """Parse a docstring and return type representations.
 
     Returns a 2-tuple: (list of arguments Types, and return Type).
     """
-    opts = hooks.options.get('docstring_parser', {})
+    hook = options.hooks.docstring_parser
 
     def pop_and_convert(name: str) -> Optional[Type]:
         t = type_map.pop(name, None)
@@ -130,8 +129,8 @@ def pop_and_convert(name: str) -> Optional[Type]:
         else:
             return parse_type_comment(t[0], line + t[1], errors)
 
-    if hooks.docstring_parser is not None:
-        type_map = hooks.docstring_parser(docstring, opts, errors)
+    if hook.func is not None:
+        type_map = options.hooks.docstring_parser.func(docstring, hook.options, errors)
         if type_map:
             arg_types = [pop_and_convert(name) for name in arg_names]
             return_type = pop_and_convert('return')
@@ -376,11 +375,10 @@ def do_func_def(self, n: Union[ast3.FunctionDef, ast3.AsyncFunctionDef],
             return_type = TypeConverter(self.errors, line=n.returns.lineno
                                         if n.returns else n.lineno).visit(n.returns)
             # hooks
-            if (not any(arg_types) and return_type is None and
-                    hooks.docstring_parser):
+            if not any(arg_types) and return_type is None:
                 doc = ast3.get_docstring(n, clean=False)
                 if doc:
-                    types = parse_docstring(doc, real_names, n.lineno, self.errors)
+                    types = parse_docstring(self.options, doc, real_names, n.lineno, self.errors)
                     if types is not None:
                         arg_types, return_type = types
 

mypy/fastparse2.py

@@ -324,11 +324,10 @@ def visit_FunctionDef(self, n: ast27.FunctionDef) -> Statement:
             arg_types = [a.type_annotation for a in args]
             return_type = converter.visit(None)
             # hooks
-            if (not any(arg_types) and return_type is None and
-                    hooks.docstring_parser):
+            if not any(arg_types) and return_type is None:
                 doc = ast27.get_docstring(n, clean=False)
                 if doc:
-                    types = parse_docstring(doc.decode('unicode_escape'),
+                    types = parse_docstring(self.options, doc.decode('unicode_escape'),
                                             real_names, n.lineno, self.errors)
                     if types is not None:
                         arg_types, return_type = types

mypy/hooks.py

@@ -1,8 +1,92 @@
-from typing import Dict, Optional, Callable, Tuple
-from mypy.errors import Errors
+import importlib.util
+import os.path
+import pprint
+from types import ModuleType
+
+from typing import Callable, Dict, Generic, Optional, Tuple, TypeVar, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from mypy.errors import Errors
+
+
+HookOptions = Dict[str, str]
+
+T = TypeVar('T', bound=Callable)
+
+
+def _safeimport(path: str) -> Optional[ModuleType]:
+    try:
+        module = __import__(path)
+    except ImportError as err:
+        if err.name == path:  # type: ignore  # ImportError stubs need to be updated
+            # No such module in the path.
+            return None
+        else:
+            raise
+    for part in path.split('.')[1:]:
+        try:
+            module = getattr(module, part)
+        except AttributeError:
+            return None
+    return module
+
+
+def locate(path: str) -> Optional[object]:
+    """Locate an object by string identifier, importing as necessary.
+
+    The two identifiers supported are:
+
+    dotted path:
+        package.module.object.child
+
+    file path followed by dotted object path]
+        /path/to/module.py:object.child
+    """
+    if ':' in path:
+        file_path, obj_path = path.split(':', 1)
+        mod_name = os.path.splitext(os.path.basename(file_path))[0]
+        spec = importlib.util.spec_from_file_location(mod_name, file_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        parts = obj_path.split('.')
+    else:
+        parts = [part for part in path.split('.') if part]
+        module, n = None, 0
+        while n < len(parts):
+            nextmodule = _safeimport('.'.join(parts[:n + 1]))
+            if nextmodule:
+                module, n = nextmodule, n + 1
+            else:
+                break
+        parts = parts[n:]
+
+    if not module:
+        return None
+
+    obj = module
+    for part in parts:
+        try:
+            obj = getattr(obj, part)
+        except AttributeError:
+            return None
+    return obj
+
+
+class Hook(Generic[T]):
+
+    def __init__(self) -> None:
+        self.func = None  # type: T
+        self.options = {}  # type: HookOptions
+
+    def __eq__(self, other: object) -> bool:
+        return self.__class__ == other.__class__ and self.__dict__ == other.__dict__
+
+    def __ne__(self, other: object) -> bool:
+        return not self == other
+
+    def __repr__(self) -> str:
+        return 'Hook({}, {})'.format(self.func, self.options)
 
-Options = Dict[str, str]
-options = {}  # type: Dict[str, Options]
 
 # The docstring_parser hook is called for each unannotated function that has a
 # docstring. The callable should accept three arguments:
@@ -20,4 +104,18 @@
 # The function's return type, if specified, is stored in the mapping with the special
 # key 'return'.  Other than 'return', each key of the mapping must be one of the
 # arguments of the documented function; otherwise, an error will be raised.
-docstring_parser = None  # type: Callable[[str, Options, Errors], Dict[str, Tuple[str, int]]]
+DocstringParserHook = Hook[Callable[[str, HookOptions, 'Errors'], Dict[str, Tuple[str, int]]]]
+
+
+class Hooks:
+    def __init__(self) -> None:
+        self.docstring_parser = DocstringParserHook()
+
+    def __eq__(self, other: object) -> bool:
+        return self.__class__ == other.__class__ and self.__dict__ == other.__dict__
+
+    def __ne__(self, other: object) -> bool:
+        return not self == other
+
+    def __repr__(self) -> str:
+        return 'Hooks({})'.format(pprint.pformat(self.__dict__))

mypy/main.py

@@ -7,17 +7,16 @@
 import re
 import sys
 import time
-from pydoc import locate
 
-from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Callable
+from typing import Any, Callable, cast, Dict, List, Mapping, Optional, Set, Tuple
 
 from mypy import build
 from mypy import defaults
 from mypy import experiments
-from mypy import hooks
 from mypy import util
 from mypy.build import BuildSource, BuildResult, PYTHON_EXTENSIONS
 from mypy.errors import CompileError
+from mypy.hooks import locate, Hook
 from mypy.options import Options, BuildType
 from mypy.report import reporter_classes
 
@@ -578,16 +577,20 @@ def get_init_file(dir: str) -> Optional[str]:
 
 
 def load_hook(prefix: str, hook_name: str, hook_path: str) -> Optional[Callable]:
-    # FIXME: no stubs for pydoc.  should we write stubs or a simple replacement for locate?
-    obj = locate(hook_path)
+    try:
+        obj = locate(hook_path)
+    except BaseException as err:
+        print("%s: Error finding hook %s at %s: %s" %
+              (prefix, hook_name, hook_path, err), file=sys.stderr)
+        return None
     if obj is None:
         print("%s: Could not find hook %s at %s" %
               (prefix, hook_name, hook_path), file=sys.stderr)
-    if not callable(obj):
+    elif not callable(obj):
         print("%s: Hook %s at %s is not callable" %
               (prefix, hook_name, hook_path), file=sys.stderr)
         return None
-    return obj
+    return cast(Callable, obj)
 
 
 # For most options, the type of the default value set in options.py is
@@ -644,32 +647,31 @@ def parse_config_file(options: Options, filename: Optional[str]) -> None:
     else:
         section = parser['mypy']
         prefix = '%s: [%s]' % (file_read, 'mypy')
-        updates, report_dirs, hook_funcs = parse_section(prefix, options, section)
+        updates, report_dirs, hooks = parse_section(prefix, options, section)
         for k, v in updates.items():
             setattr(options, k, v)
 
-        # bind hook functions to hooks module
-        for k, v in hook_funcs.items():
+        for k, v in hooks.items():
             hook_func = load_hook(prefix, k, v)
             if hook_func is not None:
-                # FIXME: dynamically check loaded function annotations against those in `hooks`?
-                setattr(hooks, k, hook_func)
+                hook = getattr(options.hooks, k)  # type: Hook
+                hook.func = hook_func
                 # look for an options section for this hook
                 if k in parser:
-                    hooks.options[k] = dict(parser[k])
+                    hook.options = dict(parser[k])
         options.report_dirs.update(report_dirs)
 
     for name, section in parser.items():
         if name.startswith('mypy-'):
             prefix = '%s: [%s]' % (file_read, name)
-            updates, report_dirs, hook_funcs = parse_section(prefix, options, section)
+            updates, report_dirs, hooks = parse_section(prefix, options, section)
             if report_dirs:
                 print("%s: Per-module sections should not specify reports (%s)" %
                       (prefix, ', '.join(s + '_report' for s in sorted(report_dirs))),
                       file=sys.stderr)
-            if hook_funcs:
+            if hooks:
                 print("%s: Per-module sections should not specify hooks (%s)" %
-                      (prefix, ', '.join(sorted(hook_funcs))),
+                      (prefix, ', '.join(sorted(hooks))),
                       file=sys.stderr)
             if set(updates) - Options.PER_MODULE_OPTIONS:
                 print("%s: Per-module sections should only specify per-module flags (%s)" %
@@ -695,17 +697,17 @@ def parse_section(prefix: str, template: Options,
     """
     results = {}  # type: Dict[str, object]
     report_dirs = {}  # type: Dict[str, str]
-    hook_funcs = {}  # type: Dict[str, str]
+    hooks = {}  # type: Dict[str, str]
     for key in section:
         key = key.replace('-', '_')
         if key.startswith('hooks.'):
             dv = section.get(key)
             key = key[6:]
-            if not hasattr(hooks, key):
+            if not hasattr(template.hooks, key):
                 print("%s: Unrecognized hook: %s = %s" % (prefix, key, dv),
                       file=sys.stderr)
             else:
-                hook_funcs[key] = dv
+                hooks[key] = dv
             continue
         elif key in config_types:
             ct = config_types[key]
@@ -755,7 +757,7 @@ def parse_section(prefix: str, template: Options,
                 if 'follow_imports' not in results:
                     results['follow_imports'] = 'error'
         results[key] = v
-    return results, report_dirs, hook_funcs
+    return results, report_dirs, hooks
 
 
 def fail(msg: str) -> None:

mypy/options.py

@@ -5,6 +5,7 @@
 from typing import Any, Mapping, Optional, Tuple, List, Pattern, Dict
 
 from mypy import defaults
+from mypy.hooks import Hooks
 
 
 class BuildType:
@@ -132,6 +133,7 @@ def __init__(self) -> None:
         self.shadow_file = None  # type: Optional[Tuple[str, str]]
         self.show_column_numbers = False  # type: bool
         self.dump_graph = False
+        self.hooks = Hooks()
 
     def __eq__(self, other: object) -> bool:
         return self.__class__ == other.__class__ and self.__dict__ == other.__dict__