Disable monkeypatches, add dependencies via new hook (typeddjango#60)

* code cleanups, disable monkeypatches, move to add_additional_deps

* disable incremental mode for tests

* add pip-wheel-metadata

* move some code from get_base_hook to get_attribute_hook to reduce dependencies

* simplify values/values_list tests and code

* disable cache for some tests failing with incremental mode

* enable incremental mode for tests typechecking

* pin mypy version

* fix tests

* lint

* fix internal crashes

Author: mkurnikov

.gitignore

@@ -7,4 +7,5 @@ out/
 .mypy_cache/
 django-sources
 build/
-dist/
+dist/
+pip-wheel-metadata/

django-stubs/conf/__init__.pyi

@@ -2,10 +2,14 @@ from typing import Any
 
 from django.utils.functional import LazyObject
 
+# explicit dependency on standard settings to make it loaded
+from . import global_settings
+
 ENVIRONMENT_VARIABLE: str = ...
 
 # required for plugin to be able to distinguish this specific instance of LazySettings from others
-class _DjangoConfLazyObject(LazyObject): ...
+class _DjangoConfLazyObject(LazyObject):
+    def __getattr__(self, item: Any) -> Any: ...
 
 class LazySettings(_DjangoConfLazyObject):
     configured: bool

django-stubs/conf/global_settings.pyi

@@ -5,14 +5,11 @@ by the DJANGO_SETTINGS_MODULE environment variable.
 
 # This is defined here as a do-nothing function because we can't import
 # django.utils.translation -- that module depends on the settings.
-from typing import Any, Dict, List, Optional, Pattern, Tuple, Protocol, Union, Callable, TYPE_CHECKING, Sequence
+from typing import Any, Dict, List, Optional, Pattern, Protocol, Sequence, Tuple, Union
 
 ####################
 # CORE             #
 ####################
-if TYPE_CHECKING:
-    from django.db.models.base import Model
-
 DEBUG: bool = ...
 
 # Whether the framework should propagate raw exceptions rather than catching
@@ -153,7 +150,7 @@ FORCE_SCRIPT_NAME = None
 #     ]
 DISALLOWED_USER_AGENTS: List[Pattern] = ...
 
-ABSOLUTE_URL_OVERRIDES: Dict[str, Callable[[Model], str]] = ...
+ABSOLUTE_URL_OVERRIDES: Dict[str, Any] = ...
 
 # List of compiled regular expression objects representing URLs that need not
 # be reported by BrokenLinkEmailsMiddleware. Here are a few examples:

mypy_django_plugin/helpers.py

@@ -4,12 +4,13 @@
 
 from mypy.mro import calculate_mro
 from mypy.nodes import (
-    AssignmentStmt, ClassDef, Expression, ImportedName, Lvalue, MypyFile, NameExpr, SymbolNode, TypeInfo,
-    SymbolTable, SymbolTableNode, Block, GDEF, MDEF, Var)
-from mypy.plugin import FunctionContext, MethodContext
+    GDEF, MDEF, AssignmentStmt, Block, CallExpr, ClassDef, Expression, ImportedName, Lvalue, MypyFile, NameExpr,
+    SymbolNode, SymbolTable, SymbolTableNode, TypeInfo, Var,
+)
+from mypy.plugin import CheckerPluginInterface, FunctionContext, MethodContext
 from mypy.types import (
-    AnyType, Instance, NoneTyp, Type, TypeOfAny, TypeVarType, UnionType,
-    TupleType, TypedDictType)
+    AnyType, Instance, NoneTyp, TupleType, Type, TypedDictType, TypeOfAny, TypeVarType, UnionType,
+)
 
 if typing.TYPE_CHECKING:
     from mypy.checker import TypeChecker
@@ -216,6 +217,7 @@ def extract_field_setter_type(tp: Instance) -> Optional[Type]:
 
 
 def extract_field_getter_type(tp: Type) -> Optional[Type]:
+    """ Extract return type of __get__ of subclass of Field"""
     if not isinstance(tp, Instance):
         return None
     if tp.type.has_base(FIELD_FULLNAME):
@@ -226,13 +228,12 @@ def extract_field_getter_type(tp: Type) -> Optional[Type]:
     return None
 
 
-def get_django_metadata(model: TypeInfo) -> Dict[str, typing.Any]:
-    return model.metadata.setdefault('django', {})
+def get_django_metadata(model_info: TypeInfo) -> Dict[str, typing.Any]:
+    return model_info.metadata.setdefault('django', {})
 
 
 def get_related_field_primary_key_names(base_model: TypeInfo) -> typing.List[str]:
-    django_metadata = get_django_metadata(base_model)
-    return django_metadata.setdefault('related_field_primary_keys', [])
+    return get_django_metadata(base_model).setdefault('related_field_primary_keys', [])
 
 
 def get_fields_metadata(model: TypeInfo) -> Dict[str, typing.Any]:
@@ -243,6 +244,10 @@ def get_lookups_metadata(model: TypeInfo) -> Dict[str, typing.Any]:
     return get_django_metadata(model).setdefault('lookups', {})
 
 
+def get_related_managers_metadata(model: TypeInfo) -> Dict[str, typing.Any]:
+    return get_django_metadata(model).setdefault('related_managers', {})
+
+
 def extract_explicit_set_type_of_model_primary_key(model: TypeInfo) -> Optional[Type]:
     """
     If field with primary_key=True is set on the model, extract its __set__ type.
@@ -310,7 +315,7 @@ def is_field_nullable(model: TypeInfo, field_name: str) -> bool:
     return get_fields_metadata(model).get(field_name, {}).get('null', False)
 
 
-def is_foreign_key(t: Type) -> bool:
+def is_foreign_key_like(t: Type) -> bool:
     if not isinstance(t, Instance):
         return False
     return has_any_of_bases(t.type, (FOREIGN_KEY_FULLNAME, ONETOONE_FIELD_FULLNAME))
@@ -366,13 +371,14 @@ def make_named_tuple(api: 'TypeChecker', fields: 'OrderedDict[str, Type]', name:
     return TupleType(list(fields.values()), fallback=fallback)
 
 
-def make_typeddict(api: 'TypeChecker', fields: 'OrderedDict[str, Type]', required_keys: typing.Set[str]) -> Type:
+def make_typeddict(api: CheckerPluginInterface, fields: 'OrderedDict[str, Type]',
+                   required_keys: typing.Set[str]) -> TypedDictType:
     object_type = api.named_generic_type('mypy_extensions._TypedDict', [])
     typed_dict_type = TypedDictType(fields, required_keys=required_keys, fallback=object_type)
     return typed_dict_type
 
 
-def make_tuple(api: 'TypeChecker', fields: typing.List[Type]) -> Type:
+def make_tuple(api: 'TypeChecker', fields: typing.List[Type]) -> TupleType:
     implicit_any = AnyType(TypeOfAny.special_form)
     fallback = api.named_generic_type('builtins.tuple', [implicit_any])
     return TupleType(fields, fallback=fallback)
@@ -386,3 +392,52 @@ def get_private_descriptor_type(type_info: TypeInfo, private_field_name: str, is
             descriptor_type = make_optional(descriptor_type)
         return descriptor_type
     return AnyType(TypeOfAny.unannotated)
+
+
+def iter_over_classdefs(module_file: MypyFile) -> typing.Iterator[ClassDef]:
+    for defn in module_file.defs:
+        if isinstance(defn, ClassDef):
+            yield defn
+
+
+def iter_call_assignments(klass: ClassDef) -> typing.Iterator[typing.Tuple[Lvalue, CallExpr]]:
+    for lvalue, rvalue in iter_over_assignments(klass):
+        if isinstance(rvalue, CallExpr):
+            yield lvalue, rvalue
+
+
+def get_related_manager_type_from_metadata(model_info: TypeInfo, related_manager_name: str,
+                                           api: CheckerPluginInterface) -> Optional[Instance]:
+    related_manager_metadata = get_related_managers_metadata(model_info)
+    if not related_manager_metadata:
+        return None
+
+    if related_manager_name not in related_manager_metadata:
+        return None
+
+    manager_class_name = related_manager_metadata[related_manager_name]['manager']
+    of = related_manager_metadata[related_manager_name]['of']
+    of_types = []
+    for of_type_name in of:
+        if of_type_name == 'any':
+            of_types.append(AnyType(TypeOfAny.implementation_artifact))
+        else:
+            try:
+                of_type = api.named_generic_type(of_type_name, [])
+            except AssertionError:
+                # Internal error: attempted lookup of unknown name
+                of_type = AnyType(TypeOfAny.implementation_artifact)
+
+            of_types.append(of_type)
+
+    return api.named_generic_type(manager_class_name, of_types)
+
+
+def get_primary_key_field_name(model_info: TypeInfo) -> Optional[str]:
+    for base in model_info.mro:
+        fields = get_fields_metadata(base)
+        for field_name, field_props in fields.items():
+            is_primary_key = field_props.get('primary_key', False)
+            if is_primary_key:
+                return field_name
+    return None

mypy_django_plugin/lookups.py

@@ -1,9 +1,9 @@
-import dataclasses
-from typing import Union, List
+from typing import List, Union
 
+import dataclasses
 from mypy.nodes import TypeInfo
 from mypy.plugin import CheckerPluginInterface
-from mypy.types import Type, Instance
+from mypy.types import Instance, Type
 
 from mypy_django_plugin import helpers
 
@@ -57,20 +57,24 @@ def resolve_lookup(api: CheckerPluginInterface, model_type_info: TypeInfo, looku
     return nodes
 
 
+def resolve_model_pk_lookup(api: CheckerPluginInterface, model_type_info: TypeInfo) -> LookupNode:
+    # Primary keys are special-cased
+    primary_key_type = helpers.extract_primary_key_type_for_get(model_type_info)
+    if primary_key_type:
+        return FieldNode(primary_key_type)
+    else:
+        # No PK, use the get type for AutoField as PK type.
+        autofield_info = api.lookup_typeinfo('django.db.models.fields.AutoField')
+        pk_type = helpers.get_private_descriptor_type(autofield_info, '_pyi_private_get_type',
+                                                      is_nullable=False)
+        return FieldNode(pk_type)
+
+
 def resolve_model_lookup(api: CheckerPluginInterface, model_type_info: TypeInfo,
                          lookup: str) -> LookupNode:
     """Resolve a lookup on the given model."""
     if lookup == 'pk':
-        # Primary keys are special-cased
-        primary_key_type = helpers.extract_primary_key_type_for_get(model_type_info)
-        if primary_key_type:
-            return FieldNode(primary_key_type)
-        else:
-            # No PK, use the get type for AutoField as PK type.
-            autofield_info = api.lookup_typeinfo('django.db.models.fields.AutoField')
-            pk_type = helpers.get_private_descriptor_type(autofield_info, '_pyi_private_get_type',
-                                                          is_nullable=False)
-            return FieldNode(pk_type)
+        return resolve_model_pk_lookup(api, model_type_info)
 
     field_name = get_actual_field_name_for_lookup_field(lookup, model_type_info)
 
@@ -82,7 +86,7 @@ def resolve_model_lookup(api: CheckerPluginInterface, model_type_info: TypeInfo,
     if field_name.endswith('_id'):
         field_name_without_id = field_name.rstrip('_id')
         foreign_key_field = model_type_info.get(field_name_without_id)
-        if foreign_key_field is not None and helpers.is_foreign_key(foreign_key_field.type):
+        if foreign_key_field is not None and helpers.is_foreign_key_like(foreign_key_field.type):
             # Hack: If field ends with '_id' and there is a model field without the '_id' suffix, then use that field.
             field_node = foreign_key_field
             field_name = field_name_without_id
@@ -92,10 +96,23 @@ def resolve_model_lookup(api: CheckerPluginInterface, model_type_info: TypeInfo,
         raise LookupException(
             f'When resolving lookup "{lookup}", could not determine type for {model_type_info.name()}.{field_name}')
 
-    if helpers.is_foreign_key(field_node_type):
+    if field_node_type.type.fullname() == 'builtins.object':
+        # could be related manager
+        related_manager_type = helpers.get_related_manager_type_from_metadata(model_type_info, field_name, api)
+        if related_manager_type:
+            model_arg = related_manager_type.args[0]
+            if not isinstance(model_arg, Instance):
+                raise LookupException(
+                    f'When resolving lookup "{lookup}", could not determine type '
+                    f'for {model_type_info.name()}.{field_name}')
+
+            return RelatedModelNode(typ=model_arg, is_nullable=False)
+
+    if helpers.is_foreign_key_like(field_node_type):
         field_type = helpers.extract_field_getter_type(field_node_type)
         is_nullable = helpers.is_optional(field_type)
         if is_nullable:
+            # type is always non-optional
             field_type = helpers.make_required(field_type)
 
         if isinstance(field_type, Instance):
@@ -104,24 +121,16 @@ def resolve_model_lookup(api: CheckerPluginInterface, model_type_info: TypeInfo,
             raise LookupException(f"Not an instance for field {field_type} lookup {lookup}")
 
     field_type = helpers.extract_field_getter_type(field_node_type)
-
     if field_type:
         return FieldNode(typ=field_type)
-    else:
-        # Not a Field
-        if field_name == 'id':
-            # If no 'id' field was fouond, use an int
-            return FieldNode(api.named_generic_type('builtins.int', []))
-
-        related_manager_arg = None
-        if field_node_type.type.has_base(helpers.RELATED_MANAGER_CLASS_FULLNAME):
-            related_manager_arg = field_node_type.args[0]
-
-        if related_manager_arg is not None:
-            # Reverse relation
-            return RelatedModelNode(typ=related_manager_arg, is_nullable=True)
-        raise LookupException(
-            f'When resolving lookup "{lookup}", could not determine type for {model_type_info.name()}.{field_name}')
+
+    # Not a Field
+    if field_name == 'id':
+        # If no 'id' field was found, use an int
+        return FieldNode(api.named_generic_type('builtins.int', []))
+
+    raise LookupException(
+        f'When resolving lookup {lookup!r}, could not determine type for {model_type_info.name()}.{field_name}')
 
 
 def get_actual_field_name_for_lookup_field(lookup: str, model_type_info: TypeInfo) -> str: