Fix typing_extensions to support PEP 560 (#549)

The main fix is rewriting Protocol class and its metaclass.

Author: ilevkivskyi

typing_extensions/src_py3/test_typing_extensions.py

@@ -20,6 +20,9 @@
 import typing
 import typing_extensions
 import collections.abc as collections_abc
+
+PEP_560 = sys.version_info[:3] >= (3, 7, 0)
+
 OLD_GENERICS = False
 try:
     from typing import _type_vars, _next_in_mro, _type_check
@@ -471,7 +474,10 @@ def test_counter_instantiation(self):
         class C(typing_extensions.Counter[T]): ...
         if TYPING_3_5_3:
             self.assertIs(type(C[int]()), C)
-            self.assertEqual(C.__bases__, (typing_extensions.Counter,))
+            if not PEP_560:
+                self.assertEqual(C.__bases__, (typing_extensions.Counter,))
+            else:
+                self.assertEqual(C.__bases__, (collections.Counter, typing.Generic))
 
     def test_counter_subclass_instantiation(self):
 
@@ -818,9 +824,10 @@ def x(self): ...
             self.assertIsSubclass(C, P)
             self.assertIsSubclass(C, PG)
             self.assertIsSubclass(BadP, PG)
-            self.assertIsSubclass(PG[int], PG)
-            self.assertIsSubclass(BadPG[int], P)
-            self.assertIsSubclass(BadPG[T], PG)
+            if not PEP_560:
+                self.assertIsSubclass(PG[int], PG)
+                self.assertIsSubclass(BadPG[int], P)
+                self.assertIsSubclass(BadPG[T], PG)
             with self.assertRaises(TypeError):
                 issubclass(C, PG[T])
             with self.assertRaises(TypeError):
@@ -1043,7 +1050,11 @@ class PR(Protocol, Generic[T, S]):
                 def meth(self): pass
             class P(PR[int, str], Protocol):
                 y = 1
-            self.assertIsSubclass(PR[int, str], PR)
+            if not PEP_560:
+                self.assertIsSubclass(PR[int, str], PR)
+            else:
+                with self.assertRaises(TypeError):
+                    self.assertIsSubclass(PR[int, str], PR)
             self.assertIsSubclass(P, PR)
             with self.assertRaises(TypeError):
                 PR[int]
@@ -1091,7 +1102,9 @@ def test_generic_protocols_repr(self):
             T = TypeVar('T')
             S = TypeVar('S')
             class P(Protocol[T, S]): pass
-            self.assertTrue(repr(P).endswith('P'))
+            # After PEP 560 unsubscripted generics have a standard repr.
+            if not PEP_560:
+                self.assertTrue(repr(P).endswith('P'))
             self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]'))
             self.assertTrue(repr(P[int, str]).endswith('P[int, str]'))
 
@@ -1135,12 +1148,13 @@ def meth(self):
                 self.assertFalse(P._is_runtime_protocol)
             self.assertTrue(PR._is_runtime_protocol)
             self.assertTrue(PG[int]._is_protocol)
-            self.assertEqual(P._get_protocol_attrs(), {'meth'})
-            self.assertEqual(PR._get_protocol_attrs(), {'x'})
-            self.assertEqual(frozenset(PG._get_protocol_attrs()),
-                             frozenset({'x', 'meth'}))
-            self.assertEqual(frozenset(PG[int]._get_protocol_attrs()),
+            self.assertEqual(typing_extensions._get_protocol_attrs(P), {'meth'})
+            self.assertEqual(typing_extensions._get_protocol_attrs(PR), {'x'})
+            self.assertEqual(frozenset(typing_extensions._get_protocol_attrs(PG)),
                              frozenset({'x', 'meth'}))
+            if not PEP_560:
+                self.assertEqual(frozenset(typing_extensions._get_protocol_attrs(PG[int])),
+                                 frozenset({'x', 'meth'}))
 
         def test_no_runtime_deco_on_nominal(self):
             with self.assertRaises(TypeError):

typing_extensions/src_py3/typing_extensions.py

@@ -5,10 +5,19 @@
 import typing
 import collections.abc as collections_abc
 
+# After PEP 560, internal typing API was substantially reworked.
+# This is especially important for Protocol class which uses internal APIs
+# quite extensivelly.
+PEP_560 = sys.version_info[:3] >= (3, 7, 0)
+
 # These are used by Protocol implementation
 # We use internal typing helpers here, but this significantly reduces
 # code duplication. (Also this is only until Protocol is in typing.)
-from typing import GenericMeta, TypingMeta, Generic, Callable, TypeVar, Tuple
+from typing import Generic, Callable, TypeVar, Tuple
+if PEP_560:
+    GenericMeta = TypingMeta = type
+else:
+    from typing import GenericMeta, TypingMeta
 OLD_GENERICS = False
 try:
     from typing import _type_vars, _next_in_mro, _type_check
@@ -729,7 +738,31 @@ def _next_in_mro(cls):
                 next_in_mro = cls.__mro__[i + 1]
         return next_in_mro
 
-if HAVE_PROTOCOLS:
+
+def _get_protocol_attrs(cls):
+    attrs = set()
+    for base in cls.__mro__[:-1]:  # without object
+        if base.__name__ in ('Protocol', 'Generic'):
+            continue
+        annotations = getattr(base, '__annotations__', {})
+        for attr in list(base.__dict__.keys()) + list(annotations.keys()):
+            if (not attr.startswith('_abc_') and attr not in (
+                    '__abstractmethods__', '__annotations__', '__weakref__',
+                    '_is_protocol', '_is_runtime_protocol', '__dict__',
+                    '__args__', '__slots__',
+                    '__next_in_mro__', '__parameters__', '__origin__',
+                    '__orig_bases__', '__extra__', '__tree_hash__',
+                    '__doc__', '__subclasshook__', '__init__', '__new__',
+                    '__module__', '_MutableMapping__marker', '_gorg')):
+                attrs.add(attr)
+    return attrs
+
+
+def _is_callable_members_only(cls):
+    return all(callable(getattr(cls, attr, None)) for attr in _get_protocol_attrs(cls))
+
+
+if HAVE_PROTOCOLS and not PEP_560:
     class _ProtocolMeta(GenericMeta):
         """Internal metaclass for Protocol.
 
@@ -817,8 +850,6 @@ def __init__(cls, *args, **kwargs):
                             base.__origin__ is Generic):
                         raise TypeError('Protocols can only inherit from other'
                                         ' protocols, got %r' % base)
-                cls._callable_members_only = all(callable(getattr(cls, attr, None))
-                                                 for attr in cls._get_protocol_attrs())
 
                 def _no_init(self, *args, **kwargs):
                     if type(self)._is_protocol:
@@ -831,7 +862,7 @@ def _proto_hook(other):
                 if not isinstance(other, type):
                     # Same error as for issubclass(1, int)
                     raise TypeError('issubclass() arg 1 must be a class')
-                for attr in cls._get_protocol_attrs():
+                for attr in _get_protocol_attrs(cls):
                     for base in other.__mro__:
                         if attr in base.__dict__:
                             if base.__dict__[attr] is None:
@@ -850,14 +881,14 @@ def __instancecheck__(self, instance):
             # We need this method for situations where attributes are
             # assigned in __init__.
             if ((not getattr(self, '_is_protocol', False) or
-                    self._callable_members_only) and
+                    _is_callable_members_only(self)) and
                     issubclass(instance.__class__, self)):
                 return True
             if self._is_protocol:
                 if all(hasattr(instance, attr) and
                         (not callable(getattr(self, attr, None)) or
                          getattr(instance, attr) is not None)
-                        for attr in self._get_protocol_attrs()):
+                        for attr in _get_protocol_attrs(self)):
                     return True
             return super(GenericMeta, self).__instancecheck__(instance)
 
@@ -874,32 +905,13 @@ def __subclasscheck__(self, cls):
                 raise TypeError("Instance and class checks can only be used with"
                                 " @runtime protocols")
             if (self.__dict__.get('_is_runtime_protocol', None) and
-                    not self._callable_members_only):
+                    not _is_callable_members_only(self)):
                 if sys._getframe(1).f_globals['__name__'] in ['abc', 'functools', 'typing']:
                     return super(GenericMeta, self).__subclasscheck__(cls)
                 raise TypeError("Protocols with non-method members"
                                 " don't support issubclass()")
             return super(GenericMeta, self).__subclasscheck__(cls)
 
-        def _get_protocol_attrs(self):
-            attrs = set()
-            for base in self.__mro__[:-1]:  # without object
-                if base.__name__ in ('Protocol', 'Generic'):
-                    continue
-                annotations = getattr(base, '__annotations__', {})
-                for attr in list(base.__dict__.keys()) + list(annotations.keys()):
-                    if (not attr.startswith('_abc_') and attr not in (
-                            '__abstractmethods__', '__annotations__', '__weakref__',
-                            '_is_protocol', '_is_runtime_protocol', '__dict__',
-                            '__args__', '__slots__', '_get_protocol_attrs',
-                            '__next_in_mro__', '__parameters__', '__origin__',
-                            '__orig_bases__', '__extra__', '__tree_hash__',
-                            '__doc__', '__subclasshook__', '__init__', '__new__',
-                            '__module__', '_MutableMapping__marker', '_gorg',
-                            '_callable_members_only')):
-                        attrs.add(attr)
-            return attrs
-
         if not OLD_GENERICS:
             @_tp_cache
             def __getitem__(self, params):
@@ -985,6 +997,189 @@ def __new__(cls, *args, **kwds):
         Protocol.__doc__ = Protocol.__doc__.format(bases="Protocol, Generic[T]" if
                                                    OLD_GENERICS else "Protocol[T]")
 
+
+elif PEP_560:
+    from typing import _type_check, _GenericAlias, _collect_type_vars
+
+    class _ProtocolMeta(abc.ABCMeta):
+        # This metaclass is a bit unfortunate and exists only because of the lack
+        # of __instancehook__.
+        def __instancecheck__(cls, instance):
+            # We need this method for situations where attributes are
+            # assigned in __init__.
+            if ((not getattr(cls, '_is_protocol', False) or
+                    _is_callable_members_only(cls)) and
+                    issubclass(instance.__class__, cls)):
+                return True
+            if cls._is_protocol:
+                if all(hasattr(instance, attr) and
+                        (not callable(getattr(cls, attr, None)) or
+                         getattr(instance, attr) is not None)
+                        for attr in _get_protocol_attrs(cls)):
+                    return True
+            return super().__instancecheck__(instance)
+
+
+    class Protocol(metaclass=_ProtocolMeta):
+        # There is quite a lot of overlapping code with typing.Generic.
+        # Unfortunately it is hard to avoid this while these live in two different modules.
+        # The duplicated code will be removed when Protocol is moved to typing.
+        """Base class for protocol classes. Protocol classes are defined as::
+
+            class Proto(Protocol):
+                def meth(self) -> int:
+                    ...
+
+        Such classes are primarily used with static type checkers that recognize
+        structural subtyping (static duck-typing), for example::
+
+            class C:
+                def meth(self) -> int:
+                    return 0
+
+            def func(x: Proto) -> int:
+                return x.meth()
+
+            func(C())  # Passes static type check
+
+        See PEP 544 for details. Protocol classes decorated with
+        @typing_extensions.runtime act as simple-minded runtime protocol that checks
+        only the presence of given attributes, ignoring their type signatures.
+
+        Protocol classes can be generic, they are defined as::
+
+            class GenProto(Protocol[T]):
+                def meth(self) -> T:
+                    ...
+        """
+        __slots__ = ()
+        _is_protocol = True
+
+        def __new__(cls, *args, **kwds):
+            if cls is Protocol:
+                raise TypeError("Type Protocol cannot be instantiated; "
+                                "it can only be used as a base class")
+            return super().__new__(cls)
+
+        @_tp_cache
+        def __class_getitem__(cls, params):
+            if not isinstance(params, tuple):
+                params = (params,)
+            if not params and cls is not Tuple:
+                raise TypeError(
+                    "Parameter list to {}[...] cannot be empty".format(cls.__qualname__))
+            msg = "Parameters to generic types must be types."
+            params = tuple(_type_check(p, msg) for p in params)
+            if cls is Protocol:
+                # Generic can only be subscripted with unique type variables.
+                if not all(isinstance(p, TypeVar) for p in params):
+                    i = 0
+                    while isinstance(params[i], TypeVar):
+                        i += 1
+                    raise TypeError(
+                        "Parameters to Protocol[...] must all be type variables."
+                        " Parameter {} is {}".format(i + 1, params[i]))
+                if len(set(params)) != len(params):
+                    raise TypeError(
+                        "Parameters to Protocol[...] must all be unique")
+            else:
+                # Subscripting a regular Generic subclass.
+                _check_generic(cls, params)
+            return _GenericAlias(cls, params)
+
+        def __init_subclass__(cls, *args, **kwargs):
+            tvars = []
+            if '__orig_bases__' in cls.__dict__:
+                error = Generic in cls.__orig_bases__
+            else:
+                error = Generic in cls.__bases__
+            if error:
+                raise TypeError("Cannot inherit from plain Generic")
+            if '__orig_bases__' in cls.__dict__:
+                tvars = _collect_type_vars(cls.__orig_bases__)
+                # Look for Generic[T1, ..., Tn] or Protocol[T1, ..., Tn].
+                # If found, tvars must be a subset of it.
+                # If not found, tvars is it.
+                # Also check for and reject plain Generic,
+                # and reject multiple Generic[...] and/or Protocol[...].
+                gvars = None
+                for base in cls.__orig_bases__:
+                    if (isinstance(base, _GenericAlias) and
+                            base.__origin__ in (Generic, Protocol)):
+                        # for error messages
+                        the_base = 'Generic' if base.__origin__ is Generic else 'Protocol'
+                        if gvars is not None:
+                            raise TypeError(
+                                "Cannot inherit from Generic[...]"
+                                " and/or Protocol[...] multiple types.")
+                        gvars = base.__parameters__
+                if gvars is None:
+                    gvars = tvars
+                else:
+                    tvarset = set(tvars)
+                    gvarset = set(gvars)
+                    if not tvarset <= gvarset:
+                        s_vars = ', '.join(str(t) for t in tvars if t not in gvarset)
+                        s_args = ', '.join(str(g) for g in gvars)
+                        raise TypeError("Some type variables ({}) are"
+                                        " not listed in {}[{}]".format(s_vars, the_base, s_args))
+                    tvars = gvars
+            cls.__parameters__ = tuple(tvars)
+
+            # Determine if this is a protocol or a concrete subclass.
+            if not cls.__dict__.get('_is_protocol', None):
+                cls._is_protocol = any(b is Protocol for b in cls.__bases__)
+
+            # Set (or override) the protocol subclass hook.
+            def _proto_hook(other):
+                if not cls.__dict__.get('_is_protocol', None):
+                    return NotImplemented
+                if not getattr(cls, '_is_runtime_protocol', False):
+                    if sys._getframe(2).f_globals['__name__'] in ['abc', 'functools']:
+                        return NotImplemented
+                    raise TypeError("Instance and class checks can only be used with"
+                                    " @runtime protocols")
+                if not _is_callable_members_only(cls):
+                    if sys._getframe(2).f_globals['__name__'] in ['abc', 'functools']:
+                        return NotImplemented
+                    raise TypeError("Protocols with non-method members"
+                                    " don't support issubclass()")
+                if not isinstance(other, type):
+                    # Same error as for issubclass(1, int)
+                    raise TypeError('issubclass() arg 1 must be a class')
+                for attr in _get_protocol_attrs(cls):
+                    for base in other.__mro__:
+                        if attr in base.__dict__:
+                            if base.__dict__[attr] is None:
+                                return NotImplemented
+                            break
+                        if (attr in getattr(base, '__annotations__', {}) and
+                                isinstance(other, _ProtocolMeta) and other._is_protocol):
+                            break
+                    else:
+                        return NotImplemented
+                return True
+            if '__subclasshook__' not in cls.__dict__:
+                cls.__subclasshook__ = _proto_hook
+
+            # We have nothing more to do for non-protocols.
+            if not cls._is_protocol:
+                return
+
+            # Check consistency of bases.
+            for base in cls.__bases__:
+                if not (base in (object, Generic, Callable) or
+                        isinstance(base, _ProtocolMeta) and base._is_protocol):
+                    raise TypeError('Protocols can only inherit from other'
+                                    ' protocols, got %r' % base)
+
+            def _no_init(self, *args, **kwargs):
+                if type(self)._is_protocol:
+                    raise TypeError('Protocols cannot be instantiated')
+            cls.__init__ = _no_init
+
+
+if HAVE_PROTOCOLS:
     def runtime(cls):
         """Mark a protocol class as a runtime protocol, so that it
         can be used with isinstance() and issubclass(). Raise TypeError