From 313ca4383f59e62fe75b4746bb12751434570b46 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Feb 2017 13:49:57 +0100 Subject: [PATCH 1/8] Optimize ABC negative cache for generics --- src/typing.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/typing.py b/src/typing.py index d3c9982d..65c5200a 100644 --- a/src/typing.py +++ b/src/typing.py @@ -856,7 +856,7 @@ def _valid_for_check(cls): "or instance checks" % cls) if ( cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] + sys._getframe(2).f_globals['__name__'] not in ['abc', 'functools'] ): raise TypeError("Parameterized generics cannot be used with class " "or instance checks") @@ -872,7 +872,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -887,7 +886,6 @@ def __extrahook__(subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -974,6 +972,7 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super().__new__(cls, name, bases, namespace, _root=True) self.__parameters__ = tvars @@ -982,8 +981,6 @@ def __new__(cls, name, bases, namespace, self.__args__ = tuple(... if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -1002,6 +999,7 @@ def __new__(cls, name, bases, namespace, self.__subclasshook__ = _make_subclasshook(self) if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry + self._abc_cache = extra._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ @@ -1009,6 +1007,36 @@ def __new__(cls, name, bases, namespace, super(GenericMeta, self).__hash__()) return self + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return _gorg(self)._abc_generic_negative_cache + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + _gorg(self)._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return _gorg(self)._abc_generic_negative_cache_version + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + _gorg(self)._abc_generic_negative_cache_version = value + + def __subclasscheck__(self, cls): + _valid_for_check(self) + return super().__subclasscheck__(cls) + def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) From 6382c8a16fe511a80fedb9b8bba79425d1f4810f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Feb 2017 14:53:06 +0100 Subject: [PATCH 2/8] Add also positive cache and registry links --- src/typing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/typing.py b/src/typing.py index 65c5200a..99dfb965 100644 --- a/src/typing.py +++ b/src/typing.py @@ -1000,6 +1000,9 @@ def __new__(cls, name, bases, namespace, if isinstance(extra, abc.ABCMeta): self._abc_registry = extra._abc_registry self._abc_cache = extra._abc_cache + elif origin is not None: + self._abc_registry = _gorg(self)._abc_registry + self._abc_cache = _gorg(self)._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ From 6d1bcf22a082805a472b8d7abe6599f429915689 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Feb 2017 15:20:36 +0100 Subject: [PATCH 3/8] Formatting --- src/typing.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/typing.py b/src/typing.py index 99dfb965..7ebdf17e 100644 --- a/src/typing.py +++ b/src/typing.py @@ -1019,6 +1019,7 @@ def _abc_negative_cache(self): if isinstance(self.__extra__, abc.ABCMeta): return self.__extra__._abc_negative_cache return _gorg(self)._abc_generic_negative_cache + @_abc_negative_cache.setter def _abc_negative_cache(self, value): if isinstance(self.__extra__, abc.ABCMeta): @@ -1030,16 +1031,13 @@ def _abc_negative_cache_version(self): if isinstance(self.__extra__, abc.ABCMeta): return self.__extra__._abc_negative_cache_version return _gorg(self)._abc_generic_negative_cache_version + @_abc_negative_cache_version.setter def _abc_negative_cache_version(self, value): if isinstance(self.__extra__, abc.ABCMeta): self.__extra__._abc_negative_cache_version = value _gorg(self)._abc_generic_negative_cache_version = value - def __subclasscheck__(self, cls): - _valid_for_check(self) - return super().__subclasscheck__(cls) - def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: _get_type_vars(self.__parameters__, tvars) @@ -1138,6 +1136,10 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + _valid_for_check(self) + return super().__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the From c464731f013a691677e7602a22aff2832d84d105 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Feb 2017 15:43:25 +0100 Subject: [PATCH 4/8] Add ABC optimization to PY2 --- python2/typing.py | 43 ++++++++++++++++++++++++++++++++++++++----- src/typing.py | 4 ++-- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/python2/typing.py b/python2/typing.py index 9b08cf49..50b7a54e 100644 --- a/python2/typing.py +++ b/python2/typing.py @@ -935,7 +935,7 @@ def _valid_for_check(cls): "or instance checks" % cls) if ( cls.__origin__ is not None and - sys._getframe(3).f_globals['__name__'] not in ['abc', 'functools'] + sys._getframe(2).f_globals['__name__'] not in ['abc', 'functools'] ): raise TypeError("Parameterized generics cannot be used with class " "or instance checks") @@ -951,7 +951,6 @@ def _make_subclasshook(cls): # Registered classes need not be checked here because # cls and its extra share the same _abc_registry. def __extrahook__(cls, subclass): - _valid_for_check(cls) res = cls.__extra__.__subclasshook__(subclass) if res is not NotImplemented: return res @@ -966,7 +965,6 @@ def __extrahook__(cls, subclass): else: # For non-ABC extras we'll just call issubclass(). def __extrahook__(cls, subclass): - _valid_for_check(cls) if cls.__extra__ and issubclass(subclass, cls.__extra__): return True return NotImplemented @@ -1044,6 +1042,7 @@ def __new__(cls, name, bases, namespace, # remove bare Generic from bases if there are other generic bases if any(isinstance(b, GenericMeta) and b is not Generic for b in bases): bases = tuple(b for b in bases if b is not Generic) + namespace.update({'__origin__': origin, '__extra__': extra}) self = super(GenericMeta, cls).__new__(cls, name, bases, namespace) self.__parameters__ = tvars @@ -1052,8 +1051,6 @@ def __new__(cls, name, bases, namespace, self.__args__ = tuple(Ellipsis if a is _TypingEllipsis else () if a is _TypingEmpty else a for a in args) if args else None - self.__origin__ = origin - self.__extra__ = extra # Speed hack (https://github.com/python/typing/issues/196). self.__next_in_mro__ = _next_in_mro(self) # Preserve base classes on subclassing (__bases__ are type erased now). @@ -1081,6 +1078,38 @@ def __init__(self, *args, **kwargs): super(GenericMeta, self).__init__(*args, **kwargs) if isinstance(self.__extra__, abc.ABCMeta): self._abc_registry = self.__extra__._abc_registry + self._abc_cache = self.__extra__._abc_cache + elif self.__origin__ is not None: + self._abc_registry = self.__origin__._abc_registry + self._abc_cache = self.__origin__._abc_cache + + # _abc_negative_cache and _abc_negative_cache_version + # realised as descriptors, since GenClass[t1, t2, ...] always + # share subclass info with GenClass. + # This is an important memory optimization. + @property + def _abc_negative_cache(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache + return _gorg(self)._abc_generic_negative_cache + + @_abc_negative_cache.setter + def _abc_negative_cache(self, value): + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + _gorg(self)._abc_generic_negative_cache = value + + @property + def _abc_negative_cache_version(self): + if isinstance(self.__extra__, abc.ABCMeta): + return self.__extra__._abc_negative_cache_version + return _gorg(self)._abc_generic_negative_cache_version + + @_abc_negative_cache_version.setter + def _abc_negative_cache_version(self, value): + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + _gorg(self)._abc_generic_negative_cache_version = value def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: @@ -1180,6 +1209,10 @@ def __getitem__(self, params): extra=self.__extra__, orig_bases=self.__orig_bases__) + def __subclasscheck__(self, cls): + _valid_for_check(self) + return super(GenericMeta, self).__subclasscheck__(cls) + def __instancecheck__(self, instance): # Since we extend ABC.__subclasscheck__ and # ABC.__instancecheck__ inlines the cache checking done by the diff --git a/src/typing.py b/src/typing.py index 7ebdf17e..4a01fff7 100644 --- a/src/typing.py +++ b/src/typing.py @@ -1001,8 +1001,8 @@ def __new__(cls, name, bases, namespace, self._abc_registry = extra._abc_registry self._abc_cache = extra._abc_cache elif origin is not None: - self._abc_registry = _gorg(self)._abc_registry - self._abc_cache = _gorg(self)._abc_cache + self._abc_registry = origin._abc_registry + self._abc_cache = origin._abc_cache if origin and hasattr(origin, '__qualname__'): # Fix for Python 3.2. self.__qualname__ = origin.__qualname__ From 423fb755fed7d8519e00c94f2b17847d3aa65eb1 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Wed, 1 Feb 2017 16:52:08 +0100 Subject: [PATCH 5/8] Inline internal helper for more speed --- python2/typing.py | 22 ++++++++-------------- src/typing.py | 22 ++++++++-------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/python2/typing.py b/python2/typing.py index 50b7a54e..ac4cfb2b 100644 --- a/python2/typing.py +++ b/python2/typing.py @@ -928,19 +928,6 @@ def _next_in_mro(cls): return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(2).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -1210,7 +1197,14 @@ def __getitem__(self, params): orig_bases=self.__orig_bases__) def __subclasscheck__(self, cls): - _valid_for_check(self) + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) return super(GenericMeta, self).__subclasscheck__(cls) def __instancecheck__(self, instance): diff --git a/src/typing.py b/src/typing.py index 4a01fff7..8d0bd332 100644 --- a/src/typing.py +++ b/src/typing.py @@ -849,19 +849,6 @@ def _next_in_mro(cls): return next_in_mro -def _valid_for_check(cls): - """An internal helper to prohibit isinstance([1], List[str]) etc.""" - if cls is Generic: - raise TypeError("Class %r cannot be used with class " - "or instance checks" % cls) - if ( - cls.__origin__ is not None and - sys._getframe(2).f_globals['__name__'] not in ['abc', 'functools'] - ): - raise TypeError("Parameterized generics cannot be used with class " - "or instance checks") - - def _make_subclasshook(cls): """Construct a __subclasshook__ callable that incorporates the associated __extra__ class in subclass checks performed @@ -1137,7 +1124,14 @@ def __getitem__(self, params): orig_bases=self.__orig_bases__) def __subclasscheck__(self, cls): - _valid_for_check(self) + if self.__origin__ is not None: + if sys._getframe(1).f_globals['__name__'] not in ['abc', 'functools']: + raise TypeError("Parameterized generics cannot be used with class " + "or instance checks") + return False + if self is Generic: + raise TypeError("Class %r cannot be used with class " + "or instance checks" % self) return super().__subclasscheck__(cls) def __instancecheck__(self, instance): From f998e24e12b360f1bc6618fbc37d28522c85ab1f Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Feb 2017 12:07:24 +0100 Subject: [PATCH 6/8] Fix negative cahce setters --- src/typing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/typing.py b/src/typing.py index 8d0bd332..4d2370d5 100644 --- a/src/typing.py +++ b/src/typing.py @@ -1011,7 +1011,8 @@ def _abc_negative_cache(self): def _abc_negative_cache(self, value): if isinstance(self.__extra__, abc.ABCMeta): self.__extra__._abc_negative_cache = value - _gorg(self)._abc_generic_negative_cache = value + else: + _gorg(self)._abc_generic_negative_cache = value @property def _abc_negative_cache_version(self): @@ -1023,7 +1024,8 @@ def _abc_negative_cache_version(self): def _abc_negative_cache_version(self, value): if isinstance(self.__extra__, abc.ABCMeta): self.__extra__._abc_negative_cache_version = value - _gorg(self)._abc_generic_negative_cache_version = value + else: + _gorg(self)._abc_generic_negative_cache_version = value def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: From 9444b535def9900f34fd48993f57395547716c44 Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Feb 2017 12:23:56 +0100 Subject: [PATCH 7/8] Fix negative cahce setters on PY2 --- python2/typing.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python2/typing.py b/python2/typing.py index ac4cfb2b..8c1f6e86 100644 --- a/python2/typing.py +++ b/python2/typing.py @@ -1084,7 +1084,8 @@ def _abc_negative_cache(self): def _abc_negative_cache(self, value): if isinstance(self.__extra__, abc.ABCMeta): self.__extra__._abc_negative_cache = value - _gorg(self)._abc_generic_negative_cache = value + else: + _gorg(self)._abc_generic_negative_cache = value @property def _abc_negative_cache_version(self): @@ -1096,7 +1097,8 @@ def _abc_negative_cache_version(self): def _abc_negative_cache_version(self, value): if isinstance(self.__extra__, abc.ABCMeta): self.__extra__._abc_negative_cache_version = value - _gorg(self)._abc_generic_negative_cache_version = value + else: + _gorg(self)._abc_generic_negative_cache_version = value def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: From 0ca5c0b0c6ba2047a65ccff733bde5cbe82bcd2b Mon Sep 17 00:00:00 2001 From: Ivan Levkivskyi Date: Thu, 2 Feb 2017 12:41:01 +0100 Subject: [PATCH 8/8] Make negative caches writable only for non-subscripted generics --- python2/typing.py | 18 ++++++++++-------- src/typing.py | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/python2/typing.py b/python2/typing.py index 8c1f6e86..287f645c 100644 --- a/python2/typing.py +++ b/python2/typing.py @@ -1082,10 +1082,11 @@ def _abc_negative_cache(self): @_abc_negative_cache.setter def _abc_negative_cache(self, value): - if isinstance(self.__extra__, abc.ABCMeta): - self.__extra__._abc_negative_cache = value - else: - _gorg(self)._abc_generic_negative_cache = value + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value @property def _abc_negative_cache_version(self): @@ -1095,10 +1096,11 @@ def _abc_negative_cache_version(self): @_abc_negative_cache_version.setter def _abc_negative_cache_version(self, value): - if isinstance(self.__extra__, abc.ABCMeta): - self.__extra__._abc_negative_cache_version = value - else: - _gorg(self)._abc_generic_negative_cache_version = value + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: diff --git a/src/typing.py b/src/typing.py index 4d2370d5..08087e3e 100644 --- a/src/typing.py +++ b/src/typing.py @@ -1009,10 +1009,11 @@ def _abc_negative_cache(self): @_abc_negative_cache.setter def _abc_negative_cache(self, value): - if isinstance(self.__extra__, abc.ABCMeta): - self.__extra__._abc_negative_cache = value - else: - _gorg(self)._abc_generic_negative_cache = value + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache = value + else: + self._abc_generic_negative_cache = value @property def _abc_negative_cache_version(self): @@ -1022,10 +1023,11 @@ def _abc_negative_cache_version(self): @_abc_negative_cache_version.setter def _abc_negative_cache_version(self, value): - if isinstance(self.__extra__, abc.ABCMeta): - self.__extra__._abc_negative_cache_version = value - else: - _gorg(self)._abc_generic_negative_cache_version = value + if self.__origin__ is None: + if isinstance(self.__extra__, abc.ABCMeta): + self.__extra__._abc_negative_cache_version = value + else: + self._abc_generic_negative_cache_version = value def _get_type_vars(self, tvars): if self.__origin__ and self.__parameters__: