From fba951e447d74ce38fb019364568503c7b054dc0 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 8 Nov 2017 13:53:16 +0100 Subject: [PATCH 01/19] Add all the files which pytype fails on during import. All of these files can not be used with pytype at HEAD. They fail during import with various error messages. --- tests/pytype_blacklist.txt | 110 ++++++++++++++++++++++++++++++++++++- 1 file changed, 107 insertions(+), 3 deletions(-) diff --git a/tests/pytype_blacklist.txt b/tests/pytype_blacklist.txt index 094130b4d73d..75cd963270b8 100644 --- a/tests/pytype_blacklist.txt +++ b/tests/pytype_blacklist.txt @@ -1,13 +1,117 @@ -# Pytype blacklist. Files will not be tested with pytype. +# Pytype blacklist. Files will not be tested with pytype. Notice that the parse +# test will work for some of the files. Loading the files will fail, but is not +# tested at the moment. +# This file is also consumed by a bazel (https://github.com/bazelbuild/bazel) +# rule in another project. Therefore it needs to end in bzl and be valid bazel. +# The pytype_test.py in typeshed sees this file as a text file. -# pytype has its own version of these files, and thus doesn't mind if it -# can't parse the typeshed version: +# This list was generated by importing the files with either python 2.7 or 3.6, +# depending on their directory. +# Files in 2and3 are blacklisted if they fail with at least one of the versions. + +# pytype has its own version of these files. Which are imported by default, +# importing the same definitions again will result in name collisions. stdlib/2/__builtin__.pyi stdlib/2/builtins.pyi stdlib/2/typing.pyi stdlib/3/builtins.pyi stdlib/3/typing.pyi +# PyType does not allow classes to inherit from an Any constant. +stdlib/2/UserList.pyi +stdlib/2/UserString.pyi +stdlib/2/multiprocessing/__init__.pyi +stdlib/2/multiprocessing/util.pyi +stdlib/2/shelve.pyi +stdlib/2/sqlite3/__init__.pyi +stdlib/2/sqlite3/dbapi2.pyi +stdlib/2and3/logging/handlers.pyi +stdlib/3.4/asyncio/subprocess.pyi +stdlib/3/fileinput.pyi +stdlib/3.4/asyncio/protocols.pyi +stdlib/2and3/logging/handlers.pyi +stdlib/3.4/asyncio/tasks.pyi +stdlib/3/sqlite3/dbapi2.pyi +stdlib/3.4/asyncio/__init__.pyi +stdlib/3/sqlite3/__init__.pyi +stdlib/3.4/asyncio/events.pyi +stdlib/3/shelve.pyi + +# Non resolvable imports +# imports non existing "builtins" module. +stdlib/2/os/__init__.pyi +# imports enum, which is undefined in Py2.7 +stdlib/2and3/plistlib.pyi + +# Unresolved dependencies +# BadDependencyError: Can't find pyi for 'RegexFlag', referenced from 're' +stdlib/3/re.pyi +# BadDependencyError: Can't find pyi for '4.asyncio.coroutines', referenced +# from '4.asyncio.locks' +stdlib/3.4/asyncio/locks.pyi +# BadDependencyError: Can't find pyi for '4.asyncio.futures', referenced from +# '4.asyncio.queues' +stdlib/3.4/asyncio/queues.pyi +# BadDependencyError: Can't find pyi for '4.asyncio.events', referenced from +# '4.asyncio.futures' +stdlib/3.4/asyncio/futures.pyi +# BadDependencyError: Can't find pyi for '4.asyncio.protocols', referenced +# from '4.asyncio.streams' +stdlib/3.4/asyncio/streams.pyi +# BadDependencyError: Can't find pyi for 'PlistFormat', referenced from +# 'plistlib' +stdlib/2and3/plistlib.pyi +# BadDependencyError: No Generator in module collections, referenced from +# 'collections.abc' +stdlib/3/collections/abc.pyi +# SymbolLookupError: Couldn't find ConnectionResetError in http.client +stdlib/3/http/client.pyi +# SymbolLookupError: Couldn't find AST in ast +stdlib/3/ast.pyi +# BadDependencyError: Duplicate top level items: +# 'concurrent.futures._base.Future, concurrent.futures._base.Future, +# concurrent.futures._base.Future', referenced from +# 'concurrent.futures.__init__' +stdlib/3/concurrent/futures/__init__.pyi +# BadDependencyError: Unreplaced NamedType: 'importlib.util' +stdlib/3/importlib/__init__.pyi +# BadDependencyError: No ChainMap in module typing, referenced from +# 'collections.__init__' +stdlib/3/collections/__init__.pyi +# BadDependencyError: Can't find pyi for 'Sigmasks', referenced from 'signal' +stdlib/3/signal.pyi +# SymbolLookupError: Couldn't find ConnectionResetError in http.client +stdlib/3/http/cookiejar.pyi +stdlib/3/urllib/request.pyi +# BadDependencyError: No _wrap_close in module os, referenced from 'pipes' +stdlib/3/pipes.pyi +# BadDependencyError: No stat_result in module os, referenced from 'posix' +stdlib/3/posix.pyi + + +# Conflicting values +# ContainerError: Conflicting values for TypeVar +# typing.ContextManager._T: multiprocessing.pool.Pool, +# multiprocessing.pool.ThreadPool +stdlib/3/multiprocessing/pool.pyi +stdlib/3/multiprocessing/context.pyi +stdlib/3/multiprocessing/process.pyi +stdlib/3/multiprocessing/__init__.pyi +stdlib/3/multiprocessing/synchronize.pyi + # pytype doesn't yet support aliases with implicit type parameters # (e.g., here, FutureT = Future[T]) stdlib/3.4/asyncio/tasks.pyi + +# ParseError: syntax error, unexpected ')', expecting NAME or NUMBER or +# ELLIPSIS +stdlib/2and3/locale.pyi +stdlib/3/io.pyi +stdlib/3/calendar.pyi +stdlib/3/fcntl.pyi +stdlib/3/gzip.pyi +stdlib/3/os/path.pyi +stdlib/3/_compression.pyi +stdlib/3/tokenize.pyi +stdlib/3/os/__init__.pyi +stdlib/3/subprocess.pyi From 98215f7d34a26a6f6866938428f5853b80cd8c9e Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 8 Nov 2017 17:42:41 +0100 Subject: [PATCH 02/19] Update pytype_blacklist.txt --- tests/pytype_blacklist.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytype_blacklist.txt b/tests/pytype_blacklist.txt index 75cd963270b8..8a0d7f00be98 100644 --- a/tests/pytype_blacklist.txt +++ b/tests/pytype_blacklist.txt @@ -103,8 +103,7 @@ stdlib/3/multiprocessing/synchronize.pyi # (e.g., here, FutureT = Future[T]) stdlib/3.4/asyncio/tasks.pyi -# ParseError: syntax error, unexpected ')', expecting NAME or NUMBER or -# ELLIPSIS +# BadDependencyError: No SupportsBytes in module typing, referenced from 'builtins' stdlib/2and3/locale.pyi stdlib/3/io.pyi stdlib/3/calendar.pyi From 7242b76f6c7d69871323b8d0c38b8e4ab34795e1 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Thu, 9 Nov 2017 15:01:56 +0100 Subject: [PATCH 03/19] PyType will ignore the PreconditionErrors in the latest version. --- tests/pytype_blacklist.txt | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/tests/pytype_blacklist.txt b/tests/pytype_blacklist.txt index 8a0d7f00be98..400628657715 100644 --- a/tests/pytype_blacklist.txt +++ b/tests/pytype_blacklist.txt @@ -17,25 +17,12 @@ stdlib/2/typing.pyi stdlib/3/builtins.pyi stdlib/3/typing.pyi -# PyType does not allow classes to inherit from an Any constant. -stdlib/2/UserList.pyi -stdlib/2/UserString.pyi -stdlib/2/multiprocessing/__init__.pyi -stdlib/2/multiprocessing/util.pyi -stdlib/2/shelve.pyi -stdlib/2/sqlite3/__init__.pyi -stdlib/2/sqlite3/dbapi2.pyi -stdlib/2and3/logging/handlers.pyi +# AttributeError: 'UnionType' object has no attribute 'name' stdlib/3.4/asyncio/subprocess.pyi stdlib/3/fileinput.pyi stdlib/3.4/asyncio/protocols.pyi -stdlib/2and3/logging/handlers.pyi -stdlib/3.4/asyncio/tasks.pyi -stdlib/3/sqlite3/dbapi2.pyi stdlib/3.4/asyncio/__init__.pyi -stdlib/3/sqlite3/__init__.pyi stdlib/3.4/asyncio/events.pyi -stdlib/3/shelve.pyi # Non resolvable imports # imports non existing "builtins" module. From 81f1d09e13347dc202ff8623cdacdf0985b316d6 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Fri, 10 Nov 2017 12:49:56 +0100 Subject: [PATCH 04/19] Update pytype_blacklist.txt --- tests/pytype_blacklist.txt | 151 ++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 85 deletions(-) diff --git a/tests/pytype_blacklist.txt b/tests/pytype_blacklist.txt index 400628657715..11ecf7c11ae4 100644 --- a/tests/pytype_blacklist.txt +++ b/tests/pytype_blacklist.txt @@ -1,103 +1,84 @@ -# Pytype blacklist. Files will not be tested with pytype. Notice that the parse -# test will work for some of the files. Loading the files will fail, but is not -# tested at the moment. -# This file is also consumed by a bazel (https://github.com/bazelbuild/bazel) -# rule in another project. Therefore it needs to end in bzl and be valid bazel. -# The pytype_test.py in typeshed sees this file as a text file. +# Pytype blacklist. Files will not be tested with pytype. -# This list was generated by importing the files with either python 2.7 or 3.6, -# depending on their directory. -# Files in 2and3 are blacklisted if they fail with at least one of the versions. - -# pytype has its own version of these files. Which are imported by default, -# importing the same definitions again will result in name collisions. +# pytype has its own version of these files, and thus doesn't mind if it +# can't parse the typeshed version: stdlib/2/__builtin__.pyi stdlib/2/builtins.pyi stdlib/2/typing.pyi stdlib/3/builtins.pyi stdlib/3/typing.pyi -# AttributeError: 'UnionType' object has no attribute 'name' -stdlib/3.4/asyncio/subprocess.pyi -stdlib/3/fileinput.pyi -stdlib/3.4/asyncio/protocols.pyi -stdlib/3.4/asyncio/__init__.pyi -stdlib/3.4/asyncio/events.pyi +# pytype doesn't yet support aliases with implicit type parameters +# (e.g., here, FutureT = Future[T]) +stdlib/3.4/asyncio/tasks.pyi -# Non resolvable imports -# imports non existing "builtins" module. +# Below files can not be loaded by PyType without an error. +# Notice that the current travis configuration loads these files with Python 2.7. +stdlib/2/ast.pyi +stdlib/2/macpath.pyi +stdlib/2/ntpath.pyi stdlib/2/os/__init__.pyi -# imports enum, which is undefined in Py2.7 -stdlib/2and3/plistlib.pyi - -# Unresolved dependencies -# BadDependencyError: Can't find pyi for 'RegexFlag', referenced from 're' -stdlib/3/re.pyi -# BadDependencyError: Can't find pyi for '4.asyncio.coroutines', referenced -# from '4.asyncio.locks' +stdlib/2/os2emxpath.pyi +stdlib/2/posixpath.pyi +stdlib/2/sqlite3/__init__.pyi +stdlib/2and3/lib2to3/pgen2/tokenize.pyi +stdlib/2and3/xml/etree/cElementTree.pyi +stdlib/3.4/asyncio/__init__.pyi +stdlib/3.4/asyncio/events.pyi +stdlib/3.4/asyncio/futures.pyi stdlib/3.4/asyncio/locks.pyi -# BadDependencyError: Can't find pyi for '4.asyncio.futures', referenced from -# '4.asyncio.queues' +stdlib/3.4/asyncio/protocols.pyi stdlib/3.4/asyncio/queues.pyi -# BadDependencyError: Can't find pyi for '4.asyncio.events', referenced from -# '4.asyncio.futures' -stdlib/3.4/asyncio/futures.pyi -# BadDependencyError: Can't find pyi for '4.asyncio.protocols', referenced -# from '4.asyncio.streams' stdlib/3.4/asyncio/streams.pyi -# BadDependencyError: Can't find pyi for 'PlistFormat', referenced from -# 'plistlib' -stdlib/2and3/plistlib.pyi -# BadDependencyError: No Generator in module collections, referenced from -# 'collections.abc' -stdlib/3/collections/abc.pyi -# SymbolLookupError: Couldn't find ConnectionResetError in http.client -stdlib/3/http/client.pyi -# SymbolLookupError: Couldn't find AST in ast +stdlib/3.4/asyncio/subprocess.pyi +stdlib/3.5/zipapp.pyi +stdlib/3/_compression.pyi stdlib/3/ast.pyi -# BadDependencyError: Duplicate top level items: -# 'concurrent.futures._base.Future, concurrent.futures._base.Future, -# concurrent.futures._base.Future', referenced from -# 'concurrent.futures.__init__' -stdlib/3/concurrent/futures/__init__.pyi -# BadDependencyError: Unreplaced NamedType: 'importlib.util' -stdlib/3/importlib/__init__.pyi -# BadDependencyError: No ChainMap in module typing, referenced from -# 'collections.__init__' stdlib/3/collections/__init__.pyi -# BadDependencyError: Can't find pyi for 'Sigmasks', referenced from 'signal' -stdlib/3/signal.pyi -# SymbolLookupError: Couldn't find ConnectionResetError in http.client +stdlib/3/concurrent/futures/__init__.pyi +stdlib/3/concurrent/futures/process.pyi +stdlib/3/concurrent/futures/thread.pyi +stdlib/3/curses/__init__.pyi +stdlib/3/email/__init__.pyi +stdlib/3/email/contentmanager.pyi +stdlib/3/email/encoders.pyi +stdlib/3/email/feedparser.pyi +stdlib/3/email/generator.pyi +stdlib/3/email/header.pyi +stdlib/3/email/headerregistry.pyi +stdlib/3/email/iterators.pyi +stdlib/3/email/message.pyi +stdlib/3/email/mime/base.pyi +stdlib/3/email/mime/message.pyi +stdlib/3/email/mime/multipart.pyi +stdlib/3/email/parser.pyi +stdlib/3/email/policy.pyi +stdlib/3/email/utils.pyi +stdlib/3/gzip.pyi +stdlib/3/html/parser.pyi +stdlib/3/http/client.pyi stdlib/3/http/cookiejar.pyi -stdlib/3/urllib/request.pyi -# BadDependencyError: No _wrap_close in module os, referenced from 'pipes' -stdlib/3/pipes.pyi -# BadDependencyError: No stat_result in module os, referenced from 'posix' -stdlib/3/posix.pyi - - -# Conflicting values -# ContainerError: Conflicting values for TypeVar -# typing.ContextManager._T: multiprocessing.pool.Pool, -# multiprocessing.pool.ThreadPool -stdlib/3/multiprocessing/pool.pyi -stdlib/3/multiprocessing/context.pyi -stdlib/3/multiprocessing/process.pyi +stdlib/3/http/server.pyi +stdlib/3/imp.pyi +stdlib/3/importlib/__init__.pyi +stdlib/3/importlib/abc.pyi +stdlib/3/importlib/machinery.pyi +stdlib/3/io.pyi +stdlib/3/json/__init__.pyi stdlib/3/multiprocessing/__init__.pyi +stdlib/3/multiprocessing/context.pyi +stdlib/3/multiprocessing/managers.pyi +stdlib/3/multiprocessing/pool.pyi stdlib/3/multiprocessing/synchronize.pyi - -# pytype doesn't yet support aliases with implicit type parameters -# (e.g., here, FutureT = Future[T]) -stdlib/3.4/asyncio/tasks.pyi - -# BadDependencyError: No SupportsBytes in module typing, referenced from 'builtins' -stdlib/2and3/locale.pyi -stdlib/3/io.pyi -stdlib/3/calendar.pyi -stdlib/3/fcntl.pyi -stdlib/3/gzip.pyi -stdlib/3/os/path.pyi -stdlib/3/_compression.pyi -stdlib/3/tokenize.pyi stdlib/3/os/__init__.pyi -stdlib/3/subprocess.pyi +stdlib/3/pipes.pyi +stdlib/3/smtplib.pyi +stdlib/3/sqlite3/__init__.pyi +stdlib/3/sre_parse.pyi +stdlib/3/time.pyi +stdlib/3/tkinter/__init__.pyi +stdlib/3/tkinter/ttk.pyi +stdlib/3/tokenize.pyi +stdlib/3/types.pyi +stdlib/3/urllib/error.pyi +stdlib/3/urllib/request.pyi From 846102ce6812f394b9c60e68514048f32ca1772c Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Fri, 10 Nov 2017 12:58:05 +0100 Subject: [PATCH 05/19] Update the pytype_test script to use pytype and not pytd This change will make the test script actually load the files and all dependencies of it, instead of just verifying that it can be parsed. --- tests/pytype_test.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 3d6f3639793a..4e6d3050f1ec 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -12,6 +12,7 @@ import sys import argparse import subprocess +import tempfile import collections parser = argparse.ArgumentParser(description="Pytype tests.") @@ -42,7 +43,15 @@ def load_blacklist(): class PytdRun(object): def __init__(self, args, dry_run=False): - self.args = args + temp = tempfile.NamedTemporaryFile(delete=False) + temp.close() # Windows compat. + + self.args = [] + self.args.extend([ + '--convert-to-pickle=%s' % temp.name, + ]) + self.args.extend(args) + self.dry_run = dry_run self.results = None @@ -50,9 +59,10 @@ def __init__(self, args, dry_run=False): self.results = (0, "", "") else: self.proc = subprocess.Popen( - ["pytd"] + args, + ["pytype"] + self.args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + os.remove(temp.name) def communicate(self): if self.results: @@ -97,7 +107,7 @@ def pytype_test(args): runs += 1 if code: - print("pytd error processing \"%s\":" % test_run.args[0]) + print("pytd error processing \"%s\":" % test_run.args[1]) print(stderr) errors += 1 From 88cccb76c6405b33d1d69780c9d472cc39c4f70c Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Fri, 10 Nov 2017 13:14:38 +0100 Subject: [PATCH 06/19] Update pytype_test.py --- tests/pytype_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 4e6d3050f1ec..1b9700b32649 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -47,9 +47,7 @@ def __init__(self, args, dry_run=False): temp.close() # Windows compat. self.args = [] - self.args.extend([ - '--convert-to-pickle=%s' % temp.name, - ]) + self.args.append('--convert-to-pickle=%s' % temp.name) self.args.extend(args) self.dry_run = dry_run From cce667805b65001947a39deaa31157691f3dee56 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 10:55:51 +0100 Subject: [PATCH 07/19] Add parse only comments and remove some files. This list was generated by running pytype with more options. 1: typeshed-location was pointed to the current typeshed instead of the one shipped with pytype. 2: --module-name= was provided as a parameter. Without this parameter wildcard imports "from _ast import *" misbehave. Files with a "# parse only" comment will only be parsed and not loaded. --- tests/pytype_blacklist.txt | 132 ++++++++++++++++++------------------- 1 file changed, 64 insertions(+), 68 deletions(-) diff --git a/tests/pytype_blacklist.txt b/tests/pytype_blacklist.txt index 11ecf7c11ae4..912dadf8d0fe 100644 --- a/tests/pytype_blacklist.txt +++ b/tests/pytype_blacklist.txt @@ -12,73 +12,69 @@ stdlib/3/typing.pyi # (e.g., here, FutureT = Future[T]) stdlib/3.4/asyncio/tasks.pyi +# builtins not found +stdlib/2/os/__init__.pyi + # Below files can not be loaded by PyType without an error. # Notice that the current travis configuration loads these files with Python 2.7. -stdlib/2/ast.pyi -stdlib/2/macpath.pyi -stdlib/2/ntpath.pyi -stdlib/2/os/__init__.pyi -stdlib/2/os2emxpath.pyi -stdlib/2/posixpath.pyi -stdlib/2/sqlite3/__init__.pyi -stdlib/2and3/lib2to3/pgen2/tokenize.pyi -stdlib/2and3/xml/etree/cElementTree.pyi -stdlib/3.4/asyncio/__init__.pyi -stdlib/3.4/asyncio/events.pyi -stdlib/3.4/asyncio/futures.pyi -stdlib/3.4/asyncio/locks.pyi -stdlib/3.4/asyncio/protocols.pyi -stdlib/3.4/asyncio/queues.pyi -stdlib/3.4/asyncio/streams.pyi -stdlib/3.4/asyncio/subprocess.pyi -stdlib/3.5/zipapp.pyi -stdlib/3/_compression.pyi -stdlib/3/ast.pyi -stdlib/3/collections/__init__.pyi -stdlib/3/concurrent/futures/__init__.pyi -stdlib/3/concurrent/futures/process.pyi -stdlib/3/concurrent/futures/thread.pyi -stdlib/3/curses/__init__.pyi -stdlib/3/email/__init__.pyi -stdlib/3/email/contentmanager.pyi -stdlib/3/email/encoders.pyi -stdlib/3/email/feedparser.pyi -stdlib/3/email/generator.pyi -stdlib/3/email/header.pyi -stdlib/3/email/headerregistry.pyi -stdlib/3/email/iterators.pyi -stdlib/3/email/message.pyi -stdlib/3/email/mime/base.pyi -stdlib/3/email/mime/message.pyi -stdlib/3/email/mime/multipart.pyi -stdlib/3/email/parser.pyi -stdlib/3/email/policy.pyi -stdlib/3/email/utils.pyi -stdlib/3/gzip.pyi -stdlib/3/html/parser.pyi -stdlib/3/http/client.pyi -stdlib/3/http/cookiejar.pyi -stdlib/3/http/server.pyi -stdlib/3/imp.pyi -stdlib/3/importlib/__init__.pyi -stdlib/3/importlib/abc.pyi -stdlib/3/importlib/machinery.pyi -stdlib/3/io.pyi -stdlib/3/json/__init__.pyi -stdlib/3/multiprocessing/__init__.pyi -stdlib/3/multiprocessing/context.pyi -stdlib/3/multiprocessing/managers.pyi -stdlib/3/multiprocessing/pool.pyi -stdlib/3/multiprocessing/synchronize.pyi -stdlib/3/os/__init__.pyi -stdlib/3/pipes.pyi -stdlib/3/smtplib.pyi -stdlib/3/sqlite3/__init__.pyi -stdlib/3/sre_parse.pyi -stdlib/3/time.pyi -stdlib/3/tkinter/__init__.pyi -stdlib/3/tkinter/ttk.pyi -stdlib/3/tokenize.pyi -stdlib/3/types.pyi -stdlib/3/urllib/error.pyi -stdlib/3/urllib/request.pyi +# The files will be parsed but not loaded to verify them. +stdlib/3.4/asyncio/__init__.pyi # parse only +stdlib/3.4/asyncio/events.pyi # parse only +stdlib/3.4/asyncio/futures.pyi # parse only +stdlib/3.4/asyncio/locks.pyi # parse only +stdlib/3.4/asyncio/protocols.pyi # parse only +stdlib/3.4/asyncio/queues.pyi # parse only +stdlib/3.4/asyncio/streams.pyi # parse only +stdlib/3.4/asyncio/subprocess.pyi # parse only +stdlib/3.5/zipapp.pyi # parse only +stdlib/3/_compression.pyi # parse only +stdlib/3/ast.pyi # parse only +stdlib/3/collections/__init__.pyi # parse only.pyi +stdlib/3/concurrent/futures/__init__.pyi # parse only +stdlib/3/concurrent/futures/process.pyi # parse only +stdlib/3/concurrent/futures/thread.pyi # parse only +stdlib/3/curses/__init__.pyi # parse only +stdlib/3/email/__init__.pyi # parse only +stdlib/3/email/contentmanager.pyi # parse only +stdlib/3/email/encoders.pyi # parse only +stdlib/3/email/feedparser.pyi # parse only +stdlib/3/email/generator.pyi # parse only +stdlib/3/email/header.pyi # parse only +stdlib/3/email/headerregistry.pyi # parse only +stdlib/3/email/iterators.pyi # parse only +stdlib/3/email/message.pyi # parse only +stdlib/3/email/mime/base.pyi # parse only +stdlib/3/email/mime/message.pyi # parse only +stdlib/3/email/mime/multipart.pyi # parse only +stdlib/3/email/parser.pyi # parse only +stdlib/3/email/policy.pyi # parse only +stdlib/3/email/utils.pyi # parse only +stdlib/3/gzip.pyi # parse only +stdlib/3/html/parser.pyi # parse only +stdlib/3/http/client.pyi # parse only +stdlib/3/http/cookiejar.pyi # parse only +stdlib/3/http/server.pyi # parse only +stdlib/3/imp.pyi # parse only +stdlib/3/importlib/__init__.pyi # parse only +stdlib/3/importlib/abc.pyi # parse only +stdlib/3/importlib/machinery.pyi # parse only +stdlib/3/io.pyi # parse only +stdlib/3/json/__init__.pyi # parse only +stdlib/3/multiprocessing/__init__.pyi # parse only +stdlib/3/multiprocessing/context.pyi # parse only +stdlib/3/multiprocessing/managers.pyi # parse only +stdlib/3/multiprocessing/pool.pyi # parse only +stdlib/3/multiprocessing/process.pyi # parse only +stdlib/3/multiprocessing/synchronize.pyi # parse only +stdlib/3/os/__init__.pyi # parse only +stdlib/3/pipes.pyi # parse only +stdlib/3/smtplib.pyi # parse only +stdlib/3/sqlite3/__init__.pyi # parse only +stdlib/3/sre_parse.pyi # parse only +stdlib/3/time.pyi # parse only +stdlib/3/tkinter/__init__.pyi # parse only +stdlib/3/tkinter/ttk.pyi # parse only +stdlib/3/tokenize.pyi # parse only +stdlib/3/types.pyi # parse only +stdlib/3/urllib/error.pyi # parse only +stdlib/3/urllib/request.pyi # parse only From edc721ac9b20ee5b6965b0149a9afdb5f0a6fc47 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 10:57:35 +0100 Subject: [PATCH 08/19] Add a switch to run pytd or pytype This change adds the ability to either run pytype or pytd, based on a "# parse only" comment in pytype_blacklist.pytd --- tests/pytype_test.py | 87 ++++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 1b9700b32649..ea9a456bcd20 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -4,7 +4,15 @@ Depends on mypy and pytype being installed. If pytype is installed: - 1. For every pyi, run "pytd " in a separate process + 1. For every pyi, do nothing if it is in pytype_blacklist.txt. + 2. If the blacklist line has a "# parse only" comment run + "pytd " in a separate process. + 3. If the file is not in the blacklist run + "pytype --typeshed-location=typeshed_location --module-name=foo \ + --convert-to-pickle=tmp_file . +Option two will parse the file, mostly syntactical correctness. Option three +will load the file and all the builtins, typeshed dependencies. This will +also discover incorrect usage of imported modules. """ import os @@ -32,23 +40,27 @@ def main(): print("--- nothing to do; exit 1 ---") sys.exit(1) +_PARSE_ONLY_REGEX = r"\s*#\s*parse only\s*" def load_blacklist(): filename = os.path.join(os.path.dirname(__file__), "pytype_blacklist.txt") - regex = r"^\s*([^\s#]+)\s*(?:#.*)?$" - + regex = r"^\s*([^\s#]+(?:%s)?)\s*(?:#.*)?$" % _PARSE_ONLY_REGEX with open(filename) as f: - return re.findall(regex, f.read(), flags=re.M) - - -class PytdRun(object): - def __init__(self, args, dry_run=False): - temp = tempfile.NamedTemporaryFile(delete=False) - temp.close() # Windows compat. - - self.args = [] - self.args.append('--convert-to-pickle=%s' % temp.name) - self.args.extend(args) + lines = re.findall(regex, f.read(), flags=re.M) + blacklist = [] + parse_only = [] + parse_only_re = re.compile(_PARSE_ONLY_REGEX) + for f in lines: + if parse_only_re.search(f): + parse_only.append(f.split("#")[0].strip()) + else: + blacklist.append(f) + return blacklist, parse_only + + +class BinaryRun(object): + def __init__(self, args, dry_run=False, close_handler=None): + self.args = args self.dry_run = dry_run self.results = None @@ -57,10 +69,12 @@ def __init__(self, args, dry_run=False): self.results = (0, "", "") else: self.proc = subprocess.Popen( - ["pytype"] + self.args, + self.args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - os.remove(temp.name) + + if close_handler: + close_handler() def communicate(self): if self.results: @@ -73,27 +87,48 @@ def communicate(self): def pytype_test(args): try: - PytdRun(["-h"]).communicate() + BinaryRun(["pytd", "-h"]).communicate() except OSError: print("Cannot run pytd. Did you install pytype?") return 0, 0 + blacklist, parse_only = load_blacklist() wanted = re.compile(r"stdlib/.*\.pyi$") - skipped = re.compile("(%s)$" % "|".join(load_blacklist())) - files = [] + skipped = re.compile("(%s)$" % "|".join(blacklist)) + parse_only = re.compile("(%s)$" % "|".join(parse_only)) + + pytype_run = [] + pytd_run = [] for root, _, filenames in os.walk("stdlib"): for f in sorted(filenames): f = os.path.join(root, f) - if wanted.search(f) and not skipped.search(f): - files.append(f) + if wanted.search(f): + if parse_only.search(f): + pytd_run.append(f) + elif not skipped.search(f): + pytype_run.append(f) running_tests = collections.deque() max_code, runs, errors = 0, 0, 0 - print("Running pytype tests...") + files = pytype_run + pytd_run while 1: while files and len(running_tests) < args.num_parallel: - test_run = PytdRun([files.pop()], dry_run=args.dry_run) + f = files.pop() + if f in pytype_run: + temp = tempfile.NamedTemporaryFile(delete=False) + temp.close() # Windows compat. + test_run = BinaryRun(["pytype", + "--typeshed-location=%s" % os.getcwd(), + "--module-name=%s" % _get_module_name(f), + '--convert-to-pickle=%s' % temp.name, + f], + dry_run=args.dry_run, + close_handler=lambda: os.remove(temp.name)) + elif f in pytd_run: + test_run = BinaryRun(["pytd", f], dry_run=args.dry_run) + else: + raise ValueError("Unknown action for file: %s" % f) running_tests.append(test_run) if not running_tests: @@ -105,13 +140,17 @@ def pytype_test(args): runs += 1 if code: - print("pytd error processing \"%s\":" % test_run.args[1]) + print "Error for args: %s" % str(test_run.args) print(stderr) errors += 1 print("Ran pytype with %d pyis, got %d errors." % (runs, errors)) return max_code, runs +def _get_module_name(filename): + """Converts a filename stdblib/m.n/module/foo to module.foo.""" + return ".".join(filename.split(os.path.sep)[2:]).replace( + ".pyi", "").replace('.__init__', '') if __name__ == '__main__': main() From 130e2334437feabe06dfb29977099502f03e7f85 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 11:06:08 +0100 Subject: [PATCH 09/19] Add parse only comment to stdlib/2/os/__init__.pyi The files tries to load builtins which are not defined for python2. --- tests/pytype_blacklist.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytype_blacklist.txt b/tests/pytype_blacklist.txt index 912dadf8d0fe..19ec1d1ea07a 100644 --- a/tests/pytype_blacklist.txt +++ b/tests/pytype_blacklist.txt @@ -13,7 +13,7 @@ stdlib/3/typing.pyi stdlib/3.4/asyncio/tasks.pyi # builtins not found -stdlib/2/os/__init__.pyi +stdlib/2/os/__init__.pyi # parse only # Below files can not be loaded by PyType without an error. # Notice that the current travis configuration loads these files with Python 2.7. From 06dd71d9c566f28d56f0093a2d318285a6576ad5 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 11:20:17 +0100 Subject: [PATCH 10/19] Make indentation a multiple of four. --- tests/pytype_test.py | 47 ++++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index ea9a456bcd20..322cefef83f2 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -51,10 +51,10 @@ def load_blacklist(): parse_only = [] parse_only_re = re.compile(_PARSE_ONLY_REGEX) for f in lines: - if parse_only_re.search(f): - parse_only.append(f.split("#")[0].strip()) - else: - blacklist.append(f) + if parse_only_re.search(f): + parse_only.append(f.split("#")[0].strip()) + else: + blacklist.append(f) return blacklist, parse_only @@ -74,7 +74,7 @@ def __init__(self, args, dry_run=False, close_handler=None): stderr=subprocess.PIPE) if close_handler: - close_handler() + close_handler() def communicate(self): if self.results: @@ -104,10 +104,10 @@ def pytype_test(args): for f in sorted(filenames): f = os.path.join(root, f) if wanted.search(f): - if parse_only.search(f): - pytd_run.append(f) - elif not skipped.search(f): - pytype_run.append(f) + if parse_only.search(f): + pytd_run.append(f) + elif not skipped.search(f): + pytype_run.append(f) running_tests = collections.deque() max_code, runs, errors = 0, 0, 0 @@ -116,19 +116,19 @@ def pytype_test(args): while files and len(running_tests) < args.num_parallel: f = files.pop() if f in pytype_run: - temp = tempfile.NamedTemporaryFile(delete=False) - temp.close() # Windows compat. - test_run = BinaryRun(["pytype", - "--typeshed-location=%s" % os.getcwd(), - "--module-name=%s" % _get_module_name(f), - '--convert-to-pickle=%s' % temp.name, - f], - dry_run=args.dry_run, - close_handler=lambda: os.remove(temp.name)) + temp = tempfile.NamedTemporaryFile(delete=False) + temp.close() # Windows compat. + test_run = BinaryRun(["pytype", + "--typeshed-location=%s" % os.getcwd(), + "--module-name=%s" % _get_module_name(f), + '--convert-to-pickle=%s' % temp.name, + f], + dry_run=args.dry_run, + close_handler=lambda: os.remove(temp.name)) elif f in pytd_run: - test_run = BinaryRun(["pytd", f], dry_run=args.dry_run) + test_run = BinaryRun(["pytd", f], dry_run=args.dry_run) else: - raise ValueError("Unknown action for file: %s" % f) + raise ValueError("Unknown action for file: %s" % f) running_tests.append(test_run) if not running_tests: @@ -140,7 +140,6 @@ def pytype_test(args): runs += 1 if code: - print "Error for args: %s" % str(test_run.args) print(stderr) errors += 1 @@ -148,9 +147,9 @@ def pytype_test(args): return max_code, runs def _get_module_name(filename): - """Converts a filename stdblib/m.n/module/foo to module.foo.""" - return ".".join(filename.split(os.path.sep)[2:]).replace( - ".pyi", "").replace('.__init__', '') + """Converts a filename stdblib/m.n/module/foo to module.foo.""" + return ".".join(filename.split(os.path.sep)[2:]).replace( + ".pyi", "").replace('.__init__', '') if __name__ == '__main__': main() From 52ae6161545e52afcb0a88d98ac46c896c1b5e7b Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 11:22:56 +0100 Subject: [PATCH 11/19] Make indentation a multiple of four. --- tests/pytype_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 322cefef83f2..c5a428f5b3a7 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -123,8 +123,9 @@ def pytype_test(args): "--module-name=%s" % _get_module_name(f), '--convert-to-pickle=%s' % temp.name, f], - dry_run=args.dry_run, - close_handler=lambda: os.remove(temp.name)) + dry_run=args.dry_run, + close_handler=lambda: os.remove(temp.name) + ) elif f in pytd_run: test_run = BinaryRun(["pytd", f], dry_run=args.dry_run) else: From e06d71c5595642a8dbfce3eb297e38c4a1383cd6 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 11:26:42 +0100 Subject: [PATCH 12/19] Fix two more linter issues. --- tests/pytype_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index c5a428f5b3a7..57c7be6f31c8 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -74,7 +74,7 @@ def __init__(self, args, dry_run=False, close_handler=None): stderr=subprocess.PIPE) if close_handler: - close_handler() + close_handler() def communicate(self): if self.results: @@ -125,7 +125,7 @@ def pytype_test(args): f], dry_run=args.dry_run, close_handler=lambda: os.remove(temp.name) - ) + ) elif f in pytd_run: test_run = BinaryRun(["pytd", f], dry_run=args.dry_run) else: From 8de94a3dca6e3d5f583a1097521d6974d7b022c1 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Tue, 14 Nov 2017 11:33:09 +0100 Subject: [PATCH 13/19] Some more linting * Replace " with ' everywhere were it was used. * Move the multi line parameters around to allow for something the linter will accept. --- tests/pytype_test.py | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 57c7be6f31c8..f00e31df2446 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -34,17 +34,17 @@ def main(): code, runs = pytype_test(args) if code: - print("--- exit status %d ---" % code) + print('--- exit status %d ---' % code) sys.exit(code) if not runs: - print("--- nothing to do; exit 1 ---") + print('--- nothing to do; exit 1 ---') sys.exit(1) -_PARSE_ONLY_REGEX = r"\s*#\s*parse only\s*" +_PARSE_ONLY_REGEX = r'\s*#\s*parse only\s*' def load_blacklist(): filename = os.path.join(os.path.dirname(__file__), "pytype_blacklist.txt") - regex = r"^\s*([^\s#]+(?:%s)?)\s*(?:#.*)?$" % _PARSE_ONLY_REGEX + regex = r'^\s*([^\s#]+(?:%s)?)\s*(?:#.*)?$' % _PARSE_ONLY_REGEX with open(filename) as f: lines = re.findall(regex, f.read(), flags=re.M) blacklist = [] @@ -52,7 +52,7 @@ def load_blacklist(): parse_only_re = re.compile(_PARSE_ONLY_REGEX) for f in lines: if parse_only_re.search(f): - parse_only.append(f.split("#")[0].strip()) + parse_only.append(f.split('#')[0].strip()) else: blacklist.append(f) return blacklist, parse_only @@ -66,7 +66,7 @@ def __init__(self, args, dry_run=False, close_handler=None): self.results = None if dry_run: - self.results = (0, "", "") + self.results = (0, '', '') else: self.proc = subprocess.Popen( self.args, @@ -87,20 +87,20 @@ def communicate(self): def pytype_test(args): try: - BinaryRun(["pytd", "-h"]).communicate() + BinaryRun(['pytd', '-h']).communicate() except OSError: - print("Cannot run pytd. Did you install pytype?") + print('Cannot run pytd. Did you install pytype?') return 0, 0 blacklist, parse_only = load_blacklist() - wanted = re.compile(r"stdlib/.*\.pyi$") - skipped = re.compile("(%s)$" % "|".join(blacklist)) - parse_only = re.compile("(%s)$" % "|".join(parse_only)) + wanted = re.compile(r'stdlib/.*\.pyi$') + skipped = re.compile('(%s)$' % '|'.join(blacklist)) + parse_only = re.compile('(%s)$' % '|'.join(parse_only)) pytype_run = [] pytd_run = [] - for root, _, filenames in os.walk("stdlib"): + for root, _, filenames in os.walk('stdlib'): for f in sorted(filenames): f = os.path.join(root, f) if wanted.search(f): @@ -118,18 +118,18 @@ def pytype_test(args): if f in pytype_run: temp = tempfile.NamedTemporaryFile(delete=False) temp.close() # Windows compat. - test_run = BinaryRun(["pytype", - "--typeshed-location=%s" % os.getcwd(), - "--module-name=%s" % _get_module_name(f), - '--convert-to-pickle=%s' % temp.name, - f], - dry_run=args.dry_run, - close_handler=lambda: os.remove(temp.name) - ) + test_run = BinaryRun( + ['pytype', + '--typeshed-location=%s' % os.getcwd(), + '--module-name=%s' % _get_module_name(f), + '--convert-to-pickle=%s' % temp.name, + f], + dry_run=args.dry_run, + close_handler=lambda: os.remove(temp.name)) elif f in pytd_run: - test_run = BinaryRun(["pytd", f], dry_run=args.dry_run) + test_run = BinaryRun(['pytd', f], dry_run=args.dry_run) else: - raise ValueError("Unknown action for file: %s" % f) + raise ValueError('Unknown action for file: %s' % f) running_tests.append(test_run) if not running_tests: @@ -144,13 +144,13 @@ def pytype_test(args): print(stderr) errors += 1 - print("Ran pytype with %d pyis, got %d errors." % (runs, errors)) + print('Ran pytype with %d pyis, got %d errors.' % (runs, errors)) return max_code, runs def _get_module_name(filename): """Converts a filename stdblib/m.n/module/foo to module.foo.""" - return ".".join(filename.split(os.path.sep)[2:]).replace( - ".pyi", "").replace('.__init__', '') + return '.'.join(filename.split(os.path.sep)[2:]).replace( + '.pyi', '').replace('.__init__', '') if __name__ == '__main__': main() From 06a9ad9b493fa7b53ec95a5f0525846ef455d89b Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 15 Nov 2017 12:20:43 +0100 Subject: [PATCH 14/19] Use os.devnull instead of a temp file. --- tests/pytype_test.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index f00e31df2446..9c9cd25b8c7c 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -59,7 +59,7 @@ def load_blacklist(): class BinaryRun(object): - def __init__(self, args, dry_run=False, close_handler=None): + def __init__(self, args, dry_run=False): self.args = args self.dry_run = dry_run @@ -73,9 +73,6 @@ def __init__(self, args, dry_run=False, close_handler=None): stdout=subprocess.PIPE, stderr=subprocess.PIPE) - if close_handler: - close_handler() - def communicate(self): if self.results: return self.results @@ -116,16 +113,13 @@ def pytype_test(args): while files and len(running_tests) < args.num_parallel: f = files.pop() if f in pytype_run: - temp = tempfile.NamedTemporaryFile(delete=False) - temp.close() # Windows compat. test_run = BinaryRun( ['pytype', '--typeshed-location=%s' % os.getcwd(), '--module-name=%s' % _get_module_name(f), - '--convert-to-pickle=%s' % temp.name, + '--convert-to-pickle=%s' % os.devnull, f], - dry_run=args.dry_run, - close_handler=lambda: os.remove(temp.name)) + dry_run=args.dry_run) elif f in pytd_run: test_run = BinaryRun(['pytd', f], dry_run=args.dry_run) else: From c24fb668c75f7aab878ad4a88f46bf77b99eea3f Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 15 Nov 2017 12:22:28 +0100 Subject: [PATCH 15/19] Move _get_module_name above call location. --- tests/pytype_test.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 9c9cd25b8c7c..0e956995d5e1 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -82,6 +82,12 @@ def communicate(self): return self.results +def _get_module_name(filename): + """Converts a filename stdblib/m.n/module/foo to module.foo.""" + return '.'.join(filename.split(os.path.sep)[2:]).replace( + '.pyi', '').replace('.__init__', '') + + def pytype_test(args): try: BinaryRun(['pytd', '-h']).communicate() @@ -141,10 +147,5 @@ def pytype_test(args): print('Ran pytype with %d pyis, got %d errors.' % (runs, errors)) return max_code, runs -def _get_module_name(filename): - """Converts a filename stdblib/m.n/module/foo to module.foo.""" - return '.'.join(filename.split(os.path.sep)[2:]).replace( - '.pyi', '').replace('.__init__', '') - if __name__ == '__main__': main() From 0c5d43a06d6b0f1555d59400a0da6a6e6526490a Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 15 Nov 2017 12:24:13 +0100 Subject: [PATCH 16/19] Rename blacklist to skip. --- tests/pytype_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 0e956995d5e1..771ad7f9a9ee 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -95,9 +95,9 @@ def pytype_test(args): print('Cannot run pytd. Did you install pytype?') return 0, 0 - blacklist, parse_only = load_blacklist() + skip, parse_only = load_blacklist() wanted = re.compile(r'stdlib/.*\.pyi$') - skipped = re.compile('(%s)$' % '|'.join(blacklist)) + skipped = re.compile('(%s)$' % '|'.join(skip)) parse_only = re.compile('(%s)$' % '|'.join(parse_only)) pytype_run = [] From ea9adbbdbd52cc7956c413332bd66892326327be Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 15 Nov 2017 12:39:55 +0100 Subject: [PATCH 17/19] Refactor the skip, parse_only list creation to use independent regexes per list. --- tests/pytype_test.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 771ad7f9a9ee..336bf8aba7ba 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -40,22 +40,24 @@ def main(): print('--- nothing to do; exit 1 ---') sys.exit(1) -_PARSE_ONLY_REGEX = r'\s*#\s*parse only\s*' def load_blacklist(): filename = os.path.join(os.path.dirname(__file__), "pytype_blacklist.txt") - regex = r'^\s*([^\s#]+(?:%s)?)\s*(?:#.*)?$' % _PARSE_ONLY_REGEX - with open(filename) as f: - lines = re.findall(regex, f.read(), flags=re.M) - blacklist = [] + skip_re = re.compile(r'^\s*([^\s#]+)\s*(?:#.*)?$') + parse_only_re = re.compile(r'^\s*([^\s#]+)\s*#\s*parse only\s*') + skip = [] parse_only = [] - parse_only_re = re.compile(_PARSE_ONLY_REGEX) - for f in lines: - if parse_only_re.search(f): - parse_only.append(f.split('#')[0].strip()) - else: - blacklist.append(f) - return blacklist, parse_only + + with open(filename) as f: + for line in f: + parse_only_match = parse_only_re.match(line) + skip_match = skip_re.match(line) + if parse_only_match: + parse_only.append(parse_only_match.group(1)) + elif skip_match: + skip.append(skip_match.group(1)) + + return skip, parse_only class BinaryRun(object): From 6946a429923a066ee75aab55300d48d304930287 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 15 Nov 2017 12:41:30 +0100 Subject: [PATCH 18/19] Remove unused tempfile import. --- tests/pytype_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 336bf8aba7ba..9b4b2b71e3b1 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -20,7 +20,6 @@ import sys import argparse import subprocess -import tempfile import collections parser = argparse.ArgumentParser(description="Pytype tests.") From 92060089a8b1f5fb0bb8d14bb83d35c60c644e05 Mon Sep 17 00:00:00 2001 From: Sebastian Steenbuck Date: Wed, 15 Nov 2017 12:42:15 +0100 Subject: [PATCH 19/19] Make whitespaces a multiple of four --- tests/pytype_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 9b4b2b71e3b1..f90c4811a0da 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -54,7 +54,7 @@ def load_blacklist(): if parse_only_match: parse_only.append(parse_only_match.group(1)) elif skip_match: - skip.append(skip_match.group(1)) + skip.append(skip_match.group(1)) return skip, parse_only