From c6585bf8ef653c2022a88b115dc00e36035cd9f0 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Mon, 9 Jan 2023 18:24:36 -0800 Subject: [PATCH 1/6] pytype_test: Mark typeshed-external dependencies as missing. --- tests/pytype_test.py | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index f1249f5c35e7..d3c8c0944efe 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -14,13 +14,16 @@ import argparse import os +import pkg_resources import sys import traceback -from collections.abc import Sequence +from collections.abc import Iterable, Sequence from pytype import config as pytype_config, load_pytd # type: ignore[import] from pytype.imports import typeshed # type: ignore[import] +import utils + TYPESHED_SUBDIRS = ["stdlib", "stubs"] TYPESHED_HOME = "TYPESHED_HOME" _LOADERS = {} @@ -56,11 +59,13 @@ def create_parser() -> argparse.ArgumentParser: return parser -def run_pytype(*, filename: str, python_version: str) -> str | None: +def run_pytype(*, filename: str, python_version: str, missing_modules: Sequence[str]) -> str | None: """Runs pytype, returning the stderr if any.""" if python_version not in _LOADERS: options = pytype_config.Options.create("", parse_pyi=True, python_version=python_version) - loader = load_pytd.create_loader(options) + # For simplicity, pretends missing modules are part of the stdlib. + missing_modules = tuple(os.path.join("stdlib", m) for m in missing_modules) + loader = load_pytd.create_loader(options, missing_modules) _LOADERS[python_version] = (options, loader) options, loader = _LOADERS[python_version] stderr: str | None @@ -131,14 +136,43 @@ def find_stubs_in_paths(paths: Sequence[str]) -> list[str]: return filenames +def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]: + """Gets module names provided by typeshed-external dependencies. + + Some typeshed stubs depend on dependencies outside of typeshed. Since pytype + isn't able to read such dependencies, we instead declare them as "missing" + modules, so that no errors are reported for them. + """ + stub_distributions = set() + for fi in files_to_test: + parts = fi.split(os.sep) + try: + idx = parts.index("stubs") + except ValueError: + continue + stub_distributions.add(parts[idx + 1]) + missing_modules = set() + for distribution in stub_distributions: + for pkg in utils.read_dependencies(distribution).external_pkgs: + # See https://stackoverflow.com/a/54853084. + top_level_file = os.path.join(pkg_resources.get_distribution(pkg).egg_info, "top_level.txt") + with open(top_level_file) as fi: + missing_modules.update(fi.read().splitlines()) + return missing_modules + + def run_all_tests(*, files_to_test: Sequence[str], print_stderr: bool, dry_run: bool) -> None: bad = [] errors = 0 total_tests = len(files_to_test) + missing_modules = get_missing_modules(files_to_test) print("Testing files with pytype...") for i, f in enumerate(files_to_test): python_version = "{0.major}.{0.minor}".format(sys.version_info) - stderr = run_pytype(filename=f, python_version=python_version) if not dry_run else None + if dry_run: + stderr = None + else: + stderr = run_pytype(filename=f, python_version=python_version, missing_modules=missing_modules) if stderr: if print_stderr: print(f"\n{stderr}") From ae65266af19845b6aa15c98a936a5e1118e40e0d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 10 Jan 2023 02:28:53 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks --- tests/pytype_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index d3c8c0944efe..6076768a826d 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -14,16 +14,15 @@ import argparse import os -import pkg_resources import sys import traceback from collections.abc import Iterable, Sequence +import pkg_resources +import utils from pytype import config as pytype_config, load_pytd # type: ignore[import] from pytype.imports import typeshed # type: ignore[import] -import utils - TYPESHED_SUBDIRS = ["stdlib", "stubs"] TYPESHED_HOME = "TYPESHED_HOME" _LOADERS = {} From 2bd832c51a3f4a682c584645a43cba0a1d74bea1 Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Tue, 10 Jan 2023 11:34:32 -0800 Subject: [PATCH 3/6] Remove period to fix hyperlink. Co-authored-by: Avasam --- tests/pytype_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 6076768a826d..a7f454208d09 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -153,7 +153,7 @@ def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]: missing_modules = set() for distribution in stub_distributions: for pkg in utils.read_dependencies(distribution).external_pkgs: - # See https://stackoverflow.com/a/54853084. + # See https://stackoverflow.com/a/54853084 top_level_file = os.path.join(pkg_resources.get_distribution(pkg).egg_info, "top_level.txt") with open(top_level_file) as fi: missing_modules.update(fi.read().splitlines()) From a83fbe62de82518fbd3585b39733dccb413a0bba Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Tue, 10 Jan 2023 11:35:47 -0800 Subject: [PATCH 4/6] Add types-setuptools to requirements-tests.txt. --- requirements-tests.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-tests.txt b/requirements-tests.txt index 8a21831e9c20..873ceb455645 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -15,4 +15,5 @@ termcolor>=2 tomli==2.0.1 tomlkit==0.11.6 types-pyyaml +types-setuptools typing-extensions From e2b00ffe7ce8450a1e20727ca9a6da5efda8349f Mon Sep 17 00:00:00 2001 From: Rebecca Chen Date: Tue, 10 Jan 2023 11:52:15 -0800 Subject: [PATCH 5/6] Address mypy errors. --- tests/pytype_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index a7f454208d09..74dfc302c61c 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -58,7 +58,7 @@ def create_parser() -> argparse.ArgumentParser: return parser -def run_pytype(*, filename: str, python_version: str, missing_modules: Sequence[str]) -> str | None: +def run_pytype(*, filename: str, python_version: str, missing_modules: Iterable[str]) -> str | None: """Runs pytype, returning the stderr if any.""" if python_version not in _LOADERS: options = pytype_config.Options.create("", parse_pyi=True, python_version=python_version) @@ -154,9 +154,9 @@ def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]: for distribution in stub_distributions: for pkg in utils.read_dependencies(distribution).external_pkgs: # See https://stackoverflow.com/a/54853084 - top_level_file = os.path.join(pkg_resources.get_distribution(pkg).egg_info, "top_level.txt") - with open(top_level_file) as fi: - missing_modules.update(fi.read().splitlines()) + top_level_file = os.path.join(pkg_resources.get_distribution(pkg).egg_info, "top_level.txt") # type: ignore[attr-defined] + with open(top_level_file) as f: + missing_modules.update(f.read().splitlines()) return missing_modules From f95ce152347068e4f1486fe1518887a4462f741a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 10 Jan 2023 21:50:46 +0000 Subject: [PATCH 6/6] Bump pinned pytype version --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index 873ceb455645..3675b6b08c18 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -10,7 +10,7 @@ packaging==22.0 pathspec pycln==2.1.2 # must match .pre-commit-config.yaml pyyaml==6.0 -pytype==2022.12.15; platform_system != "Windows" and python_version < "3.11" +pytype==2023.1.10; platform_system != "Windows" and python_version < "3.11" termcolor>=2 tomli==2.0.1 tomlkit==0.11.6