From ea03bcaf48fdc53ba0c7dc357f01de07a74eb903 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 30 Sep 2023 18:34:37 +0200 Subject: [PATCH 1/4] Make pytype respect required python versions of stubs This is the pytype part of #10722 --- tests/pytype_test.py | 79 +++++++++++++++++++++++++++++++++----------- tests/utils.py | 15 +++++++++ 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index af7dedade51f..39fa81ec8f5e 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -20,11 +20,14 @@ import os import sys import traceback +from collections import defaultdict from collections.abc import Iterable, Sequence +from pathlib import Path from packaging.requirements import Requirement -from parse_metadata import read_dependencies +from parse_metadata import read_dependencies, read_metadata +from utils import PYTHON_VERSION, chdir, colored if sys.platform == "win32": print("pytype does not support Windows.", file=sys.stderr) @@ -45,12 +48,16 @@ def main() -> None: args = create_parser().parse_args() typeshed_location = args.typeshed_location or os.getcwd() - subdir_paths = [os.path.join(typeshed_location, d) for d in TYPESHED_SUBDIRS] - check_subdirs_discoverable(subdir_paths) old_typeshed_home = os.environ.get(TYPESHED_HOME) os.environ[TYPESHED_HOME] = typeshed_location - files_to_test = determine_files_to_test(paths=args.files or subdir_paths) - run_all_tests(files_to_test=files_to_test, print_stderr=args.print_stderr, dry_run=args.dry_run) + print("Testing files with pytype...") + with chdir(typeshed_location): + check_subdirs_discoverable(TYPESHED_SUBDIRS) + files_to_test = determine_files_to_test(paths=args.files or TYPESHED_SUBDIRS) + if not files_to_test: + print(colored("Nothing to do; exit 1.", "red")) + sys.exit(1) + run_all_tests(files_to_test=files_to_test, print_stderr=args.print_stderr, dry_run=args.dry_run) if old_typeshed_home is None: del os.environ[TYPESHED_HOME] else: @@ -122,11 +129,38 @@ def check_subdirs_discoverable(subdir_paths: list[str]) -> None: raise SystemExit(f"Cannot find typeshed subdir at {p} (specify parent dir via --typeshed-location)") +def classify_files(paths: Sequence[str]) -> tuple[list[str], defaultdict[str, list[str]]]: + """Classify files into stdlib and stubs by distribution.""" + stdlib: list[str] = [] + stubs: defaultdict[str, list[str]] = defaultdict(list) + stubs_location = Path("stubs").resolve() + for path_s in paths: + path = Path(path_s).resolve() + if not path.is_relative_to(stubs_location): + stdlib.append(path_s) + elif path.samefile(stubs_location): + for subdir in path.iterdir(): + stubs[subdir.name].append(str(subdir)) + else: + distribution = path.relative_to(stubs_location).parts[0] + stubs[distribution].append(str(path)) + return stdlib, stubs + + def determine_files_to_test(*, paths: Sequence[str]) -> list[str]: """Determine all files to test, checking if it's in the exclude list and which Python versions to use. Returns a list of pairs of the file path and Python version as an int.""" - filenames = find_stubs_in_paths(paths) + stdlib, stubs = classify_files(paths) + paths_to_test = list(stdlib) + for pkg, pkg_paths in stubs.items(): + requires_python = read_metadata(pkg).requires_python + if not requires_python.contains(PYTHON_VERSION): + msg = f"skipping {pkg!r} (requires Python {requires_python}; test is being run using Python {PYTHON_VERSION})" + print(colored(msg, "yellow")) + continue + paths_to_test.extend(pkg_paths) + filenames = find_stubs_in_paths(paths_to_test) ts = typeshed.Typeshed() skipped = set(ts.read_blacklist()) files = [] @@ -204,33 +238,38 @@ def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]: def run_all_tests(*, files_to_test: Sequence[str], print_stderr: bool, dry_run: bool) -> None: - bad = [] errors = 0 total_tests = len(files_to_test) missing_modules = get_missing_modules(files_to_test) - print("Testing files with pytype...") - for i, f in enumerate(files_to_test): - python_version = "{0.major}.{0.minor}".format(sys.version_info) + for runs, f in enumerate(files_to_test, start=1): if dry_run: stderr = None else: - stderr = run_pytype(filename=f, python_version=python_version, missing_modules=missing_modules) + stderr = run_pytype(filename=f, python_version=PYTHON_VERSION, missing_modules=missing_modules) if stderr: - if print_stderr: - print(f"\n{stderr}") errors += 1 - stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1] - bad.append((_get_relative(f), python_version, stacktrace_final_line)) + if print_stderr: + print(f"{stderr}\n") + else: + stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1] + print(f"{_get_relative(f)} ({PYTHON_VERSION}): {stacktrace_final_line}\n") - runs = i + 1 if runs % 25 == 0: print(f" {runs:3d}/{total_tests:d} with {errors:3d} errors") - print(f"Ran pytype with {total_tests:d} pyis, got {errors:d} errors.") - for f, v, err in bad: - print(f"\n{f} ({v}): {err}") + file_plural = "file" if total_tests == 1 else "files" + error_plural = "error" if errors == 1 else "errors" + msg = f"Ran pytype with {total_tests:d} pyi {file_plural}, got {errors:d} {error_plural}." if errors: - raise SystemExit("\nRun again with --print-stderr to get the full stacktrace.") + color = "red" + code = 1 + if not print_stderr: + msg += "\nRun again with --print-stderr to get the full stacktrace." + else: + color = "green" + code = 0 + print(colored(msg, color)) + sys.exit(code) if __name__ == "__main__": diff --git a/tests/utils.py b/tests/utils.py index 2fd376314057..81f08532cdd3 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -7,6 +7,8 @@ import subprocess import sys import venv +from collections.abc import Iterator +from contextlib import contextmanager from functools import lru_cache from pathlib import Path from typing import Any, Final, NamedTuple @@ -138,3 +140,16 @@ def spec_matches_path(spec: pathspec.PathSpec, path: Path) -> bool: if path.is_dir(): normalized_path += "/" return spec.match_file(normalized_path) + + +# ==================================================================== +# Similar to `contextlib.chdir` on Python 3.11+ +# ==================================================================== +@contextmanager +def chdir(path: str | os.PathLike[str]) -> Iterator[None]: + old_cwd = os.getcwd() + try: + os.chdir(path) + yield + finally: + os.chdir(old_cwd) From e810d06ee85ee9bec530490057bd93cf32697b21 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sat, 30 Sep 2023 22:25:47 +0200 Subject: [PATCH 2/4] Relative paths --- tests/pytype_test.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 39fa81ec8f5e..4d3d25f8d5e7 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -133,17 +133,20 @@ def classify_files(paths: Sequence[str]) -> tuple[list[str], defaultdict[str, li """Classify files into stdlib and stubs by distribution.""" stdlib: list[str] = [] stubs: defaultdict[str, list[str]] = defaultdict(list) - stubs_location = Path("stubs").resolve() + stubs_path = Path("stubs") + stubs_absolute_path = Path(stubs_path).resolve() for path_s in paths: path = Path(path_s).resolve() - if not path.is_relative_to(stubs_location): - stdlib.append(path_s) - elif path.samefile(stubs_location): - for subdir in path.iterdir(): + if path.samefile(stubs_absolute_path): + # All stubs, classify by distribution for version checking later. + for subdir in stubs_path.iterdir(): stubs[subdir.name].append(str(subdir)) + elif path.is_relative_to(stubs_absolute_path): + # A single stub directory or file. + distribution = path.relative_to(stubs_absolute_path).parts[0] + stubs[distribution].append(path_s) else: - distribution = path.relative_to(stubs_location).parts[0] - stubs[distribution].append(str(path)) + stdlib.append(path_s) return stdlib, stubs From 028f9c0268372b2f4711ea51768ef3e1c5dfe0a6 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 1 Oct 2023 09:14:10 +0200 Subject: [PATCH 3/4] Update tests/pytype_test.py Co-authored-by: Jelle Zijlstra --- tests/pytype_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 4d3d25f8d5e7..79dc52a67887 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -134,7 +134,7 @@ def classify_files(paths: Sequence[str]) -> tuple[list[str], defaultdict[str, li stdlib: list[str] = [] stubs: defaultdict[str, list[str]] = defaultdict(list) stubs_path = Path("stubs") - stubs_absolute_path = Path(stubs_path).resolve() + stubs_absolute_path = stubs_path.resolve() for path_s in paths: path = Path(path_s).resolve() if path.samefile(stubs_absolute_path): From 1ca539b0f35f04928f11be8389565597749f1590 Mon Sep 17 00:00:00 2001 From: Ali Hamdan Date: Sun, 1 Oct 2023 09:56:12 +0200 Subject: [PATCH 4/4] Better result reporting colors, keep summary at the end, file before error stack, better alignment --- tests/pytype_test.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/pytype_test.py b/tests/pytype_test.py index 79dc52a67887..b1f92925b89b 100755 --- a/tests/pytype_test.py +++ b/tests/pytype_test.py @@ -19,6 +19,7 @@ import inspect import os import sys +import textwrap import traceback from collections import defaultdict from collections.abc import Iterable, Sequence @@ -50,7 +51,7 @@ def main() -> None: typeshed_location = args.typeshed_location or os.getcwd() old_typeshed_home = os.environ.get(TYPESHED_HOME) os.environ[TYPESHED_HOME] = typeshed_location - print("Testing files with pytype...") + print(f"Testing files with pytype on Python {PYTHON_VERSION}...") with chdir(typeshed_location): check_subdirs_discoverable(TYPESHED_SUBDIRS) files_to_test = determine_files_to_test(paths=args.files or TYPESHED_SUBDIRS) @@ -241,6 +242,7 @@ def get_missing_modules(files_to_test: Sequence[str]) -> Iterable[str]: def run_all_tests(*, files_to_test: Sequence[str], print_stderr: bool, dry_run: bool) -> None: + bad = [] errors = 0 total_tests = len(files_to_test) missing_modules = get_missing_modules(files_to_test) @@ -251,18 +253,23 @@ def run_all_tests(*, files_to_test: Sequence[str], print_stderr: bool, dry_run: stderr = run_pytype(filename=f, python_version=PYTHON_VERSION, missing_modules=missing_modules) if stderr: errors += 1 + test_file = f"{_get_relative(f)}:" if print_stderr: - print(f"{stderr}\n") - else: - stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1] - print(f"{_get_relative(f)} ({PYTHON_VERSION}): {stacktrace_final_line}\n") + print(colored(test_file, "red")) + print(f"{textwrap.indent(stderr, ' ')}") + stacktrace_final_line = stderr.rstrip().rsplit("\n", 1)[-1] + bad.append((test_file, stacktrace_final_line)) if runs % 25 == 0: - print(f" {runs:3d}/{total_tests:d} with {errors:3d} errors") + color = "red" if errors else "green" + print(colored(f" {runs:4d}/{total_tests:d} with {errors:4d} errors", color)) + + for test, err in bad: + print(colored(test, "red"), err) file_plural = "file" if total_tests == 1 else "files" error_plural = "error" if errors == 1 else "errors" - msg = f"Ran pytype with {total_tests:d} pyi {file_plural}, got {errors:d} {error_plural}." + msg = f"\nRan pytype with {total_tests:d} pyi {file_plural}, got {errors:d} {error_plural}." if errors: color = "red" code = 1