From 0ae224031bce1b33af21f3e1e0d760751aa38dfd Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 12:07:12 +0200 Subject: [PATCH 01/11] Fix mypy_test if mypy returns negative exit code --- tests/mypy_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 0ff0cd66c1dc..bf258c76e6bc 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -11,6 +11,7 @@ import sys import tempfile import time +from calendar import c from collections import defaultdict from dataclasses import dataclass from itertools import product @@ -366,7 +367,8 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: # We don't actually need pip for the stdlib testing venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) - code = max(code, this_code) + if code == 0: + code = this_code return TestResults(code, len(files)) @@ -530,7 +532,8 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe this_code, checked, _ = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - code = max(code, this_code) + if code == 0: + code = this_code files_checked += checked return TestResults(code, files_checked, packages_skipped) From f82ad0681dc4c5801ca94960162edbb25e9e7e25 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 12:09:58 +0200 Subject: [PATCH 02/11] Remove spurious import --- tests/mypy_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index bf258c76e6bc..6997bded1ccc 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -11,7 +11,6 @@ import sys import tempfile import time -from calendar import c from collections import defaultdict from dataclasses import dataclass from itertools import product From e5dd49113de24f2ade5c4bbbd40f4b85e902786c Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 12:10:18 +0200 Subject: [PATCH 03/11] Always exit with status 1 from mypy_test --- tests/mypy_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 6997bded1ccc..f90f94c0678d 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -575,8 +575,8 @@ def main() -> None: total_packages_skipped += packages_skipped_this_version if code: plural = "" if total_files_checked == 1 else "s" - print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---") - sys.exit(code) + print_error(f"--- mypy exit status {code}, {total_files_checked} file{plural} checked ---") + sys.exit(1) if total_packages_skipped: plural = "" if total_packages_skipped == 1 else "s" print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow")) From 8a5fc44db1a283fca8dab518e43214ede916727f Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 12:14:16 +0200 Subject: [PATCH 04/11] Show exit code in failure message --- tests/mypy_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index f90f94c0678d..b659bf8c4c5a 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -276,7 +276,7 @@ def run_mypy( print(colored(f"running {' '.join(mypy_command)}", "blue")) result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars) if result.returncode: - print_error("failure\n") + print_error(f"failure (exit code {result.returncode})\n") if result.stdout: print_error(result.stdout) if result.stderr: @@ -575,7 +575,7 @@ def main() -> None: total_packages_skipped += packages_skipped_this_version if code: plural = "" if total_files_checked == 1 else "s" - print_error(f"--- mypy exit status {code}, {total_files_checked} file{plural} checked ---") + print_error(f"--- error, {total_files_checked} file{plural} checked ---") sys.exit(1) if total_packages_skipped: plural = "" if total_packages_skipped == 1 else "s" From 752bd025f55d9402511f6cc5c433c68dd87fd1ee Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 12:42:44 +0200 Subject: [PATCH 05/11] Rework test result handling Turn `TestResults` into a dataclass that knows how to combine multiple results. --- tests/mypy_test.py | 120 +++++++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 48 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index b659bf8c4c5a..1fee770ee56e 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -13,6 +13,7 @@ import time from collections import defaultdict from dataclasses import dataclass +from enum import Enum from itertools import product from pathlib import Path from threading import Lock @@ -305,11 +306,37 @@ def add_third_party_files( add_configuration(configurations, distribution) -class TestResults(NamedTuple): - exit_code: int - files_checked: int +class MypyResult(Enum): + SUCCESS = 0 + FAILURE = 1 + CRASH = 2 + + +@dataclass(frozen=True) +class TestResults: + mypy_result: MypyResult = MypyResult.SUCCESS + files_checked: int = 0 packages_skipped: int = 0 + @classmethod + def from_code(self, mypy_result_code: int, files_checked: int = 0) -> TestResults: + if mypy_result_code == 0: + result = MypyResult.SUCCESS + elif mypy_result_code == 1: + result = MypyResult.FAILURE + else: + result = MypyResult.CRASH + return TestResults(result, files_checked) + + def combine(self, other: TestResults) -> TestResults: + result = self.mypy_result if self.mypy_result.value > other.mypy_result.value else other.mypy_result + checked = self.files_checked + other.files_checked + skipped = self.packages_skipped + other.packages_skipped + return TestResults(result, checked, skipped) + + def skip_package(self) -> TestResults: + return TestResults(self.mypy_result, self.files_checked, self.packages_skipped + 1) + def test_third_party_distribution( distribution: str, args: TestConfig, venv_info: VenvInfo, *, non_types_dependencies: bool @@ -326,7 +353,7 @@ def test_third_party_distribution( add_third_party_files(distribution, files, args, configurations, seen_dists) if not files and args.filter: - return TestResults(0, 0) + return TestResults() print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True) @@ -346,10 +373,10 @@ def test_third_party_distribution( testing_stdlib=False, non_types_dependencies=non_types_dependencies, ) - return TestResults(code, len(files)) + return TestResults.from_code(code, len(files)) -def test_stdlib(code: int, args: TestConfig) -> TestResults: +def test_stdlib(args: TestConfig) -> TestResults: files: list[Path] = [] stdlib = Path("stdlib") supported_versions = parse_versions(stdlib / "VERSIONS") @@ -361,15 +388,14 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: if module_min_version <= tuple(map(int, args.version.split("."))) <= module_max_version: add_files(files, (stdlib / name), args) - if files: - print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) - # We don't actually need pip for the stdlib testing - venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) - this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) - if code == 0: - code = this_code + if not files: + return TestResults() - return TestResults(code, len(files)) + print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) + # We don't actually need pip for the stdlib testing + venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) + code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) + return TestResults.from_code(code, len(files)) _PRINT_LOCK = Lock() @@ -474,10 +500,9 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar _DISTRIBUTION_TO_VENV_MAPPING.update(dict.fromkeys(distribution_list, venv_to_use)) -def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults: +def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestResults: print("Testing third-party packages...") - files_checked = 0 - packages_skipped = 0 + results = TestResults() gitignore_spec = get_gitignore_spec() distributions_to_check: dict[str, PackageDependencies] = {} @@ -494,12 +519,12 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe f"test is being run using Python {PYTHON_VERSION})" ) print(colored(msg, "yellow")) - packages_skipped += 1 + results = results.skip_package() continue if not metadata.requires_python.contains(args.version): msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})" print(colored(msg, "yellow")) - packages_skipped += 1 + results = results.skip_package() continue if ( @@ -528,33 +553,30 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe for distribution in distributions_to_check: venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution] non_types_dependencies = venv_info.python_exe != sys.executable - this_code, checked, _ = test_third_party_distribution( + this_results = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - if code == 0: - code = this_code - files_checked += checked + results = results.combine(this_results) - return TestResults(code, files_checked, packages_skipped) + return results -def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: +def test_typeshed(args: TestConfig, tempdir: Path) -> TestResults: print(f"*** Testing Python {args.version} on {args.platform}") - files_checked_this_version = 0 - packages_skipped_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") + results = TestResults() + if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - code, stdlib_files_checked, _ = test_stdlib(code, args) - files_checked_this_version += stdlib_files_checked + stdlib_results = test_stdlib(args) + results = results.combine(stdlib_results) print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): - code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir) - files_checked_this_version += third_party_files_checked - packages_skipped_this_version = third_party_packages_skipped + third_party_results = test_third_party_stubs(args, tempdir) + results = results.combine(third_party_results) print() - return TestResults(code, files_checked_this_version, packages_skipped_this_version) + return results def main() -> None: @@ -563,26 +585,28 @@ def main() -> None: platforms = args.platform or [sys.platform] filter = args.filter or DIRECTORIES_TO_TEST exclude = args.exclude or [] - code = 0 - total_files_checked = 0 - total_packages_skipped = 0 + results = TestResults() with tempfile.TemporaryDirectory() as td: td_path = Path(td) for version, platform in product(versions, platforms): config = TestConfig(args.verbose, filter, exclude, version, platform) - code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path) - total_files_checked += files_checked_this_version - total_packages_skipped += packages_skipped_this_version - if code: - plural = "" if total_files_checked == 1 else "s" - print_error(f"--- error, {total_files_checked} file{plural} checked ---") + this_results = test_typeshed(args=config, tempdir=td_path) + results = results.combine(this_results) + + if results.mypy_result == MypyResult.FAILURE: + plural = "" if results.files_checked == 1 else "s" + print_error(f"--- error, {results.files_checked} file{plural} checked ---") sys.exit(1) - if total_packages_skipped: - plural = "" if total_packages_skipped == 1 else "s" - print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow")) - if total_files_checked: - plural = "" if total_files_checked == 1 else "s" - print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green")) + if results.mypy_result == MypyResult.CRASH: + plural = "" if results.files_checked == 1 else "s" + print_error(f"--- mypy crashed, {results.files_checked} file{plural} checked ---") + sys.exit(2) + if results.packages_skipped: + plural = "" if results.packages_skipped == 1 else "s" + print(colored(f"--- {results.packages_skipped} package{plural} skipped ---", "yellow")) + if results.files_checked: + plural = "" if results.files_checked == 1 else "s" + print(colored(f"--- success, {results.files_checked} file{plural} checked ---", "green")) else: print_error("--- nothing to do; exit 1 ---") sys.exit(1) From 13269718e618634ff9cbb1d55b99a50328349a3f Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 12:47:34 +0200 Subject: [PATCH 06/11] Fix a pyright warning --- tests/mypy_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 1fee770ee56e..1405a477bf37 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -318,8 +318,8 @@ class TestResults: files_checked: int = 0 packages_skipped: int = 0 - @classmethod - def from_code(self, mypy_result_code: int, files_checked: int = 0) -> TestResults: + @staticmethod + def from_code(mypy_result_code: int, files_checked: int = 0) -> TestResults: if mypy_result_code == 0: result = MypyResult.SUCCESS elif mypy_result_code == 1: From 95b6825ccc8405d794a47162a87031b65c253bc2 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 13:30:08 +0200 Subject: [PATCH 07/11] Add packages with errors count Return new TestResult tuple from single package test functions --- tests/mypy_test.py | 110 +++++++++++++++++++++++++-------------------- 1 file changed, 61 insertions(+), 49 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 1405a477bf37..b4c5510d1797 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -306,41 +306,14 @@ def add_third_party_files( add_configuration(configurations, distribution) -class MypyResult(Enum): - SUCCESS = 0 - FAILURE = 1 - CRASH = 2 - - -@dataclass(frozen=True) -class TestResults: - mypy_result: MypyResult = MypyResult.SUCCESS - files_checked: int = 0 - packages_skipped: int = 0 - - @staticmethod - def from_code(mypy_result_code: int, files_checked: int = 0) -> TestResults: - if mypy_result_code == 0: - result = MypyResult.SUCCESS - elif mypy_result_code == 1: - result = MypyResult.FAILURE - else: - result = MypyResult.CRASH - return TestResults(result, files_checked) - - def combine(self, other: TestResults) -> TestResults: - result = self.mypy_result if self.mypy_result.value > other.mypy_result.value else other.mypy_result - checked = self.files_checked + other.files_checked - skipped = self.packages_skipped + other.packages_skipped - return TestResults(result, checked, skipped) - - def skip_package(self) -> TestResults: - return TestResults(self.mypy_result, self.files_checked, self.packages_skipped + 1) +class TestResult(NamedTuple): + mypy_return_code: int + files_checked: int def test_third_party_distribution( distribution: str, args: TestConfig, venv_info: VenvInfo, *, non_types_dependencies: bool -) -> TestResults: +) -> TestResult: """Test the stubs of a third-party distribution. Return a tuple, where the first element indicates mypy's return code @@ -353,7 +326,7 @@ def test_third_party_distribution( add_third_party_files(distribution, files, args, configurations, seen_dists) if not files and args.filter: - return TestResults() + return TestResult(0, 0) print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True) @@ -373,10 +346,10 @@ def test_third_party_distribution( testing_stdlib=False, non_types_dependencies=non_types_dependencies, ) - return TestResults.from_code(code, len(files)) + return TestResult(code, len(files)) -def test_stdlib(args: TestConfig) -> TestResults: +def test_stdlib(args: TestConfig) -> TestResult: files: list[Path] = [] stdlib = Path("stdlib") supported_versions = parse_versions(stdlib / "VERSIONS") @@ -389,13 +362,49 @@ def test_stdlib(args: TestConfig) -> TestResults: add_files(files, (stdlib / name), args) if not files: - return TestResults() + return TestResult(0, 0) print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) # We don't actually need pip for the stdlib testing venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) - return TestResults.from_code(code, len(files)) + return TestResult(code, len(files)) + + +class MypyResult(Enum): + SUCCESS = 0 + FAILURE = 1 + CRASH = 2 + + +@dataclass(frozen=True) +class TestSummary: + mypy_result: MypyResult = MypyResult.SUCCESS + files_checked: int = 0 + packages_skipped: int = 0 + packages_with_errors: int = 0 + + def register_result(self, mypy_result_code: int, files_checked: int) -> TestSummary: + if mypy_result_code == 0: + result = MypyResult.SUCCESS + errors = 0 + elif mypy_result_code == 1: + result = MypyResult.FAILURE + errors = 1 + else: + result = MypyResult.CRASH + errors = 1 + return self.combine(TestSummary(result, files_checked, 0, errors)) + + def combine(self, other: TestSummary) -> TestSummary: + result = self.mypy_result if self.mypy_result.value > other.mypy_result.value else other.mypy_result + checked = self.files_checked + other.files_checked + skipped = self.packages_skipped + other.packages_skipped + errors = self.packages_with_errors + other.packages_with_errors + return TestSummary(result, checked, skipped, errors) + + def skip_package(self) -> TestSummary: + return TestSummary(self.mypy_result, self.files_checked, self.packages_skipped + 1) _PRINT_LOCK = Lock() @@ -500,9 +509,9 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar _DISTRIBUTION_TO_VENV_MAPPING.update(dict.fromkeys(distribution_list, venv_to_use)) -def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestResults: +def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: print("Testing third-party packages...") - results = TestResults() + results = TestSummary() gitignore_spec = get_gitignore_spec() distributions_to_check: dict[str, PackageDependencies] = {} @@ -553,27 +562,27 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestResults: for distribution in distributions_to_check: venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution] non_types_dependencies = venv_info.python_exe != sys.executable - this_results = test_third_party_distribution( + code, files_checked = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - results = results.combine(this_results) + results = results.register_result(code, files_checked) return results -def test_typeshed(args: TestConfig, tempdir: Path) -> TestResults: +def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary: print(f"*** Testing Python {args.version} on {args.platform}") stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") - results = TestResults() + results = TestSummary() if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - stdlib_results = test_stdlib(args) - results = results.combine(stdlib_results) + code, files_checked = test_stdlib(args) + results = results.register_result(code, files_checked) print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): - third_party_results = test_third_party_stubs(args, tempdir) - results = results.combine(third_party_results) + tp_results = test_third_party_stubs(args, tempdir) + results = results.combine(tp_results) print() return results @@ -585,7 +594,7 @@ def main() -> None: platforms = args.platform or [sys.platform] filter = args.filter or DIRECTORIES_TO_TEST exclude = args.exclude or [] - results = TestResults() + results = TestSummary() with tempfile.TemporaryDirectory() as td: td_path = Path(td) for version, platform in product(versions, platforms): @@ -594,8 +603,11 @@ def main() -> None: results = results.combine(this_results) if results.mypy_result == MypyResult.FAILURE: - plural = "" if results.files_checked == 1 else "s" - print_error(f"--- error, {results.files_checked} file{plural} checked ---") + plural1 = "" if results.packages_with_errors == 1 else "s" + plural2 = "" if results.files_checked == 1 else "s" + print_error( + f"--- {results.packages_with_errors} package{plural1} with errors, {results.files_checked} file{plural2} checked ---" + ) sys.exit(1) if results.mypy_result == MypyResult.CRASH: plural = "" if results.files_checked == 1 else "s" From 3d9190515e36b6850d6fc334083bb50da6d79f3c Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 13:37:42 +0200 Subject: [PATCH 08/11] Return MypyResult from run_mypy() --- tests/mypy_test.py | 58 +++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index b4c5510d1797..b107228af66d 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -52,7 +52,6 @@ SUPPORTED_PLATFORMS = ("linux", "win32", "darwin") DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")] -ReturnCode: TypeAlias = int VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"] VersionTuple: TypeAlias = Tuple[int, int] Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"] @@ -223,6 +222,12 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) -> configurations.append(MypyDistConf(module_name, values.copy())) +class MypyResult(Enum): + SUCCESS = 0 + FAILURE = 1 + CRASH = 2 + + def run_mypy( args: TestConfig, configurations: list[MypyDistConf], @@ -232,7 +237,7 @@ def run_mypy( non_types_dependencies: bool, venv_info: VenvInfo, mypypath: str | None = None, -) -> ReturnCode: +) -> MypyResult: env_vars = dict(os.environ) if mypypath is not None: env_vars["MYPYPATH"] = mypypath @@ -288,7 +293,12 @@ def run_mypy( print() else: print_success_msg() - return result.returncode + if result.returncode == 0: + return MypyResult.SUCCESS + elif result.returncode == 1: + return MypyResult.FAILURE + else: + return MypyResult.CRASH def add_third_party_files( @@ -307,7 +317,7 @@ def add_third_party_files( class TestResult(NamedTuple): - mypy_return_code: int + mypy_result: MypyResult files_checked: int @@ -326,7 +336,7 @@ def test_third_party_distribution( add_third_party_files(distribution, files, args, configurations, seen_dists) if not files and args.filter: - return TestResult(0, 0) + return TestResult(MypyResult.SUCCESS, 0) print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True) @@ -337,7 +347,7 @@ def test_third_party_distribution( mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists) if args.verbose: print(colored(f"\nMYPYPATH={mypypath}", "blue")) - code = run_mypy( + result = run_mypy( args, configurations, files, @@ -346,7 +356,7 @@ def test_third_party_distribution( testing_stdlib=False, non_types_dependencies=non_types_dependencies, ) - return TestResult(code, len(files)) + return TestResult(result, len(files)) def test_stdlib(args: TestConfig) -> TestResult: @@ -362,19 +372,13 @@ def test_stdlib(args: TestConfig) -> TestResult: add_files(files, (stdlib / name), args) if not files: - return TestResult(0, 0) + return TestResult(MypyResult.SUCCESS, 0) print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) # We don't actually need pip for the stdlib testing venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) - code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) - return TestResult(code, len(files)) - - -class MypyResult(Enum): - SUCCESS = 0 - FAILURE = 1 - CRASH = 2 + result = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) + return TestResult(result, len(files)) @dataclass(frozen=True) @@ -384,17 +388,9 @@ class TestSummary: packages_skipped: int = 0 packages_with_errors: int = 0 - def register_result(self, mypy_result_code: int, files_checked: int) -> TestSummary: - if mypy_result_code == 0: - result = MypyResult.SUCCESS - errors = 0 - elif mypy_result_code == 1: - result = MypyResult.FAILURE - errors = 1 - else: - result = MypyResult.CRASH - errors = 1 - return self.combine(TestSummary(result, files_checked, 0, errors)) + def register_result(self, mypy_result: MypyResult, files_checked: int) -> TestSummary: + errors = 1 if mypy_result != MypyResult.SUCCESS else 0 + return self.combine(TestSummary(mypy_result, files_checked, 0, errors)) def combine(self, other: TestSummary) -> TestSummary: result = self.mypy_result if self.mypy_result.value > other.mypy_result.value else other.mypy_result @@ -562,10 +558,10 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: for distribution in distributions_to_check: venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution] non_types_dependencies = venv_info.python_exe != sys.executable - code, files_checked = test_third_party_distribution( + mypy_result, files_checked = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - results = results.register_result(code, files_checked) + results = results.register_result(mypy_result, files_checked) return results @@ -576,8 +572,8 @@ def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary: results = TestSummary() if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - code, files_checked = test_stdlib(args) - results = results.register_result(code, files_checked) + mypy_result, files_checked = test_stdlib(args) + results = results.register_result(mypy_result, files_checked) print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): From 3b031ea4ede918649f7a205db911fee4066c1831 Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 13:39:47 +0200 Subject: [PATCH 09/11] Reorder methods --- tests/mypy_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index b107228af66d..5d86e733a503 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -392,6 +392,9 @@ def register_result(self, mypy_result: MypyResult, files_checked: int) -> TestSu errors = 1 if mypy_result != MypyResult.SUCCESS else 0 return self.combine(TestSummary(mypy_result, files_checked, 0, errors)) + def skip_package(self) -> TestSummary: + return TestSummary(self.mypy_result, self.files_checked, self.packages_skipped + 1) + def combine(self, other: TestSummary) -> TestSummary: result = self.mypy_result if self.mypy_result.value > other.mypy_result.value else other.mypy_result checked = self.files_checked + other.files_checked @@ -399,9 +402,6 @@ def combine(self, other: TestSummary) -> TestSummary: errors = self.packages_with_errors + other.packages_with_errors return TestSummary(result, checked, skipped, errors) - def skip_package(self) -> TestSummary: - return TestSummary(self.mypy_result, self.files_checked, self.packages_skipped + 1) - _PRINT_LOCK = Lock() _DISTRIBUTION_TO_VENV_MAPPING: dict[str, VenvInfo] = {} From 23e3d0db786ddc8fce3312e7ab259a16a1c2cb9e Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 13:42:23 +0200 Subject: [PATCH 10/11] Rename variables --- tests/mypy_test.py | 50 +++++++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 5d86e733a503..99fe805dee13 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -507,7 +507,7 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: print("Testing third-party packages...") - results = TestSummary() + summary = TestSummary() gitignore_spec = get_gitignore_spec() distributions_to_check: dict[str, PackageDependencies] = {} @@ -524,12 +524,12 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: f"test is being run using Python {PYTHON_VERSION})" ) print(colored(msg, "yellow")) - results = results.skip_package() + summary = summary.skip_package() continue if not metadata.requires_python.contains(args.version): msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})" print(colored(msg, "yellow")) - results = results.skip_package() + summary = summary.skip_package() continue if ( @@ -561,27 +561,27 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: mypy_result, files_checked = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - results = results.register_result(mypy_result, files_checked) + summary = summary.register_result(mypy_result, files_checked) - return results + return summary def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary: print(f"*** Testing Python {args.version} on {args.platform}") stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") - results = TestSummary() + summary = TestSummary() if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): mypy_result, files_checked = test_stdlib(args) - results = results.register_result(mypy_result, files_checked) + summary = summary.register_result(mypy_result, files_checked) print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): tp_results = test_third_party_stubs(args, tempdir) - results = results.combine(tp_results) + summary = summary.combine(tp_results) print() - return results + return summary def main() -> None: @@ -590,31 +590,31 @@ def main() -> None: platforms = args.platform or [sys.platform] filter = args.filter or DIRECTORIES_TO_TEST exclude = args.exclude or [] - results = TestSummary() + summary = TestSummary() with tempfile.TemporaryDirectory() as td: td_path = Path(td) for version, platform in product(versions, platforms): config = TestConfig(args.verbose, filter, exclude, version, platform) - this_results = test_typeshed(args=config, tempdir=td_path) - results = results.combine(this_results) + version_summary = test_typeshed(args=config, tempdir=td_path) + summary = summary.combine(version_summary) - if results.mypy_result == MypyResult.FAILURE: - plural1 = "" if results.packages_with_errors == 1 else "s" - plural2 = "" if results.files_checked == 1 else "s" + if summary.mypy_result == MypyResult.FAILURE: + plural1 = "" if summary.packages_with_errors == 1 else "s" + plural2 = "" if summary.files_checked == 1 else "s" print_error( - f"--- {results.packages_with_errors} package{plural1} with errors, {results.files_checked} file{plural2} checked ---" + f"--- {summary.packages_with_errors} package{plural1} with errors, {summary.files_checked} file{plural2} checked ---" ) sys.exit(1) - if results.mypy_result == MypyResult.CRASH: - plural = "" if results.files_checked == 1 else "s" - print_error(f"--- mypy crashed, {results.files_checked} file{plural} checked ---") + if summary.mypy_result == MypyResult.CRASH: + plural = "" if summary.files_checked == 1 else "s" + print_error(f"--- mypy crashed, {summary.files_checked} file{plural} checked ---") sys.exit(2) - if results.packages_skipped: - plural = "" if results.packages_skipped == 1 else "s" - print(colored(f"--- {results.packages_skipped} package{plural} skipped ---", "yellow")) - if results.files_checked: - plural = "" if results.files_checked == 1 else "s" - print(colored(f"--- success, {results.files_checked} file{plural} checked ---", "green")) + if summary.packages_skipped: + plural = "" if summary.packages_skipped == 1 else "s" + print(colored(f"--- {summary.packages_skipped} package{plural} skipped ---", "yellow")) + if summary.files_checked: + plural = "" if summary.files_checked == 1 else "s" + print(colored(f"--- success, {summary.files_checked} file{plural} checked ---", "green")) else: print_error("--- nothing to do; exit 1 ---") sys.exit(1) From e7e4b7d9407683bf539be4b58079627e9865e41e Mon Sep 17 00:00:00 2001 From: Sebastian Rittau Date: Wed, 11 Oct 2023 14:16:04 +0200 Subject: [PATCH 11/11] Unfreeze TestSummary --- tests/mypy_test.py | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 99fe805dee13..81ff7a81a080 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -381,26 +381,29 @@ def test_stdlib(args: TestConfig) -> TestResult: return TestResult(result, len(files)) -@dataclass(frozen=True) +@dataclass class TestSummary: mypy_result: MypyResult = MypyResult.SUCCESS files_checked: int = 0 packages_skipped: int = 0 packages_with_errors: int = 0 - def register_result(self, mypy_result: MypyResult, files_checked: int) -> TestSummary: - errors = 1 if mypy_result != MypyResult.SUCCESS else 0 - return self.combine(TestSummary(mypy_result, files_checked, 0, errors)) + def register_result(self, mypy_result: MypyResult, files_checked: int) -> None: + if mypy_result.value > self.mypy_result.value: + self.mypy_result = mypy_result + if mypy_result != MypyResult.SUCCESS: + self.packages_with_errors += 1 + self.files_checked += files_checked - def skip_package(self) -> TestSummary: - return TestSummary(self.mypy_result, self.files_checked, self.packages_skipped + 1) + def skip_package(self) -> None: + self.packages_skipped += 1 - def combine(self, other: TestSummary) -> TestSummary: - result = self.mypy_result if self.mypy_result.value > other.mypy_result.value else other.mypy_result - checked = self.files_checked + other.files_checked - skipped = self.packages_skipped + other.packages_skipped - errors = self.packages_with_errors + other.packages_with_errors - return TestSummary(result, checked, skipped, errors) + def merge(self, other: TestSummary) -> None: + if other.mypy_result.value > self.mypy_result.value: + self.mypy_result = other.mypy_result + self.files_checked += other.files_checked + self.packages_skipped += other.packages_skipped + self.packages_with_errors += other.packages_with_errors _PRINT_LOCK = Lock() @@ -524,12 +527,12 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: f"test is being run using Python {PYTHON_VERSION})" ) print(colored(msg, "yellow")) - summary = summary.skip_package() + summary.skip_package() continue if not metadata.requires_python.contains(args.version): msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})" print(colored(msg, "yellow")) - summary = summary.skip_package() + summary.skip_package() continue if ( @@ -561,7 +564,7 @@ def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: mypy_result, files_checked = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - summary = summary.register_result(mypy_result, files_checked) + summary.register_result(mypy_result, files_checked) return summary @@ -573,12 +576,12 @@ def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary: if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): mypy_result, files_checked = test_stdlib(args) - summary = summary.register_result(mypy_result, files_checked) + summary.register_result(mypy_result, files_checked) print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): tp_results = test_third_party_stubs(args, tempdir) - summary = summary.combine(tp_results) + summary.merge(tp_results) print() return summary @@ -596,7 +599,7 @@ def main() -> None: for version, platform in product(versions, platforms): config = TestConfig(args.verbose, filter, exclude, version, platform) version_summary = test_typeshed(args=config, tempdir=td_path) - summary = summary.combine(version_summary) + summary.merge(version_summary) if summary.mypy_result == MypyResult.FAILURE: plural1 = "" if summary.packages_with_errors == 1 else "s"