diff --git a/tests/mypy_test.py b/tests/mypy_test.py index 0ff0cd66c1dc..81ff7a81a080 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -13,6 +13,7 @@ import time from collections import defaultdict from dataclasses import dataclass +from enum import Enum from itertools import product from pathlib import Path from threading import Lock @@ -51,7 +52,6 @@ SUPPORTED_PLATFORMS = ("linux", "win32", "darwin") DIRECTORIES_TO_TEST = [Path("stdlib"), Path("stubs")] -ReturnCode: TypeAlias = int VersionString: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_VERSIONS"] VersionTuple: TypeAlias = Tuple[int, int] Platform: TypeAlias = Annotated[str, "Must be one of the entries in SUPPORTED_PLATFORMS"] @@ -222,6 +222,12 @@ def add_configuration(configurations: list[MypyDistConf], distribution: str) -> configurations.append(MypyDistConf(module_name, values.copy())) +class MypyResult(Enum): + SUCCESS = 0 + FAILURE = 1 + CRASH = 2 + + def run_mypy( args: TestConfig, configurations: list[MypyDistConf], @@ -231,7 +237,7 @@ def run_mypy( non_types_dependencies: bool, venv_info: VenvInfo, mypypath: str | None = None, -) -> ReturnCode: +) -> MypyResult: env_vars = dict(os.environ) if mypypath is not None: env_vars["MYPYPATH"] = mypypath @@ -276,7 +282,7 @@ def run_mypy( print(colored(f"running {' '.join(mypy_command)}", "blue")) result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars) if result.returncode: - print_error("failure\n") + print_error(f"failure (exit code {result.returncode})\n") if result.stdout: print_error(result.stdout) if result.stderr: @@ -287,7 +293,12 @@ def run_mypy( print() else: print_success_msg() - return result.returncode + if result.returncode == 0: + return MypyResult.SUCCESS + elif result.returncode == 1: + return MypyResult.FAILURE + else: + return MypyResult.CRASH def add_third_party_files( @@ -305,15 +316,14 @@ def add_third_party_files( add_configuration(configurations, distribution) -class TestResults(NamedTuple): - exit_code: int +class TestResult(NamedTuple): + mypy_result: MypyResult files_checked: int - packages_skipped: int = 0 def test_third_party_distribution( distribution: str, args: TestConfig, venv_info: VenvInfo, *, non_types_dependencies: bool -) -> TestResults: +) -> TestResult: """Test the stubs of a third-party distribution. Return a tuple, where the first element indicates mypy's return code @@ -326,7 +336,7 @@ def test_third_party_distribution( add_third_party_files(distribution, files, args, configurations, seen_dists) if not files and args.filter: - return TestResults(0, 0) + return TestResult(MypyResult.SUCCESS, 0) print(f"testing {distribution} ({len(files)} files)... ", end="", flush=True) @@ -337,7 +347,7 @@ def test_third_party_distribution( mypypath = os.pathsep.join(str(Path("stubs", dist)) for dist in seen_dists) if args.verbose: print(colored(f"\nMYPYPATH={mypypath}", "blue")) - code = run_mypy( + result = run_mypy( args, configurations, files, @@ -346,10 +356,10 @@ def test_third_party_distribution( testing_stdlib=False, non_types_dependencies=non_types_dependencies, ) - return TestResults(code, len(files)) + return TestResult(result, len(files)) -def test_stdlib(code: int, args: TestConfig) -> TestResults: +def test_stdlib(args: TestConfig) -> TestResult: files: list[Path] = [] stdlib = Path("stdlib") supported_versions = parse_versions(stdlib / "VERSIONS") @@ -361,14 +371,39 @@ def test_stdlib(code: int, args: TestConfig) -> TestResults: if module_min_version <= tuple(map(int, args.version.split("."))) <= module_max_version: add_files(files, (stdlib / name), args) - if files: - print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) - # We don't actually need pip for the stdlib testing - venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) - this_code = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) - code = max(code, this_code) + if not files: + return TestResult(MypyResult.SUCCESS, 0) + + print(f"Testing stdlib ({len(files)} files)...", end="", flush=True) + # We don't actually need pip for the stdlib testing + venv_info = VenvInfo(pip_exe="", python_exe=sys.executable) + result = run_mypy(args, [], files, venv_info=venv_info, testing_stdlib=True, non_types_dependencies=False) + return TestResult(result, len(files)) - return TestResults(code, len(files)) + +@dataclass +class TestSummary: + mypy_result: MypyResult = MypyResult.SUCCESS + files_checked: int = 0 + packages_skipped: int = 0 + packages_with_errors: int = 0 + + def register_result(self, mypy_result: MypyResult, files_checked: int) -> None: + if mypy_result.value > self.mypy_result.value: + self.mypy_result = mypy_result + if mypy_result != MypyResult.SUCCESS: + self.packages_with_errors += 1 + self.files_checked += files_checked + + def skip_package(self) -> None: + self.packages_skipped += 1 + + def merge(self, other: TestSummary) -> None: + if other.mypy_result.value > self.mypy_result.value: + self.mypy_result = other.mypy_result + self.files_checked += other.files_checked + self.packages_skipped += other.packages_skipped + self.packages_with_errors += other.packages_with_errors _PRINT_LOCK = Lock() @@ -473,10 +508,9 @@ def setup_virtual_environments(distributions: dict[str, PackageDependencies], ar _DISTRIBUTION_TO_VENV_MAPPING.update(dict.fromkeys(distribution_list, venv_to_use)) -def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestResults: +def test_third_party_stubs(args: TestConfig, tempdir: Path) -> TestSummary: print("Testing third-party packages...") - files_checked = 0 - packages_skipped = 0 + summary = TestSummary() gitignore_spec = get_gitignore_spec() distributions_to_check: dict[str, PackageDependencies] = {} @@ -493,12 +527,12 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe f"test is being run using Python {PYTHON_VERSION})" ) print(colored(msg, "yellow")) - packages_skipped += 1 + summary.skip_package() continue if not metadata.requires_python.contains(args.version): msg = f"skipping {distribution!r} for target Python {args.version} (requires Python {metadata.requires_python})" print(colored(msg, "yellow")) - packages_skipped += 1 + summary.skip_package() continue if ( @@ -527,32 +561,30 @@ def test_third_party_stubs(code: int, args: TestConfig, tempdir: Path) -> TestRe for distribution in distributions_to_check: venv_info = _DISTRIBUTION_TO_VENV_MAPPING[distribution] non_types_dependencies = venv_info.python_exe != sys.executable - this_code, checked, _ = test_third_party_distribution( + mypy_result, files_checked = test_third_party_distribution( distribution, args, venv_info=venv_info, non_types_dependencies=non_types_dependencies ) - code = max(code, this_code) - files_checked += checked + summary.register_result(mypy_result, files_checked) - return TestResults(code, files_checked, packages_skipped) + return summary -def test_typeshed(code: int, args: TestConfig, tempdir: Path) -> TestResults: +def test_typeshed(args: TestConfig, tempdir: Path) -> TestSummary: print(f"*** Testing Python {args.version} on {args.platform}") - files_checked_this_version = 0 - packages_skipped_this_version = 0 stdlib_dir, stubs_dir = Path("stdlib"), Path("stubs") + summary = TestSummary() + if stdlib_dir in args.filter or any(stdlib_dir in path.parents for path in args.filter): - code, stdlib_files_checked, _ = test_stdlib(code, args) - files_checked_this_version += stdlib_files_checked + mypy_result, files_checked = test_stdlib(args) + summary.register_result(mypy_result, files_checked) print() if stubs_dir in args.filter or any(stubs_dir in path.parents for path in args.filter): - code, third_party_files_checked, third_party_packages_skipped = test_third_party_stubs(code, args, tempdir) - files_checked_this_version += third_party_files_checked - packages_skipped_this_version = third_party_packages_skipped + tp_results = test_third_party_stubs(args, tempdir) + summary.merge(tp_results) print() - return TestResults(code, files_checked_this_version, packages_skipped_this_version) + return summary def main() -> None: @@ -561,26 +593,31 @@ def main() -> None: platforms = args.platform or [sys.platform] filter = args.filter or DIRECTORIES_TO_TEST exclude = args.exclude or [] - code = 0 - total_files_checked = 0 - total_packages_skipped = 0 + summary = TestSummary() with tempfile.TemporaryDirectory() as td: td_path = Path(td) for version, platform in product(versions, platforms): config = TestConfig(args.verbose, filter, exclude, version, platform) - code, files_checked_this_version, packages_skipped_this_version = test_typeshed(code, args=config, tempdir=td_path) - total_files_checked += files_checked_this_version - total_packages_skipped += packages_skipped_this_version - if code: - plural = "" if total_files_checked == 1 else "s" - print_error(f"--- exit status {code}, {total_files_checked} file{plural} checked ---") - sys.exit(code) - if total_packages_skipped: - plural = "" if total_packages_skipped == 1 else "s" - print(colored(f"--- {total_packages_skipped} package{plural} skipped ---", "yellow")) - if total_files_checked: - plural = "" if total_files_checked == 1 else "s" - print(colored(f"--- success, {total_files_checked} file{plural} checked ---", "green")) + version_summary = test_typeshed(args=config, tempdir=td_path) + summary.merge(version_summary) + + if summary.mypy_result == MypyResult.FAILURE: + plural1 = "" if summary.packages_with_errors == 1 else "s" + plural2 = "" if summary.files_checked == 1 else "s" + print_error( + f"--- {summary.packages_with_errors} package{plural1} with errors, {summary.files_checked} file{plural2} checked ---" + ) + sys.exit(1) + if summary.mypy_result == MypyResult.CRASH: + plural = "" if summary.files_checked == 1 else "s" + print_error(f"--- mypy crashed, {summary.files_checked} file{plural} checked ---") + sys.exit(2) + if summary.packages_skipped: + plural = "" if summary.packages_skipped == 1 else "s" + print(colored(f"--- {summary.packages_skipped} package{plural} skipped ---", "yellow")) + if summary.files_checked: + plural = "" if summary.files_checked == 1 else "s" + print(colored(f"--- success, {summary.files_checked} file{plural} checked ---", "green")) else: print_error("--- nothing to do; exit 1 ---") sys.exit(1)