diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9b5783e015db..b3319cafa6c4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,7 @@ on: pull_request: paths-ignore: - '**/*.md' + - 'scripts/**' permissions: contents: read @@ -65,7 +66,7 @@ jobs: - run: ./tests/pytype_test.py --print-stderr mypy: - name: Test the stubs with mypy + name: Run mypy against the stubs runs-on: ubuntu-latest strategy: matrix: @@ -80,6 +81,17 @@ jobs: - run: pip install -r requirements-tests.txt - run: ./tests/mypy_test.py --platform=${{ matrix.platform }} --python-version=${{ matrix.python-version }} + regression-tests: + name: Run mypy on the test cases + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: "3.10" + - run: pip install -r requirements-tests.txt + - run: python ./tests/regr_test.py --all + pyright: name: Test the stubs with pyright runs-on: ubuntu-latest diff --git a/stubs/requests/@tests/test_cases/check_post.py b/stubs/requests/@tests/test_cases/check_post.py new file mode 100644 index 000000000000..b38ed8adc0fe --- /dev/null +++ b/stubs/requests/@tests/test_cases/check_post.py @@ -0,0 +1,13 @@ +# pyright: reportUnnecessaryTypeIgnoreComment=true + +import requests + +# Regression test for #7988 (multiple files should be allowed for the "files" argument) +# This snippet comes from the requests documentation (https://requests.readthedocs.io/en/latest/user/advanced/#post-multiple-multipart-encoded-files), +# so should pass a type checker without error +url = "https://httpbin.org/post" +multiple_files = [ + ("images", ("foo.png", open("foo.png", "rb"), "image/png")), + ("images", ("bar.png", open("bar.png", "rb"), "image/png")), +] +r = requests.post(url, files=multiple_files) diff --git a/test_cases/README.md b/test_cases/README.md index 1bd45430f677..add87f568906 100644 --- a/test_cases/README.md +++ b/test_cases/README.md @@ -1,7 +1,7 @@ ## Regression tests for typeshed -This directory contains code samples that act as a regression test for the -standard library stubs found elsewhere in the typeshed repo. +This directory contains code samples that act as a regression test for +typeshed's stdlib stubs. **This directory should *only* contain test cases for functions and classes which are known to have caused problems in the past, where the stubs are difficult to @@ -9,6 +9,14 @@ get right.** 100% test coverage for typeshed is neither necessary nor desirable, as it would lead to code duplication. Moreover, typeshed has multiple other mechanisms for spotting errors in the stubs. +### Where are the third-party test cases? + +Not all third-party stubs packages in typeshed have test cases, and not all of +them need test cases. For those that do have test cases, however, the samples +can be found in `@tests/test_cases` subdirectories for each stubs package. For +example, the test cases for `requests` can be found in the +`stubs/requests/@tests/test_cases` directory. + ### The purpose of these tests Different test cases in this directory serve different purposes. For some stubs in diff --git a/tests/README.md b/tests/README.md index c04cc3795b2f..32e848c0c602 100644 --- a/tests/README.md +++ b/tests/README.md @@ -5,6 +5,8 @@ tests the stubs with [mypy](https://github.com/python/mypy/) [pytype](https://github.com/google/pytype/). - `tests/pyright_test.py` tests the stubs with [pyright](https://github.com/microsoft/pyright). +- `tests/regr_test.py` runs mypy against the test cases for typeshed's +stubs, guarding against accidental regressions. - `tests/check_consistent.py` checks certain files in typeshed remain consistent with each other. - `tests/stubtest_stdlib.py` checks standard library stubs against the @@ -24,17 +26,15 @@ Run using: (.venv3)$ python3 tests/mypy_test.py ``` -The test has three parts. Each part uses mypy with slightly different configuration options: -- Running mypy on the stdlib stubs -- Running mypy on the third-party stubs -- Running mypy `--strict` on the regression tests in the `test_cases` directory. +The test has two parts: running mypy on the stdlib stubs, +and running mypy on the third-party stubs. -When running mypy on the stubs, this test is shallow — it verifies that all stubs can be +This test is shallow — it verifies that all stubs can be imported but doesn't check whether stubs match their implementation (in the Python standard library or a third-party package). Run `python tests/mypy_test.py --help` for information on the various configuration options -for this test script. +for this script. ## pytype\_test.py @@ -64,6 +64,13 @@ checks that would typically fail on incomplete stubs (such as `Unknown` checks). In typeshed's CI, pyright is run with these configuration settings on a subset of the stubs in typeshed (including the standard library). +## regr\_test.py + +This test runs mypy against the test cases for typeshed's stdlib and third-party +stubs. See the README in the `test_cases` directory for more information about what +these test cases are for and how they work. Run `python tests/regr_test.py --help` +for information on the various configuration options. + ## check\_consistent.py Run using: diff --git a/tests/check_consistent.py b/tests/check_consistent.py index a4e9fdcb6b9b..32f1c3c70503 100755 --- a/tests/check_consistent.py +++ b/tests/check_consistent.py @@ -15,6 +15,7 @@ from packaging.requirements import Requirement from packaging.specifiers import SpecifierSet from packaging.version import Version +from utils import get_all_testcase_directories metadata_keys = {"version", "requires", "extra_description", "obsolete_since", "no_longer_updated", "tool"} tool_keys = {"stubtest": {"skip", "apt_dependencies", "extras", "ignore_missing_stub"}} @@ -58,10 +59,21 @@ def check_stubs() -> None: def check_test_cases() -> None: - assert_consistent_filetypes(Path("test_cases"), kind=".py", allowed={"README.md"}) - bad_test_case_filename = 'Files in the `test_cases` directory must have names starting with "check_"; got "{}"' - for file in Path("test_cases").rglob("*.py"): - assert file.stem.startswith("check_"), bad_test_case_filename.format(file) + for package_name, testcase_dir in get_all_testcase_directories(): + assert_consistent_filetypes(testcase_dir, kind=".py", allowed={"README.md"}) + bad_test_case_filename = 'Files in a `test_cases` directory must have names starting with "check_"; got "{}"' + for file in testcase_dir.rglob("*.py"): + assert file.stem.startswith("check_"), bad_test_case_filename.format(file) + if package_name != "stdlib": + with open(file) as f: + lines = {line.strip() for line in f} + pyright_setting_not_enabled_msg = ( + f'Third-party test-case file "{file}" must have ' + f'"# pyright: reportUnnecessaryTypeIgnoreComment=true" ' + f"at the top of the file" + ) + has_pyright_setting_enabled = "# pyright: reportUnnecessaryTypeIgnoreComment=true" in lines + assert has_pyright_setting_enabled, pyright_setting_not_enabled_msg def check_no_symlinks() -> None: diff --git a/tests/colors.py b/tests/colors.py deleted file mode 100644 index 9f81be7cf09e..000000000000 --- a/tests/colors.py +++ /dev/null @@ -1,31 +0,0 @@ -""" -Helper module so we don't have to install types-termcolor in CI. - -This is imported by `mypy_test.py` and `stubtest_third_party.py`. -""" - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - - def colored(__str: str, __style: str) -> str: - ... - -else: - try: - from termcolor import colored - except ImportError: - - def colored(s: str, _: str) -> str: - return s - - -def print_error(error: str, end: str = "\n") -> None: - error_split = error.split("\n") - for line in error_split[:-1]: - print(colored(line, "red")) - print(colored(error_split[-1], "red"), end=end) - - -def print_success_msg() -> None: - print(colored("success", "green")) diff --git a/tests/mypy_test.py b/tests/mypy_test.py index bfa8ba44ee60..b648b2b48d04 100755 --- a/tests/mypy_test.py +++ b/tests/mypy_test.py @@ -1,18 +1,13 @@ #!/usr/bin/env python3 -"""Run mypy on various typeshed directories, with varying command-line arguments. +"""Run mypy on typeshed's stdlib and third-party stubs.""" -Depends on mypy being installed. -""" from __future__ import annotations import argparse import os import re -import shutil -import subprocess import sys import tempfile -from collections.abc import Iterable from contextlib import redirect_stderr, redirect_stdout from dataclasses import dataclass from io import StringIO @@ -26,11 +21,11 @@ from typing_extensions import Annotated, TypeAlias import tomli -from colors import colored, print_error, print_success_msg +from utils import colored, print_error, print_success_msg, read_dependencies SUPPORTED_VERSIONS = [(3, 11), (3, 10), (3, 9), (3, 8), (3, 7)] SUPPORTED_PLATFORMS = ("linux", "win32", "darwin") -TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs", "test_cases"}) +TYPESHED_DIRECTORIES = frozenset({"stdlib", "stubs"}) ReturnCode: TypeAlias = int MajorVersion: TypeAlias = int @@ -58,7 +53,9 @@ class CommandLineArgs(argparse.Namespace): filter: list[str] -parser = argparse.ArgumentParser(description="Test runner for typeshed. Patterns are unanchored regexps on the full path.") +parser = argparse.ArgumentParser( + description="Typecheck typeshed's stubs with mypy. Patterns are unanchored regexps on the full path." +) parser.add_argument("-v", "--verbose", action="count", default=0, help="More output") parser.add_argument("-x", "--exclude", type=str, nargs="*", help="Exclude pattern") parser.add_argument( @@ -239,20 +236,8 @@ def run_mypy(args: TestConfig, configurations: list[MypyDistConf], files: list[P return exit_code -def run_mypy_as_subprocess(directory: StrPath, flags: Iterable[str]) -> ReturnCode: - result = subprocess.run([sys.executable, "-m", "mypy", directory, *flags], capture_output=True) - stdout, stderr = result.stdout, result.stderr - if stderr: - print_error(stderr.decode()) - if stdout: - print_error(stdout.decode()) - return result.returncode - - -def get_mypy_flags( - args: TestConfig, temp_name: str | None, *, strict: bool = False, enforce_error_codes: bool = True -) -> list[str]: - flags = [ +def get_mypy_flags(args: TestConfig, temp_name: str) -> list[str]: + return [ "--python-version", f"{args.major}.{args.minor}", "--show-traceback", @@ -264,29 +249,15 @@ def get_mypy_flags( "--no-site-packages", "--custom-typeshed-dir", str(Path(__file__).parent.parent), + "--no-implicit-optional", + "--disallow-untyped-decorators", + "--disallow-any-generics", + "--strict-equality", + "--enable-error-code", + "ignore-without-code", + "--config-file", + temp_name, ] - if strict: - flags.append("--strict") - else: - flags.extend(["--no-implicit-optional", "--disallow-untyped-decorators", "--disallow-any-generics", "--strict-equality"]) - if temp_name is not None: - flags.extend(["--config-file", temp_name]) - if enforce_error_codes: - flags.extend(["--enable-error-code", "ignore-without-code"]) - return flags - - -def read_dependencies(distribution: str) -> list[str]: - with Path("stubs", distribution, "METADATA.toml").open("rb") as f: - data = tomli.load(f) - requires = data.get("requires", []) - assert isinstance(requires, list) - dependencies = [] - for dependency in requires: - assert isinstance(dependency, str) - assert dependency.startswith("types-") - dependencies.append(dependency[6:].split("<")[0]) - return dependencies def add_third_party_files( @@ -382,23 +353,6 @@ def test_third_party_stubs(code: int, args: TestConfig) -> TestResults: return TestResults(code, files_checked) -def test_the_test_cases(code: int, args: TestConfig) -> TestResults: - test_case_files = list(map(str, Path("test_cases").rglob("*.py"))) - num_test_case_files = len(test_case_files) - flags = get_mypy_flags(args, None, strict=True, enforce_error_codes=False) - print(f"Running mypy on the test_cases directory ({num_test_case_files} files)...") - print("Running mypy " + " ".join(flags)) - # --warn-unused-ignores doesn't work for files inside typeshed. - # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory. - with tempfile.TemporaryDirectory() as td: - shutil.copytree(Path("test_cases"), Path(td) / "test_cases") - this_code = run_mypy_as_subprocess(td, flags) - if not this_code: - print_success_msg() - code = max(code, this_code) - return TestResults(code, num_test_case_files) - - def test_typeshed(code: int, args: TestConfig) -> TestResults: print(f"*** Testing Python {args.major}.{args.minor} on {args.platform}") files_checked_this_version = 0 @@ -412,11 +366,6 @@ def test_typeshed(code: int, args: TestConfig) -> TestResults: files_checked_this_version += third_party_files_checked print() - if "test_cases" in args.directories: - code, test_case_files_checked = test_the_test_cases(code, args) - files_checked_this_version += test_case_files_checked - print() - return TestResults(code, files_checked_this_version) diff --git a/tests/regr_test.py b/tests/regr_test.py new file mode 100644 index 000000000000..02f27af80f97 --- /dev/null +++ b/tests/regr_test.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""Run mypy on the test cases for the stdlib and third-party stubs.""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from itertools import filterfalse, product +from pathlib import Path +from typing_extensions import TypeAlias + +from utils import ( + PackageInfo, + colored, + get_all_testcase_directories, + print_error, + print_success_msg, + read_dependencies, + testcase_dir_from_package_name, +) + +ReturnCode: TypeAlias = int + +SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"] +SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"] + + +def package_with_test_cases(package_name: str) -> PackageInfo: + """Helper function for argument-parsing""" + + if package_name == "stdlib": + return PackageInfo("stdlib", Path("test_cases")) + test_case_dir = testcase_dir_from_package_name(package_name) + if test_case_dir.is_dir(): + return PackageInfo(package_name, test_case_dir) + raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!") + + +parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs") +parser.add_argument( + "packages_to_test", + type=package_with_test_cases, + nargs="*", + action="extend", + help="Test only these packages (defaults to all typeshed stubs that have test cases)", +) +parser.add_argument( + "--all", + action="store_true", + help=( + 'Run tests on all available platforms and versions (defaults to "False"). ' + "Note that this cannot be specified if --platform and/or --python-version are specified." + ), +) +parser.add_argument( + "--platform", + dest="platforms_to_test", + choices=SUPPORTED_PLATFORMS, + nargs="*", + action="extend", + help=( + "Run mypy for certain OS platforms (defaults to sys.platform). " + "Note that this cannot be specified if --all is also specified." + ), +) +parser.add_argument( + "-p", + "--python-version", + dest="versions_to_test", + choices=SUPPORTED_VERSIONS, + nargs="*", + action="extend", + help=( + "Run mypy for certain Python versions (defaults to sys.version_info[:2])" + "Note that this cannot be specified if --all is also specified." + ), +) + + +def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> list[str]: + seen = seen if seen is not None else {package_name} + for dependency in filterfalse(seen.__contains__, read_dependencies(package_name)): + seen.update(get_recursive_requirements(dependency, seen)) + return sorted(seen | {package_name}) + + +def test_testcase_directory(package: PackageInfo, version: str, platform: str) -> ReturnCode: + package_name, test_case_directory = package + is_stdlib = package_name == "stdlib" + + msg = f"Running mypy --platform {platform} --python-version {version} on the " + msg += "standard library test cases..." if is_stdlib else f"test cases for {package_name!r}..." + print(msg, end=" ") + + flags = [ + "--python-version", + version, + "--show-traceback", + "--show-error-codes", + "--no-error-summary", + "--platform", + platform, + "--no-site-packages", + "--strict", + ] + + # --warn-unused-ignores doesn't work for files inside typeshed. + # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory. + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + new_test_case_dir = td_path / "test_cases" + shutil.copytree(test_case_directory, new_test_case_dir) + env_vars = dict(os.environ) + if is_stdlib: + flags.extend(["--custom-typeshed-dir", str(Path(__file__).parent.parent)]) + else: + # HACK: we want to run these test cases in an isolated environment -- + # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml + # (and all stub packages required by those stub packages, etc. etc.), + # but none of the other stubs in typeshed. + # + # The best way of doing that without stopping --warn-unused-ignore from working + # seems to be to create a "new typeshed" directory in a tempdir + # that has only the required stubs copied over. + new_typeshed = td_path / "typeshed" + os.mkdir(new_typeshed) + shutil.copytree(Path("stdlib"), new_typeshed / "stdlib") + requirements = get_recursive_requirements(package_name) + for requirement in requirements: + shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement) + env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*"))) + flags.extend(["--custom-typeshed-dir", str(td_path / "typeshed")]) + result = subprocess.run([sys.executable, "-m", "mypy", new_test_case_dir, *flags], capture_output=True, env=env_vars) + + if result.returncode: + print_error("failure\n") + replacements = (str(new_test_case_dir), str(test_case_directory)) + if result.stderr: + print_error(result.stderr.decode(), fix_path=replacements) + if result.stdout: + print_error(result.stdout.decode(), fix_path=replacements) + else: + print_success_msg() + return result.returncode + + +def main() -> ReturnCode: + args = parser.parse_args() + + testcase_directories = args.packages_to_test or get_all_testcase_directories() + if args.all: + if args.platforms_to_test: + raise TypeError("Cannot specify both --platform and --all") + if args.versions_to_test: + raise TypeError("Cannot specify both --python-version and --all") + platforms_to_test, versions_to_test = SUPPORTED_PLATFORMS, SUPPORTED_VERSIONS + else: + platforms_to_test = args.platforms_to_test or [sys.platform] + versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"] + + code = 0 + for platform, version, directory in product(platforms_to_test, versions_to_test, testcase_directories): + code = max(code, test_testcase_directory(directory, version, platform)) + if code: + print_error("\nTest completed with errors") + else: + print(colored("\nTest completed successfully!", "green")) + + return code + + +if __name__ == "__main__": + try: + code = main() + except KeyboardInterrupt: + print_error("Test aborted due to KeyboardInterrupt!") + code = 1 + raise SystemExit(code) diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py index 17e8240f8d37..cd4a33417c80 100755 --- a/tests/stubtest_third_party.py +++ b/tests/stubtest_third_party.py @@ -13,7 +13,7 @@ from typing import NoReturn import tomli -from colors import colored, print_error, print_success_msg +from utils import colored, print_error, print_success_msg @functools.lru_cache() diff --git a/tests/typecheck_typeshed.py b/tests/typecheck_typeshed.py index e2798ddca004..2106755b5df5 100644 --- a/tests/typecheck_typeshed.py +++ b/tests/typecheck_typeshed.py @@ -8,7 +8,7 @@ from itertools import product from typing_extensions import TypeAlias -from colors import colored, print_error +from utils import colored, print_error ReturnCode: TypeAlias = int diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 000000000000..e941afe36a0d --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,81 @@ +"""Utilities that are imported by multiple scripts in the tests directory.""" + +import os +from functools import cache +from pathlib import Path +from typing import TYPE_CHECKING, NamedTuple + +import tomli + +# ==================================================================== +# Some simple hacks so we don't have to install types-termcolor in CI, +# and so that tests can be run locally without termcolor installed, +# if desired +# ==================================================================== + +if TYPE_CHECKING: + + def colored(__str: str, __style: str) -> str: + ... + +else: + try: + from termcolor import colored + except ImportError: + + def colored(s: str, _: str) -> str: + return s + + +def print_error(error: str, end: str = "\n", fix_path: tuple[str, str] = ("", "")) -> None: + error_split = error.split("\n") + old, new = fix_path + for line in error_split[:-1]: + print(colored(line.replace(old, new), "red")) + print(colored(error_split[-1], "red"), end=end) + + +def print_success_msg() -> None: + print(colored("success", "green")) + + +# ==================================================================== +# Reading dependencies from METADATA.toml files +# ==================================================================== + + +@cache +def read_dependencies(distribution: str) -> tuple[str, ...]: + with Path("stubs", distribution, "METADATA.toml").open("rb") as f: + data = tomli.load(f) + requires = data.get("requires", []) + assert isinstance(requires, list) + dependencies = [] + for dependency in requires: + assert isinstance(dependency, str) + assert dependency.startswith("types-"), f"unrecognized dependency {dependency!r}" + dependencies.append(dependency[6:].split("<")[0]) + return tuple(dependencies) + + +# ==================================================================== +# Getting test-case directories from package names +# ==================================================================== + + +class PackageInfo(NamedTuple): + name: str + test_case_directory: Path + + +def testcase_dir_from_package_name(package_name: str) -> Path: + return Path("stubs", package_name, "@tests/test_cases") + + +def get_all_testcase_directories() -> list[PackageInfo]: + testcase_directories = [PackageInfo("stdlib", Path("test_cases"))] + for package_name in os.listdir("stubs"): + potential_testcase_dir = testcase_dir_from_package_name(package_name) + if potential_testcase_dir.is_dir(): + testcase_directories.append(PackageInfo(package_name, potential_testcase_dir)) + return sorted(testcase_directories)