From 641c686a0f6007cad435141fb18a412fb7a13b09 Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Sun, 18 Dec 2022 19:29:29 +0000
Subject: [PATCH 01/14] `regr_test.py`: Allow non-types dependencies

---
 tests/mypy_test.py            |   2 +-
 tests/regr_test.py            | 116 ++++++++++++++++++++--------------
 tests/stubtest_third_party.py |  31 ++-------
 tests/utils.py                |  95 +++++++++++++++++++++++-----
 4 files changed, 152 insertions(+), 92 deletions(-)

diff --git a/tests/mypy_test.py b/tests/mypy_test.py
index 6eee3213a037..55f846b3d8f1 100644
--- a/tests/mypy_test.py
+++ b/tests/mypy_test.py
@@ -274,7 +274,7 @@ def add_third_party_files(
     seen_dists.add(distribution)
 
     stubs_dir = Path("stubs")
-    dependencies = get_recursive_requirements(distribution)
+    dependencies = get_recursive_requirements(distribution).typeshed_pkgs
 
     for dependency in dependencies:
         if dependency in seen_dists:
diff --git a/tests/regr_test.py b/tests/regr_test.py
index 87be4f8c8d7a..4493c5c61043 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -18,7 +18,9 @@
     PackageInfo,
     colored,
     get_all_testcase_directories,
+    get_mypy_req,
     get_recursive_requirements,
+    make_venv,
     print_error,
     print_success_msg,
     testcase_dir_from_package_name,
@@ -85,14 +87,66 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
 )
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
-    package_name, test_case_directory = package
-    is_stdlib = package_name == "stdlib"
+def run_testcases(
+    package: PackageInfo, flags: list[str], tmpdir_path: Path, python_minor_version: int
+) -> tuple[Path, subprocess.CompletedProcess[str]]:
+    python_exe = sys.executable
+    new_test_case_dir = tmpdir_path / "test_cases"
+    shutil.copytree(package.test_case_directory, new_test_case_dir)
+    env_vars = dict(os.environ)
+    if package.is_stdlib:
+        flags.extend(["--no-site-packages", "--custom-typeshed-dir", str(Path(__file__).parent.parent)])
+    else:
+        # HACK: we want to run these test cases in an isolated environment --
+        # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
+        # (and all stub packages required by those stub packages, etc. etc.),
+        # but none of the other stubs in typeshed.
+        #
+        # The best way of doing that without stopping --warn-unused-ignore from working
+        # seems to be to create a "new typeshed" directory in a tempdir
+        # that has only the required stubs copied over.
+        new_typeshed = tmpdir_path / "typeshed"
+        new_typeshed.mkdir()
+        shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
+        requirements = get_recursive_requirements(package.name)
+        # mypy refuses to consider a directory a "valid typeshed directory"
+        # unless there's a stubs/mypy-extensions path inside it,
+        # so add that to the list of stubs to copy over to the new directory
+        for requirement in set(requirements.typeshed_pkgs) | {package.name, "mypy-extensions"}:
+            shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
+
+        if requirements.external_pkgs:
+            pip_exe, python_exe = make_venv(tmpdir_path / ".venv")
+            try:
+                subprocess.run([pip_exe, "install", get_mypy_req(), *requirements.external_pkgs], check=True, capture_output=True)
+            except subprocess.CalledProcessError as e:
+                print(e.stderr)
+                raise
+        else:
+            flags.append("--no-site-packages")
+
+        env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
+        flags.extend(["--custom-typeshed-dir", str(new_typeshed)])
+
+    # If the test-case filename ends with -py39,
+    # only run the test if --python-version was set to 3.9 or higher (for example)
+    for path in new_test_case_dir.rglob("*.py"):
+        if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
+            minor_version_required = int(match[1])
+            assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
+            if minor_version_required <= python_minor_version:
+                flags.append(str(path))
+        else:
+            flags.append(str(path))
+
+    return new_test_case_dir, subprocess.run([python_exe, "-m", "mypy", *flags], capture_output=True, text=True, env=env_vars)
 
+
+def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
     msg = f"Running mypy --platform {platform} --python-version {version} on the "
-    msg += "standard library test cases..." if is_stdlib else f"test cases for {package_name!r}..."
+    msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..."
     if not quiet:
-        print(msg, end=" ")
+        print(msg, end=" ", flush=True)
 
     # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
     flags = [
@@ -103,53 +157,17 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
         "--no-error-summary",
         "--platform",
         platform,
-        "--no-site-packages",
         "--strict",
         "--pretty",
     ]
 
     # --warn-unused-ignores doesn't work for files inside typeshed.
-    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory.
+    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
+    # and run the test cases inside of that.
     with tempfile.TemporaryDirectory() as td:
-        td_path = Path(td)
-        new_test_case_dir = td_path / "test_cases"
-        shutil.copytree(test_case_directory, new_test_case_dir)
-        env_vars = dict(os.environ)
-        if is_stdlib:
-            flags.extend(["--custom-typeshed-dir", str(Path(__file__).parent.parent)])
-        else:
-            # HACK: we want to run these test cases in an isolated environment --
-            # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
-            # (and all stub packages required by those stub packages, etc. etc.),
-            # but none of the other stubs in typeshed.
-            #
-            # The best way of doing that without stopping --warn-unused-ignore from working
-            # seems to be to create a "new typeshed" directory in a tempdir
-            # that has only the required stubs copied over.
-            new_typeshed = td_path / "typeshed"
-            os.mkdir(new_typeshed)
-            shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
-            requirements = get_recursive_requirements(package_name)
-            # mypy refuses to consider a directory a "valid typeshed directory"
-            # unless there's a stubs/mypy-extensions path inside it,
-            # so add that to the list of stubs to copy over to the new directory
-            for requirement in requirements + ["mypy-extensions"]:
-                shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
-            env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
-            flags.extend(["--custom-typeshed-dir", str(td_path / "typeshed")])
-
-        # If the test-case filename ends with -py39,
-        # only run the test if --python-version was set to 3.9 or higher (for example)
-        for path in new_test_case_dir.rglob("*.py"):
-            if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
-                minor_version_required = int(match[1])
-                assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
-                if minor_version_required <= int(version.split(".")[1]):
-                    flags.append(str(path))
-            else:
-                flags.append(str(path))
-
-        result = subprocess.run([sys.executable, "-m", "mypy", *flags], capture_output=True, env=env_vars)
+        new_test_case_dir, result = run_testcases(
+            package=package, flags=flags, tmpdir_path=Path(td), python_minor_version=int(version.split(".")[1])
+        )
 
     if result.returncode:
         if quiet:
@@ -158,11 +176,11 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
             # If there are errors, the output is inscrutable if this isn't printed.
             print(msg, end=" ")
         print_error("failure\n")
-        replacements = (str(new_test_case_dir), str(test_case_directory))
+        replacements = (str(new_test_case_dir), str(package.test_case_directory))
         if result.stderr:
-            print_error(result.stderr.decode(), fix_path=replacements)
+            print_error(result.stderr, fix_path=replacements)
         if result.stdout:
-            print_error(result.stdout.decode(), fix_path=replacements)
+            print_error(result.stdout, fix_path=replacements)
     elif not quiet:
         print_success_msg()
     return result.returncode
diff --git a/tests/stubtest_third_party.py b/tests/stubtest_third_party.py
index d9b612396f82..98e8015a30ee 100755
--- a/tests/stubtest_third_party.py
+++ b/tests/stubtest_third_party.py
@@ -4,23 +4,15 @@
 from __future__ import annotations
 
 import argparse
-import functools
 import os
 import subprocess
 import sys
 import tempfile
-import venv
 from pathlib import Path
 from typing import NoReturn
 
 import tomli
-from utils import colored, print_error, print_success_msg
-
-
-@functools.lru_cache()
-def get_mypy_req() -> str:
-    with open("requirements-tests.txt", encoding="UTF-8") as f:
-        return next(line.strip() for line in f if "mypy" in line)
+from utils import colored, get_mypy_req, make_venv, print_error, print_success_msg
 
 
 def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: bool = False) -> bool:
@@ -44,25 +36,10 @@ def run_stubtest(dist: Path, *, verbose: bool = False, specified_stubs_only: boo
     with tempfile.TemporaryDirectory() as tmp:
         venv_dir = Path(tmp)
         try:
-            venv.create(venv_dir, with_pip=True, clear=True)
-        except subprocess.CalledProcessError as e:
-            if "ensurepip" in e.cmd:
-                print_error("fail")
-                print_error(
-                    "stubtest requires a Python installation with ensurepip. "
-                    "If on Linux, you may need to install the python3-venv package."
-                )
+            pip_exe, python_exe = make_venv(venv_dir)
+        except Exception:
+            print_error("fail")
             raise
-
-        if sys.platform == "win32":
-            pip = venv_dir / "Scripts" / "pip.exe"
-            python = venv_dir / "Scripts" / "python.exe"
-        else:
-            pip = venv_dir / "bin" / "pip"
-            python = venv_dir / "bin" / "python"
-
-        pip_exe, python_exe = str(pip), str(python)
-
         dist_version = metadata["version"]
         extras = stubtest_meta.get("extras", [])
         assert isinstance(dist_version, str)
diff --git a/tests/utils.py b/tests/utils.py
index 5c8c9aec5a74..eba5e8706531 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -4,13 +4,17 @@
 
 import os
 import re
+import subprocess
+import sys
+import venv
 from functools import cache
-from itertools import filterfalse
 from pathlib import Path
 from typing import NamedTuple
+from typing_extensions import Annotated
 
 import pathspec  # type: ignore[import]
 import tomli
+from packaging.requirements import Requirement
 
 # Used to install system-wide packages for different OS types:
 METADATA_MAPPING = {"linux": "apt_dependencies", "darwin": "brew_dependencies", "win32": "choco_dependencies"}
@@ -45,25 +49,82 @@ def print_success_msg() -> None:
 # ====================================================================
 
 
+class PackageDependencies(NamedTuple):
+    typeshed_pkgs: tuple[str, ...]
+    external_pkgs: tuple[str, ...]
+
+
 @cache
-def read_dependencies(distribution: str) -> tuple[str, ...]:
+def read_dependencies(distribution: str) -> PackageDependencies:
     with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
         data = tomli.load(f)
-    requires = data.get("requires", [])
-    assert isinstance(requires, list)
-    dependencies = []
-    for dependency in requires:
-        assert isinstance(dependency, str)
-        assert dependency.startswith("types-"), f"unrecognized dependency {dependency!r}"
-        dependencies.append(dependency[6:].split("<")[0])
-    return tuple(dependencies)
+    dependencies = data.get("requires", [])
+    assert isinstance(dependencies, list)
+    typeshed, external = [], []
+    for dependency in dependencies:
+        if dependency.startswith("types-"):
+            maybe_typeshed_dependency = Requirement(dependency).name.removeprefix("types-")
+            if maybe_typeshed_dependency in os.listdir("stubs"):
+                typeshed.append(maybe_typeshed_dependency)
+            else:
+                external.append(dependency)
+        else:
+            external.append(dependency)
+    return PackageDependencies(tuple(typeshed), tuple(external))
+
+
+def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> PackageDependencies:
+    typeshed: set[str] = set()
+    external: set[str] = set()
+    seen = seen if seen is not None else {package_name}
+    non_recursive_requirements = read_dependencies(package_name)
+    typeshed.update(non_recursive_requirements.typeshed_pkgs)
+    external.update(non_recursive_requirements.external_pkgs)
+    for pkg in non_recursive_requirements.typeshed_pkgs:
+        if pkg in seen:
+            continue
+        reqs = get_recursive_requirements(pkg)
+        typeshed.update(reqs.typeshed_pkgs)
+        external.update(reqs.external_pkgs)
+        seen.add(pkg)
+    return PackageDependencies(tuple(sorted(typeshed)), tuple(sorted(external)))
 
 
-def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> list[str]:
-    seen = seen if seen is not None else {package_name}
-    for dependency in filterfalse(seen.__contains__, read_dependencies(package_name)):
-        seen.update(get_recursive_requirements(dependency, seen))
-    return sorted(seen | {package_name})
+# ====================================================================
+# Dynamic venv creation
+# ====================================================================
+
+
+class VenvInfo(NamedTuple):
+    pip_exe: Annotated[str, "A path to the venv's pip executable"]
+    python_exe: Annotated[str, "A path to the venv's python executable"]
+
+
+def make_venv(venv_dir: Path) -> VenvInfo:
+    try:
+        venv.create(venv_dir, with_pip=True, clear=True)
+    except subprocess.CalledProcessError as e:
+        if "ensurepip" in e.cmd:
+            print_error(
+                "stubtest requires a Python installation with ensurepip. "
+                "If on Linux, you may need to install the python3-venv package."
+            )
+        raise
+
+    if sys.platform == "win32":
+        pip = venv_dir / "Scripts" / "pip.exe"
+        python = venv_dir / "Scripts" / "python.exe"
+    else:
+        pip = venv_dir / "bin" / "pip"
+        python = venv_dir / "bin" / "python"
+
+    return VenvInfo(str(pip), str(python))
+
+
+@cache
+def get_mypy_req() -> str:
+    with open("requirements-tests.txt", encoding="UTF-8") as f:
+        return next(line.strip() for line in f if "mypy" in line)
 
 
 # ====================================================================
@@ -83,6 +144,10 @@ class PackageInfo(NamedTuple):
     name: str
     test_case_directory: Path
 
+    @property
+    def is_stdlib(self) -> bool:
+        return self.name == "stdlib"
+
 
 def testcase_dir_from_package_name(package_name: str) -> Path:
     return Path("stubs", package_name, "@tests/test_cases")

From c60c379b5e6d9dc9079f25bd3d657122a3994bda Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Sun, 18 Dec 2022 19:35:56 +0000
Subject: [PATCH 02/14] Add some non-types dependencies to see what happens

---
 stubs/protobuf/METADATA.toml | 1 +
 stubs/urllib3/METADATA.toml  | 1 +
 2 files changed, 2 insertions(+)

diff --git a/stubs/protobuf/METADATA.toml b/stubs/protobuf/METADATA.toml
index ad087c2d74b6..93efd72ab916 100644
--- a/stubs/protobuf/METADATA.toml
+++ b/stubs/protobuf/METADATA.toml
@@ -1,2 +1,3 @@
 version = "4.21.*"
+requires = ["tomli"]
 extra_description = "Generated with aid from mypy-protobuf v3.4.0"
diff --git a/stubs/urllib3/METADATA.toml b/stubs/urllib3/METADATA.toml
index 0976018b4a1d..fa3091f85b70 100644
--- a/stubs/urllib3/METADATA.toml
+++ b/stubs/urllib3/METADATA.toml
@@ -1,4 +1,5 @@
 version = "1.26.*"
+requires = ["packaging"]
 
 [tool.stubtest]
 extras = ["socks"]

From 9cb0b225c72831a17805f3d65e6fc777b5a2c29b Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Sun, 18 Dec 2022 19:44:47 +0000
Subject: [PATCH 03/14] See what happens if we use a non-types dependency in a
 test case

---
 stubs/requests/@tests/test_cases/check_post.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/stubs/requests/@tests/test_cases/check_post.py b/stubs/requests/@tests/test_cases/check_post.py
index 59c75395e961..06ef5fb73be3 100644
--- a/stubs/requests/@tests/test_cases/check_post.py
+++ b/stubs/requests/@tests/test_cases/check_post.py
@@ -4,6 +4,9 @@
 
 from collections.abc import Iterable
 
+# For the purposes of testing this PR, we're pretending packaging is a non-types dependency of types-urllib3
+# Since types-urllib3 is a dependency of requests, packaging should be available in the test cases for requests
+import packaging.version
 import requests
 
 # =================================================================================================
@@ -56,3 +59,5 @@ def gen() -> Iterable[bytes]:
 requests.post("http://httpbin.org/anything", data=[("foo", "bar")]).json()["form"]
 requests.post("http://httpbin.org/anything", data=((b"foo", b"bar"),)).json()["form"]
 requests.post("http://httpbin.org/anything", data=(("foo", "bar"),)).json()["form"]
+
+foo: packaging.version.Version

From 4800d608fbbf03dd415644280ce9b7aced4721db Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Sun, 18 Dec 2022 19:51:07 +0000
Subject: [PATCH 04/14] Check the test still fails if the test cases have
 something bad in them

---
 stubs/requests/@tests/test_cases/check_post.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/stubs/requests/@tests/test_cases/check_post.py b/stubs/requests/@tests/test_cases/check_post.py
index 06ef5fb73be3..0132c7d0969d 100644
--- a/stubs/requests/@tests/test_cases/check_post.py
+++ b/stubs/requests/@tests/test_cases/check_post.py
@@ -61,3 +61,4 @@ def gen() -> Iterable[bytes]:
 requests.post("http://httpbin.org/anything", data=(("foo", "bar"),)).json()["form"]
 
 foo: packaging.version.Version
+x = requests.does_not_exist.how_fun()

From 75b874b7a42050c203bf768ab6cc32370597829b Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Sun, 18 Dec 2022 19:58:28 +0000
Subject: [PATCH 05/14] Revert back to the first commit

---
 stubs/protobuf/METADATA.toml                   | 1 -
 stubs/requests/@tests/test_cases/check_post.py | 6 ------
 stubs/urllib3/METADATA.toml                    | 1 -
 3 files changed, 8 deletions(-)

diff --git a/stubs/protobuf/METADATA.toml b/stubs/protobuf/METADATA.toml
index 93efd72ab916..ad087c2d74b6 100644
--- a/stubs/protobuf/METADATA.toml
+++ b/stubs/protobuf/METADATA.toml
@@ -1,3 +1,2 @@
 version = "4.21.*"
-requires = ["tomli"]
 extra_description = "Generated with aid from mypy-protobuf v3.4.0"
diff --git a/stubs/requests/@tests/test_cases/check_post.py b/stubs/requests/@tests/test_cases/check_post.py
index 0132c7d0969d..59c75395e961 100644
--- a/stubs/requests/@tests/test_cases/check_post.py
+++ b/stubs/requests/@tests/test_cases/check_post.py
@@ -4,9 +4,6 @@
 
 from collections.abc import Iterable
 
-# For the purposes of testing this PR, we're pretending packaging is a non-types dependency of types-urllib3
-# Since types-urllib3 is a dependency of requests, packaging should be available in the test cases for requests
-import packaging.version
 import requests
 
 # =================================================================================================
@@ -59,6 +56,3 @@ def gen() -> Iterable[bytes]:
 requests.post("http://httpbin.org/anything", data=[("foo", "bar")]).json()["form"]
 requests.post("http://httpbin.org/anything", data=((b"foo", b"bar"),)).json()["form"]
 requests.post("http://httpbin.org/anything", data=(("foo", "bar"),)).json()["form"]
-
-foo: packaging.version.Version
-x = requests.does_not_exist.how_fun()
diff --git a/stubs/urllib3/METADATA.toml b/stubs/urllib3/METADATA.toml
index fa3091f85b70..0976018b4a1d 100644
--- a/stubs/urllib3/METADATA.toml
+++ b/stubs/urllib3/METADATA.toml
@@ -1,5 +1,4 @@
 version = "1.26.*"
-requires = ["packaging"]
 
 [tool.stubtest]
 extras = ["socks"]

From 65127ff9a59360146f115af01977cd1867f4e2aa Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Mon, 19 Dec 2022 13:32:06 +0000
Subject: [PATCH 06/14] Minor cleanups to the implementation

---
 tests/regr_test.py | 9 ++++++---
 tests/utils.py     | 8 +++-----
 2 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/tests/regr_test.py b/tests/regr_test.py
index 4493c5c61043..aaf95269cebe 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -112,13 +112,14 @@ def run_testcases(
         # mypy refuses to consider a directory a "valid typeshed directory"
         # unless there's a stubs/mypy-extensions path inside it,
         # so add that to the list of stubs to copy over to the new directory
-        for requirement in set(requirements.typeshed_pkgs) | {package.name, "mypy-extensions"}:
+        for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}:
             shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
 
         if requirements.external_pkgs:
             pip_exe, python_exe = make_venv(tmpdir_path / ".venv")
+            pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
             try:
-                subprocess.run([pip_exe, "install", get_mypy_req(), *requirements.external_pkgs], check=True, capture_output=True)
+                subprocess.run(pip_command, check=True, capture_output=True)
             except subprocess.CalledProcessError as e:
                 print(e.stderr)
                 raise
@@ -139,7 +140,9 @@ def run_testcases(
         else:
             flags.append(str(path))
 
-    return new_test_case_dir, subprocess.run([python_exe, "-m", "mypy", *flags], capture_output=True, text=True, env=env_vars)
+    mypy_command = [python_exe, "-m", "mypy"] + flags
+    result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
+    return new_test_case_dir, result
 
 
 def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
diff --git a/tests/utils.py b/tests/utils.py
index eba5e8706531..db6a97aefb41 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -62,6 +62,7 @@ def read_dependencies(distribution: str) -> PackageDependencies:
     assert isinstance(dependencies, list)
     typeshed, external = [], []
     for dependency in dependencies:
+        assert isinstance(dependency, str)
         if dependency.startswith("types-"):
             maybe_typeshed_dependency = Requirement(dependency).name.removeprefix("types-")
             if maybe_typeshed_dependency in os.listdir("stubs"):
@@ -73,20 +74,17 @@ def read_dependencies(distribution: str) -> PackageDependencies:
     return PackageDependencies(tuple(typeshed), tuple(external))
 
 
-def get_recursive_requirements(package_name: str, seen: set[str] | None = None) -> PackageDependencies:
+@cache
+def get_recursive_requirements(package_name: str) -> PackageDependencies:
     typeshed: set[str] = set()
     external: set[str] = set()
-    seen = seen if seen is not None else {package_name}
     non_recursive_requirements = read_dependencies(package_name)
     typeshed.update(non_recursive_requirements.typeshed_pkgs)
     external.update(non_recursive_requirements.external_pkgs)
     for pkg in non_recursive_requirements.typeshed_pkgs:
-        if pkg in seen:
-            continue
         reqs = get_recursive_requirements(pkg)
         typeshed.update(reqs.typeshed_pkgs)
         external.update(reqs.external_pkgs)
-        seen.add(pkg)
     return PackageDependencies(tuple(sorted(typeshed)), tuple(sorted(external)))
 
 

From 239de0504e7968065a1416219e3093460e03ee07 Mon Sep 17 00:00:00 2001
From: Alex Waygood <Alex.Waygood@Gmail.com>
Date: Mon, 19 Dec 2022 14:39:44 +0000
Subject: [PATCH 07/14] Update regr_test.py

---
 tests/regr_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/regr_test.py b/tests/regr_test.py
index aaf95269cebe..6bb2c35d9653 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -119,7 +119,7 @@ def run_testcases(
             pip_exe, python_exe = make_venv(tmpdir_path / ".venv")
             pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
             try:
-                subprocess.run(pip_command, check=True, capture_output=True)
+                subprocess.run(pip_command, check=True, capture_output=True, text=True)
             except subprocess.CalledProcessError as e:
                 print(e.stderr)
                 raise

From 42f722fb75b21b57718bf1eee2566263def97e8a Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Mon, 19 Dec 2022 20:16:50 +0000
Subject: [PATCH 08/14] Improve implementation of the function; add docstrings

---
 tests/utils.py | 42 ++++++++++++++++++++++++++++++++++++------
 1 file changed, 36 insertions(+), 6 deletions(-)

diff --git a/tests/utils.py b/tests/utils.py
index db6a97aefb41..176c44f895b4 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -7,6 +7,7 @@
 import subprocess
 import sys
 import venv
+from collections.abc import Mapping
 from functools import cache
 from pathlib import Path
 from typing import NamedTuple
@@ -54,8 +55,32 @@ class PackageDependencies(NamedTuple):
     external_pkgs: tuple[str, ...]
 
 
+@cache
+def get_pypi_name_to_typeshed_name_mapping() -> Mapping[str, str]:
+    stub_name_map = {}
+    for typeshed_name in os.listdir("stubs"):
+        with Path("stubs", typeshed_name, "METADATA.toml").open("rb") as f:
+            pypi_name = tomli.load(f).get("stub_distribution", f"types-{typeshed_name}")
+        assert isinstance(pypi_name, str)
+        stub_name_map[pypi_name] = typeshed_name
+    return stub_name_map
+
+
 @cache
 def read_dependencies(distribution: str) -> PackageDependencies:
+    """Read the dependencies listed in a METADATA.toml file for a stubs package.
+
+    Once the dependencies have been read,
+    determine which dependencies are typeshed-internal dependencies,
+    and which dependencies are external (non-types) dependencies.
+    For typeshed dependencies, translate the "dependency name" into the "package name";
+    for external dependencies, leave them as they are in the METADATA.toml file.
+
+    Note that this function may consider things to be typeshed stubs
+    even if they haven't yet been uploaded to PyPI.
+    If a typeshed stub is removed, this function will consider it to be an external dependency.
+    """
+    pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping()
     with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
         data = tomli.load(f)
     dependencies = data.get("requires", [])
@@ -63,12 +88,9 @@ def read_dependencies(distribution: str) -> PackageDependencies:
     typeshed, external = [], []
     for dependency in dependencies:
         assert isinstance(dependency, str)
-        if dependency.startswith("types-"):
-            maybe_typeshed_dependency = Requirement(dependency).name.removeprefix("types-")
-            if maybe_typeshed_dependency in os.listdir("stubs"):
-                typeshed.append(maybe_typeshed_dependency)
-            else:
-                external.append(dependency)
+        maybe_typeshed_dependency = Requirement(dependency).name
+        if maybe_typeshed_dependency in pypi_name_to_typeshed_name_mapping:
+            typeshed.append(pypi_name_to_typeshed_name_mapping[maybe_typeshed_dependency])
         else:
             external.append(dependency)
     return PackageDependencies(tuple(typeshed), tuple(external))
@@ -76,6 +98,14 @@ def read_dependencies(distribution: str) -> PackageDependencies:
 
 @cache
 def get_recursive_requirements(package_name: str) -> PackageDependencies:
+    """Recursively gather dependencies for a single stubs package.
+
+    For example, if the stubs for `caldav`
+    declare a dependency on typeshed's stubs for `requests`,
+    and the stubs for requests declare a dependency on typeshed's stubs for `urllib3`,
+    `get_recursive_requirements("caldav")` will determine that the stubs for `caldav`
+    have both `requests` and `urllib3` as typeshed-internal dependencies.
+    """
     typeshed: set[str] = set()
     external: set[str] = set()
     non_recursive_requirements = read_dependencies(package_name)

From e9486014c645ff782e6cd12eaf6797f360d621ce Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Tue, 20 Dec 2022 12:12:13 +0000
Subject: [PATCH 09/14] Significantly optimize; more misc cleanups

---
 tests/regr_test.py | 151 ++++++++++++++++++++++++++-------------------
 tests/utils.py     |  20 +++---
 2 files changed, 98 insertions(+), 73 deletions(-)

diff --git a/tests/regr_test.py b/tests/regr_test.py
index 6bb2c35d9653..82102879f4d2 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -16,6 +16,7 @@
 
 from utils import (
     PackageInfo,
+    VenvInfo,
     colored,
     get_all_testcase_directories,
     get_mypy_req,
@@ -28,6 +29,10 @@
 
 ReturnCode: TypeAlias = int
 
+TEST_CASES = "test_cases"
+VENV_DIR = ".venv"
+TYPESHED = "typeshed"
+
 SUPPORTED_PLATFORMS = ["linux", "darwin", "win32"]
 SUPPORTED_VERSIONS = ["3.11", "3.10", "3.9", "3.8", "3.7"]
 
@@ -36,7 +41,7 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
     """Helper function for argument-parsing"""
 
     if package_name == "stdlib":
-        return PackageInfo("stdlib", Path("test_cases"))
+        return PackageInfo("stdlib", Path(TEST_CASES))
     test_case_dir = testcase_dir_from_package_name(package_name)
     if test_case_dir.is_dir():
         if not os.listdir(test_case_dir):
@@ -87,47 +92,79 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
 )
 
 
-def run_testcases(
-    package: PackageInfo, flags: list[str], tmpdir_path: Path, python_minor_version: int
-) -> tuple[Path, subprocess.CompletedProcess[str]]:
-    python_exe = sys.executable
-    new_test_case_dir = tmpdir_path / "test_cases"
+def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: Path) -> None:
+    # --warn-unused-ignores doesn't work for files inside typeshed.
+    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
+    # and run the test cases inside of that.
     shutil.copytree(package.test_case_directory, new_test_case_dir)
+    if package.is_stdlib:
+        return
+
+    # HACK: we want to run these test cases in an isolated environment --
+    # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
+    # (and all stub packages required by those stub packages, etc. etc.),
+    # but none of the other stubs in typeshed.
+    #
+    # The best way of doing that without stopping --warn-unused-ignore from working
+    # seems to be to create a "new typeshed" directory in a tempdir
+    # that has only the required stubs copied over.
+    new_typeshed = tempdir / TYPESHED
+    new_typeshed.mkdir()
+    shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
+    requirements = get_recursive_requirements(package.name)
+    # mypy refuses to consider a directory a "valid typeshed directory"
+    # unless there's a stubs/mypy-extensions path inside it,
+    # so add that to the list of stubs to copy over to the new directory
+    for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}:
+        shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
+
+    if requirements.external_pkgs:
+        pip_exe = make_venv(tempdir / VENV_DIR).pip_exe
+        pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
+        try:
+            subprocess.run(pip_command, check=True, capture_output=True, text=True)
+        except subprocess.CalledProcessError as e:
+            print(e.stderr)
+            raise
+
+
+def run_testcases(package: PackageInfo, version: str, platform: str, *, tempdir: Path) -> subprocess.CompletedProcess[str]:
     env_vars = dict(os.environ)
+    new_test_case_dir = tempdir / TEST_CASES
+    testcasedir_already_setup = new_test_case_dir.exists() and new_test_case_dir.is_dir()
+
+    if not testcasedir_already_setup:
+        setup_testcase_dir(package, tempdir=tempdir, new_test_case_dir=new_test_case_dir)
+
+    # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
+    flags = [
+        "--python-version",
+        version,
+        "--show-traceback",
+        "--show-error-codes",
+        "--no-error-summary",
+        "--platform",
+        platform,
+        "--strict",
+        "--pretty",
+        "--no-incremental",
+    ]
+
     if package.is_stdlib:
-        flags.extend(["--no-site-packages", "--custom-typeshed-dir", str(Path(__file__).parent.parent)])
+        python_exe = sys.executable
+        custom_typeshed = Path(__file__).parent.parent
+        flags.append("--no-site-packages")
     else:
-        # HACK: we want to run these test cases in an isolated environment --
-        # we want mypy to see all stub packages listed in the "requires" field of METADATA.toml
-        # (and all stub packages required by those stub packages, etc. etc.),
-        # but none of the other stubs in typeshed.
-        #
-        # The best way of doing that without stopping --warn-unused-ignore from working
-        # seems to be to create a "new typeshed" directory in a tempdir
-        # that has only the required stubs copied over.
-        new_typeshed = tmpdir_path / "typeshed"
-        new_typeshed.mkdir()
-        shutil.copytree(Path("stdlib"), new_typeshed / "stdlib")
-        requirements = get_recursive_requirements(package.name)
-        # mypy refuses to consider a directory a "valid typeshed directory"
-        # unless there's a stubs/mypy-extensions path inside it,
-        # so add that to the list of stubs to copy over to the new directory
-        for requirement in {package.name, *requirements.typeshed_pkgs, "mypy-extensions"}:
-            shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
-
-        if requirements.external_pkgs:
-            pip_exe, python_exe = make_venv(tmpdir_path / ".venv")
-            pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
-            try:
-                subprocess.run(pip_command, check=True, capture_output=True, text=True)
-            except subprocess.CalledProcessError as e:
-                print(e.stderr)
-                raise
+        custom_typeshed = tempdir / TYPESHED
+        env_vars["MYPYPATH"] = os.pathsep.join(map(str, custom_typeshed.glob("stubs/*")))
+        has_non_types_dependencies = (tempdir / VENV_DIR).exists()
+        if has_non_types_dependencies:
+            python_exe = VenvInfo.of_existing_venv(tempdir / VENV_DIR).python_exe
         else:
+            python_exe = sys.executable
             flags.append("--no-site-packages")
 
-        env_vars["MYPYPATH"] = os.pathsep.join(map(str, new_typeshed.glob("stubs/*")))
-        flags.extend(["--custom-typeshed-dir", str(new_typeshed)])
+    flags.extend(["--custom-typeshed-dir", str(custom_typeshed)])
 
     # If the test-case filename ends with -py39,
     # only run the test if --python-version was set to 3.9 or higher (for example)
@@ -135,42 +172,22 @@ def run_testcases(
         if match := re.fullmatch(r".*-py3(\d{1,2})", path.stem):
             minor_version_required = int(match[1])
             assert f"3.{minor_version_required}" in SUPPORTED_VERSIONS
-            if minor_version_required <= python_minor_version:
-                flags.append(str(path))
-        else:
-            flags.append(str(path))
+            python_minor_version = int(version.split(".")[1])
+            if minor_version_required > python_minor_version:
+                continue
+        flags.append(str(path))
 
     mypy_command = [python_exe, "-m", "mypy"] + flags
-    result = subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
-    return new_test_case_dir, result
+    return subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, quiet: bool) -> ReturnCode:
+def test_testcase_directory(package: PackageInfo, version: str, platform: str, *, quiet: bool, tempdir: Path) -> ReturnCode:
     msg = f"Running mypy --platform {platform} --python-version {version} on the "
     msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..."
     if not quiet:
         print(msg, end=" ", flush=True)
 
-    # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
-    flags = [
-        "--python-version",
-        version,
-        "--show-traceback",
-        "--show-error-codes",
-        "--no-error-summary",
-        "--platform",
-        platform,
-        "--strict",
-        "--pretty",
-    ]
-
-    # --warn-unused-ignores doesn't work for files inside typeshed.
-    # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
-    # and run the test cases inside of that.
-    with tempfile.TemporaryDirectory() as td:
-        new_test_case_dir, result = run_testcases(
-            package=package, flags=flags, tmpdir_path=Path(td), python_minor_version=int(version.split(".")[1])
-        )
+    result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir)
 
     if result.returncode:
         if quiet:
@@ -179,7 +196,7 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, q
             # If there are errors, the output is inscrutable if this isn't printed.
             print(msg, end=" ")
         print_error("failure\n")
-        replacements = (str(new_test_case_dir), str(package.test_case_directory))
+        replacements = (str(tempdir / TEST_CASES), str(package.test_case_directory))
         if result.stderr:
             print_error(result.stderr, fix_path=replacements)
         if result.stdout:
@@ -204,8 +221,12 @@ def main() -> ReturnCode:
         versions_to_test = args.versions_to_test or [f"3.{sys.version_info[1]}"]
 
     code = 0
-    for platform, version, directory in product(platforms_to_test, versions_to_test, testcase_directories):
-        code = max(code, test_testcase_directory(directory, version, platform, args.quiet))
+    for testcase_dir in testcase_directories:
+        with tempfile.TemporaryDirectory() as td:
+            tempdir = Path(td)
+            for platform, version in product(platforms_to_test, versions_to_test):
+                this_code = test_testcase_directory(testcase_dir, version, platform, quiet=args.quiet, tempdir=tempdir)
+                code = max(code, this_code)
     if code:
         print_error("\nTest completed with errors")
     else:
diff --git a/tests/utils.py b/tests/utils.py
index 176c44f895b4..d5a3faa11501 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -127,6 +127,17 @@ class VenvInfo(NamedTuple):
     pip_exe: Annotated[str, "A path to the venv's pip executable"]
     python_exe: Annotated[str, "A path to the venv's python executable"]
 
+    @staticmethod
+    def of_existing_venv(venv_dir: Path) -> VenvInfo:
+        if sys.platform == "win32":
+            pip = venv_dir / "Scripts" / "pip.exe"
+            python = venv_dir / "Scripts" / "python.exe"
+        else:
+            pip = venv_dir / "bin" / "pip"
+            python = venv_dir / "bin" / "python"
+
+        return VenvInfo(str(pip), str(python))
+
 
 def make_venv(venv_dir: Path) -> VenvInfo:
     try:
@@ -139,14 +150,7 @@ def make_venv(venv_dir: Path) -> VenvInfo:
             )
         raise
 
-    if sys.platform == "win32":
-        pip = venv_dir / "Scripts" / "pip.exe"
-        python = venv_dir / "Scripts" / "python.exe"
-    else:
-        pip = venv_dir / "bin" / "pip"
-        python = venv_dir / "bin" / "python"
-
-    return VenvInfo(str(pip), str(python))
+    return VenvInfo.of_existing_venv(venv_dir)
 
 
 @cache

From 9823065d1d3c8d369a02067a9fa5c44d5afa5ce1 Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Tue, 20 Dec 2022 12:38:24 +0000
Subject: [PATCH 10/14] Improve logging configurability

---
 .github/workflows/tests.yml |  2 +-
 tests/regr_test.py          | 56 +++++++++++++++++++++++++++++--------
 2 files changed, 46 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 44f2c756c886..e5285416405c 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -100,7 +100,7 @@ jobs:
           cache: pip
           cache-dependency-path: requirements-tests.txt
       - run: pip install -r requirements-tests.txt
-      - run: python ./tests/regr_test.py --all --quiet
+      - run: python ./tests/regr_test.py --all --verbosity QUIET
 
   pyright:
     name: Test typeshed with pyright
diff --git a/tests/regr_test.py b/tests/regr_test.py
index 82102879f4d2..1273d52f070d 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -10,6 +10,7 @@
 import subprocess
 import sys
 import tempfile
+from enum import IntEnum
 from itertools import product
 from pathlib import Path
 from typing_extensions import TypeAlias
@@ -50,6 +51,12 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
     raise argparse.ArgumentTypeError(f"No test cases found for {package_name!r}!")
 
 
+class Verbosity(IntEnum):
+    QUIET = 0
+    NORMAL = 1
+    VERBOSE = 2
+
+
 parser = argparse.ArgumentParser(description="Script to run mypy against various test cases for typeshed's stubs")
 parser.add_argument(
     "packages_to_test",
@@ -66,7 +73,12 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
         "Note that this cannot be specified if --platform and/or --python-version are specified."
     ),
 )
-parser.add_argument("--quiet", action="store_true", help="Print less output to the terminal")
+parser.add_argument(
+    "--verbosity",
+    choices=[member.name for member in Verbosity],
+    default=Verbosity.NORMAL.name,
+    help="Control how much output to print to the terminal",
+)
 parser.add_argument(
     "--platform",
     dest="platforms_to_test",
@@ -92,7 +104,13 @@ def package_with_test_cases(package_name: str) -> PackageInfo:
 )
 
 
-def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: Path) -> None:
+def verbose_log(msg: str) -> None:
+    print(colored("\n" + msg, "blue"))
+
+
+def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: Path, verbosity: Verbosity) -> None:
+    if verbosity is verbosity.VERBOSE:
+        verbose_log(f"Setting up testcase dir in {tempdir}")
     # --warn-unused-ignores doesn't work for files inside typeshed.
     # SO, to work around this, we copy the test_cases directory into a TemporaryDirectory,
     # and run the test cases inside of that.
@@ -119,8 +137,12 @@ def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: P
         shutil.copytree(Path("stubs", requirement), new_typeshed / "stubs" / requirement)
 
     if requirements.external_pkgs:
+        if verbosity is Verbosity.VERBOSE:
+            verbose_log(f"Setting up venv in {tempdir / VENV_DIR}")
         pip_exe = make_venv(tempdir / VENV_DIR).pip_exe
         pip_command = [pip_exe, "install", get_mypy_req(), *requirements.external_pkgs]
+        if verbosity is Verbosity.VERBOSE:
+            verbose_log(f"{pip_command=}")
         try:
             subprocess.run(pip_command, check=True, capture_output=True, text=True)
         except subprocess.CalledProcessError as e:
@@ -128,15 +150,18 @@ def setup_testcase_dir(package: PackageInfo, tempdir: Path, new_test_case_dir: P
             raise
 
 
-def run_testcases(package: PackageInfo, version: str, platform: str, *, tempdir: Path) -> subprocess.CompletedProcess[str]:
+def run_testcases(
+    package: PackageInfo, version: str, platform: str, *, tempdir: Path, verbosity: Verbosity
+) -> subprocess.CompletedProcess[str]:
     env_vars = dict(os.environ)
     new_test_case_dir = tempdir / TEST_CASES
     testcasedir_already_setup = new_test_case_dir.exists() and new_test_case_dir.is_dir()
 
     if not testcasedir_already_setup:
-        setup_testcase_dir(package, tempdir=tempdir, new_test_case_dir=new_test_case_dir)
+        setup_testcase_dir(package, tempdir=tempdir, new_test_case_dir=new_test_case_dir, verbosity=verbosity)
 
-    # "--enable-error-code ignore-without-code" is purposefully ommited. See https://github.com/python/typeshed/pull/8083
+    # "--enable-error-code ignore-without-code" is purposefully ommited.
+    # See https://github.com/python/typeshed/pull/8083
     flags = [
         "--python-version",
         version,
@@ -178,19 +203,27 @@ def run_testcases(package: PackageInfo, version: str, platform: str, *, tempdir:
         flags.append(str(path))
 
     mypy_command = [python_exe, "-m", "mypy"] + flags
+    if verbosity is Verbosity.VERBOSE:
+        verbose_log(f"\n{mypy_command=}")
+        if "MYPYPATH" in env_vars:
+            verbose_log(f"{env_vars['MYPYPATH']=}")
+        else:
+            verbose_log("MYPYPATH not set")
     return subprocess.run(mypy_command, capture_output=True, text=True, env=env_vars)
 
 
-def test_testcase_directory(package: PackageInfo, version: str, platform: str, *, quiet: bool, tempdir: Path) -> ReturnCode:
+def test_testcase_directory(
+    package: PackageInfo, version: str, platform: str, *, verbosity: Verbosity, tempdir: Path
+) -> ReturnCode:
     msg = f"Running mypy --platform {platform} --python-version {version} on the "
     msg += "standard library test cases..." if package.is_stdlib else f"test cases for {package.name!r}..."
-    if not quiet:
+    if verbosity > Verbosity.QUIET:
         print(msg, end=" ", flush=True)
 
-    result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir)
+    result = run_testcases(package=package, version=version, platform=platform, tempdir=tempdir, verbosity=verbosity)
 
     if result.returncode:
-        if quiet:
+        if verbosity > Verbosity.QUIET:
             # We'll already have printed this if --quiet wasn't passed.
             # If--quiet was passed, only print this if there were errors.
             # If there are errors, the output is inscrutable if this isn't printed.
@@ -201,7 +234,7 @@ def test_testcase_directory(package: PackageInfo, version: str, platform: str, *
             print_error(result.stderr, fix_path=replacements)
         if result.stdout:
             print_error(result.stdout, fix_path=replacements)
-    elif not quiet:
+    elif verbosity > Verbosity.QUIET:
         print_success_msg()
     return result.returncode
 
@@ -210,6 +243,7 @@ def main() -> ReturnCode:
     args = parser.parse_args()
 
     testcase_directories = args.packages_to_test or get_all_testcase_directories()
+    verbosity = Verbosity[args.verbosity]
     if args.all:
         if args.platforms_to_test:
             parser.error("Cannot specify both --platform and --all")
@@ -225,7 +259,7 @@ def main() -> ReturnCode:
         with tempfile.TemporaryDirectory() as td:
             tempdir = Path(td)
             for platform, version in product(platforms_to_test, versions_to_test):
-                this_code = test_testcase_directory(testcase_dir, version, platform, quiet=args.quiet, tempdir=tempdir)
+                this_code = test_testcase_directory(testcase_dir, version, platform, verbosity=verbosity, tempdir=tempdir)
                 code = max(code, this_code)
     if code:
         print_error("\nTest completed with errors")

From 40333ec89ab1e8bb3e678c013a583643c19eae77 Mon Sep 17 00:00:00 2001
From: AlexWaygood <alex.waygood@gmail.com>
Date: Tue, 20 Dec 2022 12:56:35 +0000
Subject: [PATCH 11/14] .

---
 tests/regr_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/regr_test.py b/tests/regr_test.py
index 1273d52f070d..c11437fa081c 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -204,7 +204,7 @@ def run_testcases(
 
     mypy_command = [python_exe, "-m", "mypy"] + flags
     if verbosity is Verbosity.VERBOSE:
-        verbose_log(f"\n{mypy_command=}")
+        verbose_log(f"{mypy_command=}")
         if "MYPYPATH" in env_vars:
             verbose_log(f"{env_vars['MYPYPATH']=}")
         else:

From e888c59dc641ff3790b69e4f0037f4170aba90e6 Mon Sep 17 00:00:00 2001
From: Alex Waygood <Alex.Waygood@Gmail.com>
Date: Tue, 20 Dec 2022 15:07:06 +0000
Subject: [PATCH 12/14] Update utils.py

---
 tests/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/utils.py b/tests/utils.py
index d5a3faa11501..cdfba98000b9 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -82,8 +82,7 @@ def read_dependencies(distribution: str) -> PackageDependencies:
     """
     pypi_name_to_typeshed_name_mapping = get_pypi_name_to_typeshed_name_mapping()
     with Path("stubs", distribution, "METADATA.toml").open("rb") as f:
-        data = tomli.load(f)
-    dependencies = data.get("requires", [])
+        dependencies = tomli.load(f).get("requires", [])
     assert isinstance(dependencies, list)
     typeshed, external = [], []
     for dependency in dependencies:

From f154c103e778edce99d4fc00a07fa0472d6f1438 Mon Sep 17 00:00:00 2001
From: Alex Waygood <Alex.Waygood@Gmail.com>
Date: Wed, 21 Dec 2022 18:58:42 +0000
Subject: [PATCH 13/14] Update tests/regr_test.py

---
 tests/regr_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/regr_test.py b/tests/regr_test.py
index c11437fa081c..60e10c416387 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -172,7 +172,6 @@ def run_testcases(
         platform,
         "--strict",
         "--pretty",
-        "--no-incremental",
     ]
 
     if package.is_stdlib:

From 7077c95f78d58e371cee9c70de68989640905f49 Mon Sep 17 00:00:00 2001
From: Jelle Zijlstra <jelle.zijlstra@gmail.com>
Date: Fri, 23 Dec 2022 13:49:32 -0800
Subject: [PATCH 14/14] Update tests/regr_test.py

---
 tests/regr_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/regr_test.py b/tests/regr_test.py
index 60e10c416387..785a971c076a 100644
--- a/tests/regr_test.py
+++ b/tests/regr_test.py
@@ -224,7 +224,7 @@ def test_testcase_directory(
     if result.returncode:
         if verbosity > Verbosity.QUIET:
             # We'll already have printed this if --quiet wasn't passed.
-            # If--quiet was passed, only print this if there were errors.
+            # If --quiet was passed, only print this if there were errors.
             # If there are errors, the output is inscrutable if this isn't printed.
             print(msg, end=" ")
         print_error("failure\n")