Improve the output of tests/stubtest_third_party.py when it fails (#11763)

Author: AlexWaygood

scripts/stubsabot.py

@@ -498,6 +498,7 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U
         "Release": f"{pypi_info.pypi_root}/{relevant_version}",
         "Homepage": project_urls.get("Homepage"),
         "Repository": stub_info.upstream_repository,
+        "Typeshed stubs": f"https://github.com/{TYPESHED_OWNER}/typeshed/tree/main/{stub_info.distribution}",
         "Changelog": project_urls.get("Changelog") or project_urls.get("Changes") or project_urls.get("Change Log"),
     }
     links = {k: v for k, v in maybe_links.items() if v is not None}

tests/stubtest_third_party.py

@@ -13,7 +13,7 @@
 from typing import NoReturn
 
 from parse_metadata import NoSuchStubError, get_recursive_requirements, read_metadata
-from utils import PYTHON_VERSION, colored, get_mypy_req, print_error, print_success_msg
+from utils import PYTHON_VERSION, colored, get_mypy_req, print_divider, print_error, print_success_msg
 
 
 def run_stubtest(
@@ -24,7 +24,7 @@ def run_stubtest(
         metadata = read_metadata(dist_name)
     except NoSuchStubError as e:
         parser.error(str(e))
-    print(f"{dist_name}... ", end="")
+    print(f"{dist_name}... ", end="", flush=True)
 
     stubtest_settings = metadata.stubtest_settings
     if stubtest_settings.skipped:
@@ -131,28 +131,44 @@ def run_stubtest(
         try:
             subprocess.run(stubtest_cmd, env=stubtest_env, check=True, capture_output=True)
         except subprocess.CalledProcessError as e:
-            print_error("fail")
+            print_error("fail\n")
+
+            print_divider()
+            print("Commands run:")
             print_commands(dist, pip_cmd, stubtest_cmd, mypypath)
+
+            print_divider()
+            print("Command output:\n")
             print_command_output(e)
 
-            print("Python version: ", file=sys.stderr)
+            print_divider()
+            print(f"Upstream repository: {metadata.upstream_repository}")
+            print(f"Typeshed source code: https://github.com/python/typeshed/tree/main/stubs/{dist.name}")
+
+            print("Python version: ", file=sys.stderr, end="", flush=True)
             ret = subprocess.run([sys.executable, "-VV"], capture_output=True)
             print_command_output(ret)
-
             print("Ran with the following environment:", file=sys.stderr)
             ret = subprocess.run([pip_exe, "freeze", "--all"], capture_output=True)
             print_command_output(ret)
 
+            allowlist_path_relative = allowlist_path.relative_to(Path.cwd())
             if allowlist_path.exists():
                 print(
-                    f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_path}', file=sys.stderr
+                    f'To fix "unused allowlist" errors, remove the corresponding entries from {allowlist_path_relative}',
+                    file=sys.stderr,
                 )
                 print(file=sys.stderr)
             else:
-                print(f"Re-running stubtest with --generate-allowlist.\nAdd the following to {allowlist_path}:", file=sys.stderr)
+                print(
+                    f"Re-running stubtest with --generate-allowlist.\nAdd the following to {allowlist_path_relative}:",
+                    file=sys.stderr,
+                )
                 ret = subprocess.run([*stubtest_cmd, "--generate-allowlist"], env=stubtest_env, capture_output=True)
                 print_command_output(ret)
 
+            print_divider()
+
             return False
         else:
             print_success_msg()

tests/utils.py

@@ -56,6 +56,14 @@ def print_success_msg() -> None:
     print(colored("success", "green"))
 
 
+def print_divider() -> None:
+    """Print a row of * symbols across the screen.
+
+    This can be useful to divide terminal output into separate sections.
+    """
+    print("*" * 70)
+
+
 # ====================================================================
 # Dynamic venv creation
 # ====================================================================