Skip to content

Commit e5d52a3

Browse files
AvasamAlexWaygood
Avasam
and
AlexWaygood
authored
Script to run all checks locally (#8798)
Co-authored-by: Alex Waygood <[email protected]>
1 parent 61de308 commit e5d52a3

File tree

3 files changed

+193
-5
lines changed

3 files changed

+193
-5
lines changed

‎scripts/create_baseline_stubs.py

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -171,11 +171,8 @@ def main() -> None:
171171

172172
print("\nDone!\n\nSuggested next steps:")
173173
print(f" 1. Manually review the generated stubs in {stub_dir}")
174-
print(f' 2. Run "MYPYPATH={stub_dir} python3 -m mypy.stubtest {package}" to check the stubs against runtime')
175-
print(f' 3. Run "mypy {stub_dir}" to check for errors')
176-
print(f' 4. Run "black {stub_dir}" and "isort {stub_dir}" (if you\'ve made code changes)')
177-
print(f' 5. Run "flake8 {stub_dir}" to check for e.g. unused imports')
178-
print(" 6. Commit the changes on a new branch and create a typeshed PR")
174+
print(" 2. Optionally run tests and autofixes (see tests/README.md for details")
175+
print(" 3. Commit the changes on a new branch and create a typeshed PR (don't force-push!)")
179176

180177

181178
if __name__ == "__main__":

‎scripts/runtests.py

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
#!/usr/bin/env python3
2+
from __future__ import annotations
3+
4+
import json
5+
import re
6+
import subprocess
7+
import sys
8+
from pathlib import Path
9+
10+
try:
11+
from termcolor import colored
12+
except ImportError:
13+
14+
def colored(text: str, color: str = "") -> str: # type: ignore[misc]
15+
return text
16+
17+
18+
_STRICTER_CONFIG_FILE = "pyrightconfig.stricter.json"
19+
_SUCCESS = colored("Success", "green")
20+
_SKIPPED = colored("Skipped", "yellow")
21+
_FAILED = colored("Failed", "red")
22+
# We're using the oldest supported version because it's the most likely to produce errors
23+
# due to unsupported syntax, feature, or bug in a tool.
24+
_PYTHON_VERSION = "3.7"
25+
26+
27+
def _parse_jsonc(json_text: str) -> str:
28+
# strip comments from the file
29+
lines = [line for line in json_text.split("\n") if not line.strip().startswith("//")]
30+
# strip trailing commas from the file
31+
valid_json = re.sub(r",(\s*?[\}\]])", r"\1", "\n".join(lines))
32+
return valid_json
33+
34+
35+
def _get_strict_params(stub_path: str) -> list[str]:
36+
with open(_STRICTER_CONFIG_FILE) as file:
37+
data = json.loads(_parse_jsonc(file.read()))
38+
if stub_path in data["exclude"]:
39+
return []
40+
return ["-p", _STRICTER_CONFIG_FILE]
41+
42+
43+
def main() -> None:
44+
try:
45+
path = sys.argv[1]
46+
except IndexError:
47+
print("Missing path argument in format <folder>/<stub>", file=sys.stderr)
48+
sys.exit(1)
49+
path_tokens = Path(path).parts
50+
assert len(path_tokens) == 2, "Path argument should be in format <folder>/<stub>"
51+
folder, stub = path_tokens
52+
assert folder in {"stdlib", "stubs"}, "Only the 'stdlib' and 'stubs' folders are supported"
53+
stubtest_result: subprocess.CompletedProcess[bytes] | None = None
54+
pytype_result: subprocess.CompletedProcess[bytes] | None = None
55+
56+
# Run formatters first. Order matters.
57+
print("\nRunning pycln...")
58+
subprocess.run([sys.executable, "-m", "pycln", path, "--all"])
59+
print("\nRunning isort...")
60+
subprocess.run([sys.executable, "-m", "isort", path])
61+
print("\nRunning Black...")
62+
black_result = subprocess.run([sys.executable, "-m", "black", path])
63+
if black_result.returncode == 123:
64+
print("Could not run tests due to an internal error with Black. See above for details.", file=sys.stderr)
65+
sys.exit(black_result.returncode)
66+
67+
print("\nRunning Flake8...")
68+
flake8_result = subprocess.run([sys.executable, "-m", "flake8", path])
69+
70+
print("\nRunning check_consistent.py...")
71+
check_consistent_result = subprocess.run([sys.executable, "tests/check_consistent.py"])
72+
print("\nRunning check_new_syntax.py...")
73+
check_new_syntax_result = subprocess.run([sys.executable, "tests/check_new_syntax.py"])
74+
75+
print(f"\nRunning Pyright on Python {_PYTHON_VERSION}...")
76+
pyright_result = subprocess.run(
77+
[sys.executable, "tests/pyright_test.py", path, "--pythonversion", _PYTHON_VERSION] + _get_strict_params(path),
78+
stderr=subprocess.PIPE,
79+
text=True,
80+
)
81+
if re.match(r"error (runn|find)ing npx", pyright_result.stderr):
82+
print(colored("\nSkipping Pyright tests: npx is not installed or can't be run!", "yellow"))
83+
pyright_returncode = 0
84+
pyright_skipped = True
85+
else:
86+
print(pyright_result.stderr)
87+
pyright_returncode = pyright_result.returncode
88+
pyright_skipped = False
89+
90+
print(f"\nRunning mypy for Python {_PYTHON_VERSION}...")
91+
mypy_result = subprocess.run([sys.executable, "tests/mypy_test.py", path, "--python-version", _PYTHON_VERSION])
92+
# If mypy failed, stubtest will fail without any helpful error
93+
if mypy_result.returncode == 0:
94+
if folder == "stdlib":
95+
print("\nRunning stubtest...")
96+
stubtest_result = subprocess.run([sys.executable, "tests/stubtest_stdlib.py", stub])
97+
else:
98+
run_stubtest_query = (
99+
f"\nRun stubtest for {stub!r} (Y/N)?\n\n"
100+
"NOTE: Running third-party stubtest involves downloading and executing arbitrary code from PyPI.\n"
101+
f"Only run stubtest if you trust the {stub!r} package.\n"
102+
)
103+
run_stubtest_answer = input(colored(run_stubtest_query, "yellow")).lower()
104+
while run_stubtest_answer not in {"yes", "no", "y", "n"}:
105+
run_stubtest_answer = input(colored("Invalid response; please try again.\n", "red")).lower()
106+
if run_stubtest_answer in {"yes", "y"}:
107+
print("\nRunning stubtest.")
108+
stubtest_result = subprocess.run([sys.executable, "tests/stubtest_third_party.py", stub])
109+
else:
110+
print(colored(f"\nSkipping stubtest for {stub!r}...", "yellow"))
111+
else:
112+
print(colored("\nSkipping stubtest since mypy failed.", "yellow"))
113+
114+
if sys.platform == "win32":
115+
print(colored("\nSkipping pytype on Windows. You can run the test with WSL.", "yellow"))
116+
else:
117+
print("\nRunning pytype...")
118+
pytype_result = subprocess.run([sys.executable, "tests/pytype_test.py", path])
119+
120+
print(f"\nRunning regression tests for Python {_PYTHON_VERSION}...")
121+
regr_test_result = subprocess.run(
122+
[sys.executable, "tests/regr_test.py", "stdlib" if folder == "stdlib" else stub, "--python-version", _PYTHON_VERSION],
123+
stderr=subprocess.PIPE,
124+
text=True,
125+
)
126+
# No test means they all ran successfully (0 out of 0). Not all 3rd-party stubs have regression tests.
127+
if "No test cases found" in regr_test_result.stderr:
128+
regr_test_returncode = 0
129+
print(colored(f"\nNo test cases found for {stub!r}!", "green"))
130+
else:
131+
regr_test_returncode = regr_test_result.returncode
132+
print(regr_test_result.stderr)
133+
134+
any_failure = any(
135+
[
136+
flake8_result.returncode,
137+
check_consistent_result.returncode,
138+
check_new_syntax_result.returncode,
139+
pyright_returncode,
140+
mypy_result.returncode,
141+
getattr(stubtest_result, "returncode", 0),
142+
getattr(pytype_result, "returncode", 0),
143+
regr_test_returncode,
144+
]
145+
)
146+
147+
if any_failure:
148+
print(colored("\n\n--- TEST SUMMARY: One or more tests failed. See above for details. ---\n", "red"))
149+
else:
150+
print(colored("\n\n--- TEST SUMMARY: All tests passed! ---\n", "green"))
151+
print("Flake8:", _SUCCESS if flake8_result.returncode == 0 else _FAILED)
152+
print("Check consistent:", _SUCCESS if check_consistent_result.returncode == 0 else _FAILED)
153+
print("Check new syntax:", _SUCCESS if check_new_syntax_result.returncode == 0 else _FAILED)
154+
if pyright_skipped:
155+
print("Pyright:", _SKIPPED)
156+
else:
157+
print("Pyright:", _SUCCESS if pyright_returncode == 0 else _FAILED)
158+
print("mypy:", _SUCCESS if mypy_result.returncode == 0 else _FAILED)
159+
if stubtest_result is None:
160+
print("stubtest:", _SKIPPED)
161+
else:
162+
print("stubtest:", _SUCCESS if stubtest_result.returncode == 0 else _FAILED)
163+
if pytype_result is None:
164+
print("pytype:", _SKIPPED)
165+
else:
166+
print("pytype:", _SUCCESS if pytype_result.returncode == 0 else _FAILED)
167+
print("Regression test:", _SUCCESS if regr_test_returncode == 0 else _FAILED)
168+
169+
sys.exit(int(any_failure))
170+
171+
172+
if __name__ == "__main__":
173+
try:
174+
main()
175+
except KeyboardInterrupt:
176+
print(colored("\nTests aborted due to KeyboardInterrupt!\n", "red"))
177+
sys.exit(1)

‎tests/README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,20 @@ in the `tests` and `scripts` directories.
1919
To run the tests, follow the [setup instructions](../CONTRIBUTING.md#preparing-the-environment)
2020
in the `CONTRIBUTING.md` document. In particular, we recommend running with Python 3.9+.
2121

22+
## Run all tests for a specific stub
23+
24+
Run using:
25+
```
26+
(.venv3)$ python3 scripts/runtests.py <stdlib-or-stubs>/<stub-to-test>
27+
```
28+
29+
This script will run all tests below for a specific typeshed directory. If a
30+
test supports multiple python versions, the oldest supported by typeshed will
31+
be selected. A summary of the results will be printed to the terminal.
32+
33+
You must provide a single argument which is a path to the stubs to test, like
34+
so: `stdlib/os` or `stubs/requests`.
35+
2236
## mypy\_test.py
2337

2438
Run using:

0 commit comments

Comments
 (0)