From f8c3ac77c8c23d3bcc39e4a6d2310ef43cbeac6c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Dec 2023 13:54:29 +0100 Subject: [PATCH 1/4] gh-109413: regrtest: add WorkerRunTests class --- Lib/test/libregrtest/main.py | 1 - Lib/test/libregrtest/run_workers.py | 12 +++++------- Lib/test/libregrtest/runtests.py | 28 ++++++++++++++++------------ Lib/test/libregrtest/worker.py | 6 +++--- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 55fc3a820d3451..16f6974ae32465 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -423,7 +423,6 @@ def create_run_tests(self, tests: TestTuple): python_cmd=self.python_cmd, randomize=self.randomize, random_seed=self.random_seed, - json_file=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 35aaf90ffc4299..18a0342f0611cf 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -18,7 +18,7 @@ from .logger import Logger from .result import TestResult, State from .results import TestResults -from .runtests import RunTests, JsonFile, JsonFileType +from .runtests import RunTests, WorkerRunTests, JsonFile, JsonFileType from .single import PROGRESS_MIN_TIME from .utils import ( StrPath, TestName, @@ -162,7 +162,7 @@ def stop(self) -> None: self._stopped = True self._kill() - def _run_process(self, runtests: RunTests, output_fd: int, + def _run_process(self, runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> int | None: popen = create_worker_process(runtests, output_fd, tmp_dir) self._popen = popen @@ -252,9 +252,7 @@ def create_json_file(self, stack: contextlib.ExitStack) -> tuple[JsonFile, TextI json_file = JsonFile(json_fd, JsonFileType.UNIX_FD) return (json_file, json_tmpfile) - def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> RunTests: - """Create the worker RunTests.""" - + def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> WorkerRunTests: tests = (test_name,) if self.runtests.rerun: match_tests = self.runtests.get_match_tests(test_name) @@ -267,12 +265,12 @@ def create_worker_runtests(self, test_name: TestName, json_file: JsonFile) -> Ru if self.runtests.output_on_failure: kwargs['verbose'] = True kwargs['output_on_failure'] = False - return self.runtests.copy( + return self.runtests.create_worker_runtests( tests=tests, json_file=json_file, **kwargs) - def run_tmp_files(self, worker_runtests: RunTests, + def run_tmp_files(self, worker_runtests: WorkerRunTests, stdout_fd: int) -> tuple[int | None, list[StrPath]]: # gh-93353: Check for leaked temporary files in the parent process, # since the deletion of temporary files can happen late during diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index b765ba5b41d236..318e00886eb7d6 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -93,12 +93,11 @@ class RunTests: python_cmd: tuple[str, ...] | None randomize: bool random_seed: int | str - json_file: JsonFile | None - def copy(self, **override): + def create_worker_runtests(self, **override): state = dataclasses.asdict(self) state.update(override) - return RunTests(**state) + return WorkerRunTests(**state) def get_match_tests(self, test_name) -> FilterTuple | None: if self.match_tests_dict is not None: @@ -120,13 +119,6 @@ def iter_tests(self): else: yield from self.tests - def as_json(self) -> StrJSON: - return json.dumps(self, cls=_EncodeRunTests) - - @staticmethod - def from_json(worker_json: StrJSON) -> 'RunTests': - return json.loads(worker_json, object_hook=_decode_runtests) - def json_file_use_stdout(self) -> bool: # Use STDOUT in two cases: # @@ -141,9 +133,21 @@ def json_file_use_stdout(self) -> bool: ) +@dataclasses.dataclass(slots=True, frozen=True) +class WorkerRunTests(RunTests): + json_file: JsonFile | None + + def as_json(self) -> StrJSON: + return json.dumps(self, cls=_EncodeRunTests) + + @staticmethod + def from_json(worker_json: StrJSON) -> 'RunTests': + return json.loads(worker_json, object_hook=_decode_runtests) + + class _EncodeRunTests(json.JSONEncoder): def default(self, o: Any) -> dict[str, Any]: - if isinstance(o, RunTests): + if isinstance(o, WorkerRunTests): result = dataclasses.asdict(o) result["__runtests__"] = True return result @@ -158,6 +162,6 @@ def _decode_runtests(data: dict[str, Any]) -> RunTests | dict[str, Any]: data['hunt_refleak'] = HuntRefleak(**data['hunt_refleak']) if data['json_file']: data['json_file'] = JsonFile(**data['json_file']) - return RunTests(**data) + return WorkerRunTests(**data) else: return data diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index b3bb0b7f34a060..7a6d33d4499943 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -7,7 +7,7 @@ from test.support import os_helper, Py_DEBUG from .setup import setup_process, setup_test_dir -from .runtests import RunTests, JsonFile, JsonFileType +from .runtests import WorkerRunTests, JsonFile, JsonFileType from .single import run_single_test from .utils import ( StrPath, StrJSON, TestFilter, @@ -17,7 +17,7 @@ USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg")) -def create_worker_process(runtests: RunTests, output_fd: int, +def create_worker_process(runtests: WorkerRunTests, output_fd: int, tmp_dir: StrPath | None = None) -> subprocess.Popen: python_cmd = runtests.python_cmd worker_json = runtests.as_json() @@ -73,7 +73,7 @@ def create_worker_process(runtests: RunTests, output_fd: int, def worker_process(worker_json: StrJSON) -> NoReturn: - runtests = RunTests.from_json(worker_json) + runtests = WorkerRunTests.from_json(worker_json) test_name = runtests.tests[0] match_tests: TestFilter = runtests.match_tests json_file: JsonFile = runtests.json_file From 5bc8b68e78f2a46f79b367c90e4fadb0d74736fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Dec 2023 13:56:12 +0100 Subject: [PATCH 2/4] Fix mypy --- Lib/test/libregrtest/runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 318e00886eb7d6..aaba52887f0ecf 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -141,7 +141,7 @@ def as_json(self) -> StrJSON: return json.dumps(self, cls=_EncodeRunTests) @staticmethod - def from_json(worker_json: StrJSON) -> 'RunTests': + def from_json(worker_json: StrJSON) -> 'WorkerRunTests': return json.loads(worker_json, object_hook=_decode_runtests) From 1806795c02007b80768bbd861f14e78f8206a78d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Dec 2023 13:57:47 +0100 Subject: [PATCH 3/4] Fix another mypy bug --- Lib/test/libregrtest/runtests.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index aaba52887f0ecf..e5ea8c944ec153 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -94,6 +94,11 @@ class RunTests: randomize: bool random_seed: int | str + def copy(self, **override) -> 'RunTests': + state = dataclasses.asdict(self) + state.update(override) + return RunTests(**state) + def create_worker_runtests(self, **override): state = dataclasses.asdict(self) state.update(override) From a69e3cc417ece554721d62fd3f3a24a8e1886e4c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 1 Dec 2023 14:15:58 +0100 Subject: [PATCH 4/4] Update Lib/test/libregrtest/runtests.py Co-authored-by: Alex Waygood --- Lib/test/libregrtest/runtests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index e5ea8c944ec153..edd72276320e41 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -140,7 +140,7 @@ def json_file_use_stdout(self) -> bool: @dataclasses.dataclass(slots=True, frozen=True) class WorkerRunTests(RunTests): - json_file: JsonFile | None + json_file: JsonFile def as_json(self) -> StrJSON: return json.dumps(self, cls=_EncodeRunTests)