Skip to content

Commit a43aeb8

Browse files
committed
Fix timeout
Pass the timeout unchanged to regrtest, but add 10 minutes to the buildbot step timeout. No longer remove 5 minutes to the timeout passed to regrtest. Set UnixAsanBuild timeout to TEST_TIMEOUT (default): test_multiprocessing_fork is skipped on ASAN build.
1 parent becb34a commit a43aeb8

File tree

1 file changed

+43
-52
lines changed

1 file changed

+43
-52
lines changed

master/custom/factories.py

Lines changed: 43 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -34,12 +34,21 @@
3434
TEST_TIMEOUT = 20 * 60
3535

3636

37+
def step_timeout(timeout):
38+
# timeout is the regrtest timeout in seconds. If a test runs longer than
39+
# the timeout, it should be killed by faulthandler. Give 10 minutes to
40+
# faulthandler to kill the process. Tests should always be shorter than the
41+
# buildbot step timeout, unless faulthandler fails to kill the process.
42+
return timeout + 10 * 60
43+
44+
3745
def regrtest_has_cleanup(branch):
3846
return branch not in {CUSTOM_BRANCH_NAME}
3947

4048

41-
class TaggedBuildFactory(factory.BuildFactory):
49+
class BaseBuild(factory.BuildFactory):
4250
factory_tags = []
51+
test_timeout = TEST_TIMEOUT
4352

4453
def __init__(self, source, *, extra_tags=[], **kwargs):
4554
super().__init__([source])
@@ -57,13 +66,12 @@ def has_option(option, test_options):
5766
return option in ' '.join(test_options)
5867

5968

60-
class UnixBuild(TaggedBuildFactory):
69+
class UnixBuild(BaseBuild):
6170
configureFlags = ["--with-pydebug"]
6271
compile_environ = {}
6372
interpreterFlags = ""
6473
testFlags = ["-j2"]
6574
makeTarget = "all"
66-
test_timeout = None
6775
test_environ = {}
6876
build_out_of_tree = False
6977

@@ -95,10 +103,6 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
95103
testopts = list(self.testFlags)
96104
if not has_option("-R", self.testFlags):
97105
testopts.extend(("--junit-xml", JUNIT_FILENAME))
98-
# Timeout for the buildworker process
99-
self.test_timeout = self.test_timeout or TEST_TIMEOUT
100-
# Timeout for faulthandler
101-
faulthandler_timeout = self.test_timeout - 5 * 60
102106
if parallel:
103107
compile = ["make", parallel, self.makeTarget]
104108
testopts.append(parallel)
@@ -110,8 +114,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
110114
"make",
111115
"buildbottest",
112116
"TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
113-
"TESTPYTHONOPTS=" + self.interpreterFlags,
114-
"TESTTIMEOUT=" + str(faulthandler_timeout),
117+
f"TESTPYTHONOPTS={self.interpreterFlags}",
118+
f"TESTTIMEOUT={self.test_timeout}",
115119
]
116120

117121
self.addStep(Compile(command=compile,
@@ -127,15 +131,13 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
127131
**oot_kwargs
128132
)
129133
)
130-
self.addStep(
131-
Test(
132-
command=test,
133-
timeout=self.test_timeout,
134-
usePTY=test_with_PTY,
135-
env=self.test_environ,
136-
**oot_kwargs
137-
)
138-
)
134+
self.addStep(Test(
135+
command=test,
136+
timeout=step_timeout(self.test_timeout),
137+
usePTY=test_with_PTY,
138+
env=self.test_environ,
139+
**oot_kwargs
140+
))
139141
if branch not in ("3",) and not has_option("-R", self.testFlags):
140142
filename = JUNIT_FILENAME
141143
if self.build_out_of_tree:
@@ -187,22 +189,21 @@ class UnixNoGilRefleakBuild(UnixBuild):
187189
factory_tags = ["nogil", "refleak"]
188190

189191

190-
class UnixInstalledBuild(TaggedBuildFactory):
192+
class UnixInstalledBuild(BaseBuild):
191193
buildersuffix = ".installed"
192194
configureFlags = []
193195
interpreterFlags = ["-Wdefault", "-bb", "-E"]
194196
defaultTestOpts = ["-rwW", "-uall", "-j2"]
195197
makeTarget = "all"
196198
installTarget = "install"
197-
test_timeout = None
198199
factory_tags = ["installed"]
199200

200201
def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
201202
if branch == MAIN_BRANCH_NAME:
202203
branch = MAIN_BRANCH_VERSION
203204
elif branch == "custom":
204205
branch = "3"
205-
installed_python = "./target/bin/python%s" % branch
206+
installed_python = f"./target/bin/python{branch}"
206207
self.addStep(
207208
Configure(
208209
command=["./configure", "--prefix", "$(PWD)/target"]
@@ -213,11 +214,7 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
213214
compile = ["make", self.makeTarget]
214215
install = ["make", self.installTarget]
215216
testopts = list(self.defaultTestOpts)
216-
# Timeout for the buildworker process
217-
self.test_timeout = self.test_timeout or TEST_TIMEOUT
218-
# Timeout for faulthandler
219-
faulthandler_timeout = self.test_timeout - 5 * 60
220-
testopts.append(f"--timeout={faulthandler_timeout}")
217+
testopts.append(f"--timeout={self.test_timeout}")
221218
if parallel:
222219
compile = ["make", parallel, self.makeTarget]
223220
install = ["make", parallel, self.installTarget]
@@ -241,9 +238,11 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
241238
warnOnFailure=True,
242239
)
243240
)
244-
self.addStep(
245-
Test(command=test, timeout=self.test_timeout, usePTY=test_with_PTY)
246-
)
241+
self.addStep(Test(
242+
command=test,
243+
timeout=step_timeout(self.test_timeout),
244+
usePTY=test_with_PTY,
245+
))
247246
self.addStep(Uninstall())
248247
self.addStep(Clean())
249248

@@ -256,8 +255,6 @@ class UnixAsanBuild(UnixBuild):
256255
# SIGSEGV is ignored on purpose.
257256
compile_environ = {'ASAN_OPTIONS': 'detect_leaks=0:allocator_may_return_null=1:handle_segv=0'}
258257
test_environ = {'ASAN_OPTIONS': 'detect_leaks=0:allocator_may_return_null=1:handle_segv=0'}
259-
# Sometimes test_multiprocessing_fork times out after 15 minutes
260-
test_timeout = TEST_TIMEOUT * 2
261258

262259

263260
class UnixAsanDebugBuild(UnixAsanBuild):
@@ -512,15 +509,14 @@ class MacOSArmWithBrewBuild(UnixBuild):
512509
##############################################################################
513510

514511

515-
class BaseWindowsBuild(TaggedBuildFactory):
512+
class BaseWindowsBuild(BaseBuild):
516513
build_command = [r"Tools\buildbot\build.bat"]
517514
test_command = [r"Tools\buildbot\test.bat"]
518515
clean_command = [r"Tools\buildbot\clean.bat"]
519516
python_command = [r"python.bat"]
520517
buildFlags = ["-p", "Win32"]
521518
testFlags = ["-p", "Win32", "-j2"]
522519
cleanFlags = []
523-
test_timeout = None
524520
factory_tags = ["win32"]
525521

526522
def setup(self, parallel, branch, **kwargs):
@@ -542,9 +538,11 @@ def setup(self, parallel, branch, **kwargs):
542538
warnOnFailure=True,
543539
)
544540
)
545-
timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT
546-
test_command.extend(("--timeout", timeout - (5 * 60)))
547-
self.addStep(Test(command=test_command, timeout=timeout))
541+
test_command.extend(("--timeout", str(self.test_timeout)))
542+
self.addStep(Test(
543+
command=test_command,
544+
timeout=step_timeout(self.test_timeout),
545+
))
548546
if branch not in ("3",) and not has_option("-R", self.testFlags):
549547
self.addStep(UploadTestResults(branch))
550548
self.addStep(Clean(command=clean_command))
@@ -721,17 +719,12 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
721719
if branch in BRANCH_WITH_FAIL_RERUN:
722720
testopts.append("--fail-rerun")
723721

724-
# Timeout for the buildworker process
725-
self.test_timeout = self.test_timeout or TEST_TIMEOUT
726-
# Timeout for faulthandler
727-
faulthandler_timeout = self.test_timeout - 5 * 60
728-
729722
test = [
730723
"make",
731724
"buildbottest",
732725
"TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
733-
"TESTPYTHONOPTS=" + self.interpreterFlags,
734-
"TESTTIMEOUT=" + str(faulthandler_timeout),
726+
f"TESTPYTHONOPTS={self.interpreterFlags}",
727+
f"TESTTIMEOUT={self.test_timeout}",
735728
]
736729

737730
if parallel:
@@ -757,15 +750,13 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
757750
workdir=oot_host_path,
758751
)
759752
)
760-
self.addStep(
761-
Test(
762-
command=test,
763-
timeout=self.test_timeout,
764-
usePTY=test_with_PTY,
765-
env=self.test_environ,
766-
workdir=oot_host_path,
767-
)
768-
)
753+
self.addStep(Test(
754+
command=test,
755+
timeout=step_timeout(self.test_timeout),
756+
usePTY=test_with_PTY,
757+
env=self.test_environ,
758+
workdir=oot_host_path,
759+
))
769760
if branch not in ("3",) and not has_option("-R", self.testFlags):
770761
filename = os.path.join(oot_host_path, JUNIT_FILENAME)
771762
self.addStep(UploadTestResults(branch, filename=filename))

0 commit comments

Comments
 (0)