Skip to content

Fix timeout #408

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 28, 2023
Merged

Fix timeout #408

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 43 additions & 52 deletions master/custom/factories.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,12 +34,21 @@
TEST_TIMEOUT = 20 * 60


def step_timeout(timeout):
# timeout is the regrtest timeout in seconds. If a test runs longer than
# the timeout, it should be killed by faulthandler. Give 10 minutes to
# faulthandler to kill the process. Tests should always be shorter than the
# buildbot step timeout, unless faulthandler fails to kill the process.
return timeout + 10 * 60


def regrtest_has_cleanup(branch):
return branch not in {CUSTOM_BRANCH_NAME}


class TaggedBuildFactory(factory.BuildFactory):
class BaseBuild(factory.BuildFactory):
factory_tags = []
test_timeout = TEST_TIMEOUT

def __init__(self, source, *, extra_tags=[], **kwargs):
super().__init__([source])
Expand All @@ -57,13 +66,12 @@ def has_option(option, test_options):
return option in ' '.join(test_options)


class UnixBuild(TaggedBuildFactory):
class UnixBuild(BaseBuild):
configureFlags = ["--with-pydebug"]
compile_environ = {}
interpreterFlags = ""
testFlags = ["-j2"]
makeTarget = "all"
test_timeout = None
test_environ = {}
build_out_of_tree = False

Expand Down Expand Up @@ -95,10 +103,6 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
testopts = list(self.testFlags)
if not has_option("-R", self.testFlags):
testopts.extend(("--junit-xml", JUNIT_FILENAME))
# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
faulthandler_timeout = self.test_timeout - 5 * 60
if parallel:
compile = ["make", parallel, self.makeTarget]
testopts.append(parallel)
Expand All @@ -110,8 +114,8 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
"make",
"buildbottest",
"TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
"TESTPYTHONOPTS=" + self.interpreterFlags,
"TESTTIMEOUT=" + str(faulthandler_timeout),
f"TESTPYTHONOPTS={self.interpreterFlags}",
f"TESTTIMEOUT={self.test_timeout}",
]

self.addStep(Compile(command=compile,
Expand All @@ -127,15 +131,13 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
**oot_kwargs
)
)
self.addStep(
Test(
command=test,
timeout=self.test_timeout,
usePTY=test_with_PTY,
env=self.test_environ,
**oot_kwargs
)
)
self.addStep(Test(
command=test,
timeout=step_timeout(self.test_timeout),
usePTY=test_with_PTY,
env=self.test_environ,
**oot_kwargs
))
if branch not in ("3",) and not has_option("-R", self.testFlags):
filename = JUNIT_FILENAME
if self.build_out_of_tree:
Expand Down Expand Up @@ -187,22 +189,21 @@ class UnixNoGilRefleakBuild(UnixBuild):
factory_tags = ["nogil", "refleak"]


class UnixInstalledBuild(TaggedBuildFactory):
class UnixInstalledBuild(BaseBuild):
buildersuffix = ".installed"
configureFlags = []
interpreterFlags = ["-Wdefault", "-bb", "-E"]
defaultTestOpts = ["-rwW", "-uall", "-j2"]
makeTarget = "all"
installTarget = "install"
test_timeout = None
factory_tags = ["installed"]

def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
if branch == MAIN_BRANCH_NAME:
branch = MAIN_BRANCH_VERSION
elif branch == "custom":
branch = "3"
installed_python = "./target/bin/python%s" % branch
installed_python = f"./target/bin/python{branch}"
self.addStep(
Configure(
command=["./configure", "--prefix", "$(PWD)/target"]
Expand All @@ -213,11 +214,7 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
compile = ["make", self.makeTarget]
install = ["make", self.installTarget]
testopts = list(self.defaultTestOpts)
# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
faulthandler_timeout = self.test_timeout - 5 * 60
testopts.append(f"--timeout={faulthandler_timeout}")
testopts.append(f"--timeout={self.test_timeout}")
if parallel:
compile = ["make", parallel, self.makeTarget]
install = ["make", parallel, self.installTarget]
Expand All @@ -241,9 +238,11 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
warnOnFailure=True,
)
)
self.addStep(
Test(command=test, timeout=self.test_timeout, usePTY=test_with_PTY)
)
self.addStep(Test(
command=test,
timeout=step_timeout(self.test_timeout),
usePTY=test_with_PTY,
))
self.addStep(Uninstall())
self.addStep(Clean())

Expand All @@ -256,8 +255,6 @@ class UnixAsanBuild(UnixBuild):
# SIGSEGV is ignored on purpose.
compile_environ = {'ASAN_OPTIONS': 'detect_leaks=0:allocator_may_return_null=1:handle_segv=0'}
test_environ = {'ASAN_OPTIONS': 'detect_leaks=0:allocator_may_return_null=1:handle_segv=0'}
# Sometimes test_multiprocessing_fork times out after 15 minutes
test_timeout = TEST_TIMEOUT * 2


class UnixAsanDebugBuild(UnixAsanBuild):
Expand Down Expand Up @@ -512,15 +509,14 @@ class MacOSArmWithBrewBuild(UnixBuild):
##############################################################################


class BaseWindowsBuild(TaggedBuildFactory):
class BaseWindowsBuild(BaseBuild):
build_command = [r"Tools\buildbot\build.bat"]
test_command = [r"Tools\buildbot\test.bat"]
clean_command = [r"Tools\buildbot\clean.bat"]
python_command = [r"python.bat"]
buildFlags = ["-p", "Win32"]
testFlags = ["-p", "Win32", "-j2"]
cleanFlags = []
test_timeout = None
factory_tags = ["win32"]

def setup(self, parallel, branch, **kwargs):
Expand All @@ -542,9 +538,11 @@ def setup(self, parallel, branch, **kwargs):
warnOnFailure=True,
)
)
timeout = self.test_timeout if self.test_timeout else TEST_TIMEOUT
test_command.extend(("--timeout", timeout - (5 * 60)))
self.addStep(Test(command=test_command, timeout=timeout))
test_command.extend(("--timeout", str(self.test_timeout)))
self.addStep(Test(
command=test_command,
timeout=step_timeout(self.test_timeout),
))
if branch not in ("3",) and not has_option("-R", self.testFlags):
self.addStep(UploadTestResults(branch))
self.addStep(Clean(command=clean_command))
Expand Down Expand Up @@ -721,17 +719,12 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
if branch in BRANCH_WITH_FAIL_RERUN:
testopts.append("--fail-rerun")

# Timeout for the buildworker process
self.test_timeout = self.test_timeout or TEST_TIMEOUT
# Timeout for faulthandler
faulthandler_timeout = self.test_timeout - 5 * 60

test = [
"make",
"buildbottest",
"TESTOPTS=" + " ".join(testopts) + " ${BUILDBOT_TESTOPTS}",
"TESTPYTHONOPTS=" + self.interpreterFlags,
"TESTTIMEOUT=" + str(faulthandler_timeout),
f"TESTPYTHONOPTS={self.interpreterFlags}",
f"TESTTIMEOUT={self.test_timeout}",
]

if parallel:
Expand All @@ -757,15 +750,13 @@ def setup(self, parallel, branch, test_with_PTY=False, **kwargs):
workdir=oot_host_path,
)
)
self.addStep(
Test(
command=test,
timeout=self.test_timeout,
usePTY=test_with_PTY,
env=self.test_environ,
workdir=oot_host_path,
)
)
self.addStep(Test(
command=test,
timeout=step_timeout(self.test_timeout),
usePTY=test_with_PTY,
env=self.test_environ,
workdir=oot_host_path,
))
if branch not in ("3",) and not has_option("-R", self.testFlags):
filename = os.path.join(oot_host_path, JUNIT_FILENAME)
self.addStep(UploadTestResults(branch, filename=filename))
Expand Down