Skip to content

adding locking to prevent two parallel builds of the same installation directory #3009

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 32 commits into from
Mar 30, 2020
Merged

adding locking to prevent two parallel builds of the same installation directory #3009

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
0260023
adding a locking feature that will prevent two parallel builds of the…
mboisson Sep 16, 2019
4311615
appeasing hound
mboisson Sep 16, 2019
0d74bd0
adding silent mode for printed messages to hopefully make tests pass.
mboisson Sep 16, 2019
e6bd046
adding the option to specify the lock path using a global argument
mboisson Sep 16, 2019
e467ba3
appeasing hound
mboisson Sep 16, 2019
9c725e7
made one more message silent. changed the default lockpath to use ins…
mboisson Sep 17, 2019
dfb9a7b
appeasing hound
mboisson Sep 17, 2019
655d494
fix mkdir of lockpath to include parents
mboisson Mar 9, 2020
4e63736
Merge branch 'upstream_develop' into build_lock
mboisson Mar 13, 2020
69cf227
fixed test for toy_build, given locks
mboisson Mar 13, 2020
21900fc
fixing test for toy_build, to take locks into account
mboisson Mar 13, 2020
5efac48
trying to fix test agian
mboisson Mar 13, 2020
4edcfd9
trying to fix test agian
mboisson Mar 13, 2020
a9038ca
trying to fix test agian
mboisson Mar 13, 2020
f355c1a
trying to fix test agian
mboisson Mar 13, 2020
a6f92f5
bug fix: actually proceed with installation after waiting for lock to…
boegel Mar 29, 2020
41ad45b
don't print messages when creating/removing lock, just log an info me…
boegel Mar 29, 2020
0e7c818
make wait-for-lock interval configurable via --wait-for-lock
boegel Mar 29, 2020
fbb7c7f
rename --lockpath to --locks-dir
boegel Mar 29, 2020
878d611
add test for lock that prevents two identical installations happening…
boegel Mar 29, 2020
79a880a
add --ignore-locks option
boegel Mar 29, 2020
9724ed9
fix broken toy tests, take into account .locks dir in software directory
boegel Mar 30, 2020
5adfc37
fix failing test_module_only
boegel Mar 30, 2020
f77ced5
use directory as lock
boegel Mar 30, 2020
81f7833
ensure cleaner error when lock dir can't be created
boegel Mar 30, 2020
e39976b
Merge pull request #20 from boegel/build_lock
mboisson Mar 30, 2020
af2f6e5
enhance lock test to check for clean error
boegel Mar 30, 2020
5c5d242
appease the Hound
boegel Mar 30, 2020
774f589
also accept 'Permission denied' error in test for failing lock creation
boegel Mar 30, 2020
697ff6e
Merge pull request #21 from boegel/build_lock
mboisson Mar 30, 2020
1b85909
sort lists before comparing them in test_toy_modaltsoftname
boegel Mar 30, 2020
4b01f31
Merge pull request #22 from boegel/build_lock
mboisson Mar 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 35 additions & 13 deletions easybuild/framework/easyblock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2943,20 +2943,42 @@ def run_all_steps(self, run_test_cases):

print_msg("building and installing %s..." % self.full_mod_name, log=self.log, silent=self.silent)
trace_msg("installation prefix: %s" % self.installdir)
try:
for (step_name, descr, step_methods, skippable) in steps:
if self._skip_step(step_name, skippable):
print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
else:
if self.dry_run:
self.dry_run_msg("%s... [DRY RUN]\n", descr)
else:
print_msg("%s..." % descr, log=self.log, silent=self.silent)
self.current_step = step_name
self.run_step(step_name, step_methods)

except StopException:
pass
lockpath = build_option('lockpath') or build_path()
if not os.path.exists(lockpath):
mkdir(lockpath)
lockfile_name = os.path.join(lockpath, ".%s.lock" % self.installdir.replace('/','_'))
if os.path.exists(lockfile_name):
if build_option('wait_on_lock'):
while os.path.exists(lockfile_name):
print_msg("Lock file %s exists. Waiting 60 seconds." % lockfile_name, silent=self.silent)
time.sleep(60)
else:
print_msg("Build aborted. Lock file %s exists." % lockfile_name, silent=self.silent)
return False
else:
try:
# create a new lock file
print_msg("Creating lock file %s" % lockfile_name)
f = open(lockfile_name, "w+")
f.close()

for (step_name, descr, step_methods, skippable) in steps:
if self._skip_step(step_name, skippable):
print_msg("%s [skipped]" % descr, log=self.log, silent=self.silent)
else:
if self.dry_run:
self.dry_run_msg("%s... [DRY RUN]\n", descr)
else:
print_msg("%s..." % descr, log=self.log, silent=self.silent)
self.current_step = step_name
self.run_step(step_name, step_methods)

except StopException:
pass
finally:
print_msg("Removing lock file %s" % lockfile_name, silent=self.silent)
os.remove(lockfile_name)

# return True for successfull build (or stopped build)
return True
Expand Down
2 changes: 2 additions & 0 deletions easybuild/tools/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'job_output_dir',
'job_polling_interval',
'job_target_resource',
'lockpath',
'modules_footer',
'modules_header',
'mpi_cmd_template',
Expand Down Expand Up @@ -221,6 +222,7 @@ def mk_full_default_path(name, prefix=DEFAULT_PREFIX):
'use_f90cache',
'use_existing_modules',
'set_default_module',
'wait_on_lock',
],
True: [
'cleanup_builddir',
Expand Down
3 changes: 3 additions & 0 deletions easybuild/tools/options.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ def basic_options(self):
"and skipping check for OS dependencies", None, 'store_true', False, 'f'),
'job': ("Submit the build as a job", None, 'store_true', False),
'logtostdout': ("Redirect main log to stdout", None, 'store_true', False, 'l'),
'lockpath': ("Specifies which path should be used to store lock files", None, 'store_or_None', None),
'missing-modules': ("Print list of missing modules for dependencies of specified easyconfigs",
None, 'store_true', False, 'M'),
'only-blocks': ("Only build listed blocks", 'strlist', 'extend', None, 'b', {'metavar': 'BLOCKS'}),
Expand Down Expand Up @@ -441,6 +442,8 @@ def override_options(self):
None, 'store_true', False),
'verify-easyconfig-filenames': ("Verify whether filename of specified easyconfigs matches with contents",
None, 'store_true', False),
'wait-on-lock': ("Wait until lock file is removed when a lock if found",
None, 'store_true', False),
'zip-logs': ("Zip logs that are copied to install directory, using specified command",
None, 'store_or_None', 'gzip'),

Expand Down