Skip to content

Add module-write hook #3728

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
smoors merged 2 commits into from
Jun 9, 2021
Merged

Add module-write hook #3728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b61190d
Add module-write hook
Flamefire Jun 7, 2021
a5ff50a
Fix comments claiming hooks was a list
Flamefire Jun 9, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions easybuild/framework/easyblock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@
from easybuild.tools.hooks import BUILD_STEP, CLEANUP_STEP, CONFIGURE_STEP, EXTENSIONS_STEP, FETCH_STEP, INSTALL_STEP
from easybuild.tools.hooks import MODULE_STEP, PACKAGE_STEP, PATCH_STEP, PERMISSIONS_STEP, POSTITER_STEP, POSTPROC_STEP
from easybuild.tools.hooks import PREPARE_STEP, READY_STEP, SANITYCHECK_STEP, SOURCE_STEP, TEST_STEP, TESTCASES_STEP
from easybuild.tools.hooks import load_hooks, run_hook
from easybuild.tools.hooks import MODULE_WRITE, load_hooks, run_hook
from easybuild.tools.run import run_cmd
from easybuild.tools.jenkins import write_to_xml
from easybuild.tools.module_generator import ModuleGeneratorLua, ModuleGeneratorTcl, module_generator, dependencies_for
Expand Down Expand Up @@ -144,7 +144,7 @@ def __init__(self, ec):
# keep track of original working directory, so we can go back there
self.orig_workdir = os.getcwd()

# list of pre- and post-step hooks
# dict of all hooks (mapping of name to function)
self.hooks = load_hooks(build_option('hooks'))

# list of patch/source files, along with checksums
Expand Down Expand Up @@ -3165,6 +3165,10 @@ def make_module_step(self, fake=False):
txt += self.make_module_extra()
txt += self.make_module_footer()

hook_txt = run_hook(MODULE_WRITE, self.hooks, args=[self, mod_filepath, txt])
if hook_txt is not None:
txt = hook_txt

if self.dry_run:
# only report generating actual module file during dry run, don't mention temporary module files
if not fake:
Expand Down
30 changes: 14 additions & 16 deletions easybuild/tools/hooks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@

START = 'start'
PARSE = 'parse'
MODULE_WRITE = 'module_write'
END = 'end'

PRE_PREF = 'pre_'
Expand All @@ -69,7 +70,7 @@
INSTALL_STEP, EXTENSIONS_STEP, POSTPROC_STEP, SANITYCHECK_STEP, CLEANUP_STEP, MODULE_STEP,
PERMISSIONS_STEP, PACKAGE_STEP, TESTCASES_STEP]

HOOK_NAMES = [START, PARSE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
HOOK_NAMES = [START, PARSE, MODULE_WRITE] + [p + s for s in STEP_NAMES for p in [PRE_PREF, POST_PREF]] + [END]
KNOWN_HOOKS = [h + HOOK_SUFF for h in HOOK_NAMES]


Expand Down Expand Up @@ -99,7 +100,7 @@ def load_hooks(hooks_path):
if attr.endswith(HOOK_SUFF):
hook = getattr(imported_hooks, attr)
if callable(hook):
hooks.update({attr: hook})
hooks[attr] = hook
else:
_log.debug("Skipping non-callable attribute '%s' when loading hooks", attr)
_log.info("Found hooks: %s", sorted(hooks.keys()))
Expand All @@ -119,11 +120,8 @@ def load_hooks(hooks_path):


def verify_hooks(hooks):
"""Check whether list of obtained hooks only includes known hooks."""
unknown_hooks = []
for key in sorted(hooks):
if key not in KNOWN_HOOKS:
unknown_hooks.append(key)
"""Check whether obtained hooks only includes known hooks."""
unknown_hooks = [key for key in sorted(hooks) if key not in KNOWN_HOOKS]

if unknown_hooks:
error_lines = ["Found one or more unknown hooks:"]
Expand All @@ -147,7 +145,7 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):
Find hook with specified label.

:param label: name of hook
:param hooks: list of defined hooks
:param hooks: dict of defined hooks
:param pre_step_hook: indicates whether hook to run is a pre-step hook
:param post_step_hook: indicates whether hook to run is a post-step hook
"""
Expand All @@ -162,27 +160,26 @@ def find_hook(label, hooks, pre_step_hook=False, post_step_hook=False):

hook_name = hook_prefix + label + HOOK_SUFF

for key in hooks:
if key == hook_name:
_log.info("Found %s hook", hook_name)
res = hooks[key]
break
res = hooks.get(hook_name)
if res:
_log.info("Found %s hook", hook_name)

return res


def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None, msg=None):
"""
Run hook with specified label.
Run hook with specified label and return result of calling the hook or None.

:param label: name of hook
:param hooks: list of defined hooks
:param hooks: dict of defined hooks
:param pre_step_hook: indicates whether hook to run is a pre-step hook
:param post_step_hook: indicates whether hook to run is a post-step hook
:param args: arguments to pass to hook function
:param msg: custom message that is printed when hook is called
"""
hook = find_hook(label, hooks, pre_step_hook=pre_step_hook, post_step_hook=post_step_hook)
res = None
if hook:
if args is None:
args = []
Expand All @@ -197,4 +194,5 @@ def run_hook(label, hooks, pre_step_hook=False, post_step_hook=False, args=None,
print_msg(msg)

_log.info("Running '%s' hook function (arguments: %s)...", hook.__name__, args)
hook(*args)
res = hook(*args)
return res
107 changes: 65 additions & 42 deletions test/framework/toy_build.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import stat
import sys
import tempfile
import textwrap
from distutils.version import LooseVersion
from test.framework.utilities import EnhancedTestCase, TestLoaderFiltered
from test.framework.package import mock_fpm
Expand Down Expand Up @@ -2638,34 +2639,38 @@ def test_toy_build_trace(self):
def test_toy_build_hooks(self):
"""Test use of --hooks."""
hooks_file = os.path.join(self.test_prefix, 'my_hooks.py')
hooks_file_txt = '\n'.join([
"import os",
'',
"def start_hook():",
" print('start hook triggered')",
'',
"def parse_hook(ec):",
" print('%s %s' % (ec.name, ec.version))",
hooks_file_txt = textwrap.dedent("""
import os

def start_hook():
print('start hook triggered')

def parse_hook(ec):
print('%s %s' % (ec.name, ec.version))
# print sources value to check that raw untemplated strings are exposed in parse_hook
" print(ec['sources'])",
print(ec['sources'])
# try appending to postinstallcmd to see whether the modification is actually picked up
# (required templating to be disabled before parse_hook is called)
" ec['postinstallcmds'].append('echo toy')",
" print(ec['postinstallcmds'][-1])",
'',
"def pre_configure_hook(self):",
" print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))",
'',
"def post_configure_hook(self):",
" print('post-configure: toy.source: %s' % os.path.exists('toy.source'))",
'',
"def post_install_hook(self):",
" print('in post-install hook for %s v%s' % (self.name, self.version))",
" print(', '.join(sorted(os.listdir(self.installdir))))",
'',
"def end_hook():",
" print('end hook triggered, all done!')",
])
ec['postinstallcmds'].append('echo toy')
print(ec['postinstallcmds'][-1])

def pre_configure_hook(self):
print('pre-configure: toy.source: %s' % os.path.exists('toy.source'))

def post_configure_hook(self):
print('post-configure: toy.source: %s' % os.path.exists('toy.source'))

def post_install_hook(self):
print('in post-install hook for %s v%s' % (self.name, self.version))
print(', '.join(sorted(os.listdir(self.installdir))))

def module_write_hook(self, module_path, module_txt):
print('in module-write hook hook for %s' % os.path.basename(module_path))
return module_txt.replace('Toy C program, 100% toy.', 'Not a toy anymore')

def end_hook():
print('end hook triggered, all done!')
""")
write_file(hooks_file, hooks_file_txt)

self.mock_stderr(True)
Expand All @@ -2676,26 +2681,44 @@ def test_toy_build_hooks(self):
self.mock_stderr(False)
self.mock_stdout(False)

test_mod_path = os.path.join(self.test_installpath, 'modules', 'all')
toy_mod_file = os.path.join(test_mod_path, 'toy', '0.0')
if get_module_syntax() == 'Lua':
toy_mod_file += '.lua'

self.assertEqual(stderr, '')
expected_output = '\n'.join([
"== Running start hook...",
"start hook triggered",
"== Running parse hook for toy-0.0.eb...",
"toy 0.0",
"['%(name)s-%(version)s.tar.gz']",
"echo toy",
"== Running pre-configure hook...",
"pre-configure: toy.source: True",
"== Running post-configure hook...",
"post-configure: toy.source: False",
"== Running post-install hook...",
"in post-install hook for toy v0.0",
"bin, lib",
"== Running end hook...",
"end hook triggered, all done!",
])
# There are 4 modules written:
# Sanitycheck for extensions and main easyblock (1 each), main and devel module
expected_output = textwrap.dedent("""
== Running start hook...
start hook triggered
== Running parse hook for toy-0.0.eb...
toy 0.0
['%(name)s-%(version)s.tar.gz']
echo toy
== Running pre-configure hook...
pre-configure: toy.source: True
== Running post-configure hook...
post-configure: toy.source: False
== Running post-install hook...
in post-install hook for toy v0.0
bin, lib
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running module_write hook...
in module-write hook hook for {mod_name}
== Running end hook...
end hook triggered, all done!
""").strip().format(mod_name=os.path.basename(toy_mod_file))
self.assertEqual(stdout.strip(), expected_output)

toy_mod = read_file(toy_mod_file)
self.assertIn('Not a toy anymore', toy_mod)

def test_toy_multi_deps(self):
"""Test installation of toy easyconfig that uses multi_deps."""
test_ecs_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
Expand Down