Skip to content

Commit 0b1057e

Browse files
committed
Ensure mail event names are validated in config
Moved validation of workflow event names to parsec
1 parent 428420e commit 0b1057e

File tree

9 files changed

+175
-117
lines changed

9 files changed

+175
-117
lines changed

changes.d/6838.fix.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
Workflow and task `handler events` names are now validated. Outdated Cylc 7 workflow event names are automatically upgraded.
1+
Workflow and task `handler events` and `mail events` names are now validated. Outdated Cylc 7 workflow event names are automatically upgraded.

cylc/flow/cfgspec/globalcfg.py

Lines changed: 51 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,39 +15,55 @@
1515
# along with this program. If not, see <http://www.gnu.org/licenses/>.
1616
"""Cylc site and user configuration file spec."""
1717

18+
from contextlib import suppress
1819
import os
1920
from pathlib import Path
2021
from sys import stderr
21-
from textwrap import dedent, indent
22-
from typing import List, Optional, Tuple, Any, Union
22+
from textwrap import (
23+
dedent,
24+
indent,
25+
)
26+
from typing import (
27+
Any,
28+
Dict,
29+
List,
30+
Optional,
31+
Tuple,
32+
Union,
33+
)
2334

24-
from contextlib import suppress
2535
from packaging.version import Version
2636

27-
from cylc.flow import LOG
28-
from cylc.flow import __version__ as CYLC_VERSION
29-
from cylc.flow.platforms import validate_platforms
37+
from cylc.flow import (
38+
LOG,
39+
__version__ as CYLC_VERSION,
40+
)
3041
from cylc.flow.exceptions import GlobalConfigError
3142
from cylc.flow.hostuserutil import get_user_home
3243
from cylc.flow.network.client_factory import CommsMeth
33-
from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
3444
from cylc.flow.parsec.config import (
3545
ConfigNode as Conf,
3646
ParsecConfig,
3747
)
3848
from cylc.flow.parsec.exceptions import (
39-
ParsecError,
4049
ItemNotFoundError,
50+
ParsecError,
4151
ValidationError,
4252
)
4353
from cylc.flow.parsec.upgrade import upgrader
44-
from cylc.flow.parsec.util import printcfg, expand_many_section
54+
from cylc.flow.parsec.util import (
55+
expand_many_section,
56+
printcfg,
57+
)
4558
from cylc.flow.parsec.validate import (
4659
CylcConfigValidator as VDR,
4760
DurationFloat,
4861
Range,
4962
cylc_config_validate,
5063
)
64+
from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
65+
from cylc.flow.platforms import validate_platforms
66+
from cylc.flow.workflow_events import WorkflowEventHandler
5167

5268

5369
PLATFORM_REGEX_TEXT = '''
@@ -150,7 +166,7 @@
150166
Configure the workflow event handling system.
151167
'''
152168

153-
EVENTS_SETTINGS = { # workflow events
169+
EVENTS_SETTINGS: Dict[str, Union[str, Dict[str, Any]]] = { # workflow events
154170
'handlers': '''
155171
Configure :term:`event handlers` that run when certain workflow
156172
events occur.
@@ -171,17 +187,26 @@
171187
172188
:ref:`user_guide.scheduler.workflow_events`
173189
''',
174-
'handler events': '''
175-
Specify the events for which workflow event handlers should be invoked.
190+
'handler events': {
191+
'desc': '''
192+
Specify the events for which workflow event handlers should be
193+
invoked.
176194
177-
.. seealso::
195+
.. seealso::
178196
179-
:ref:`user_guide.scheduler.workflow_events`
180-
''',
181-
'mail events': '''
182-
Specify the workflow events for which notification emails should
183-
be sent.
184-
''',
197+
:ref:`user_guide.scheduler.workflow_events`
198+
''',
199+
'options': WorkflowEventHandler.EVENTS.copy(),
200+
'depr_options': WorkflowEventHandler.EVENTS_DEPRECATED.copy(),
201+
},
202+
'mail events': {
203+
'desc': '''
204+
Specify the workflow events for which notification emails should
205+
be sent.
206+
''',
207+
'options': WorkflowEventHandler.EVENTS.copy(),
208+
'depr_options': WorkflowEventHandler.EVENTS_DEPRECATED.copy(),
209+
},
185210
'startup handlers': f'''
186211
Handlers to run at scheduler startup.
187212
@@ -1038,7 +1063,12 @@ def default_for(
10381063

10391064
with Conf('events',
10401065
desc=default_for(EVENTS_DESCR, '[scheduler][events]')):
1041-
for item, desc in EVENTS_SETTINGS.items():
1066+
for item, val in EVENTS_SETTINGS.items():
1067+
if isinstance(val, dict):
1068+
desc: str = val.pop('desc', '')
1069+
else:
1070+
desc = val
1071+
val = {}
10421072
desc = default_for(desc, f"[scheduler][events]{item}")
10431073
vdr_type = VDR.V_STRING_LIST
10441074
default: Any = Conf.UNSET
@@ -1058,7 +1088,7 @@ def default_for(
10581088
default = DurationFloat(300)
10591089
else:
10601090
default = None
1061-
Conf(item, vdr_type, default, desc=desc)
1091+
Conf(item, vdr_type, default, desc=desc, **val)
10621092

10631093
with Conf('mail', desc=(
10641094
default_for(MAIL_DESCR, "[scheduler][mail]", section=True)

cylc/flow/cfgspec/workflow.py

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -411,9 +411,15 @@ def get_script_common_text(this: str, example: Optional[str] = None):
411411
)
412412
))
413413

414-
with Conf('events',
415-
desc=global_default(EVENTS_DESCR, '[scheduler][events]')):
416-
for item, desc in EVENTS_SETTINGS.items():
414+
with Conf(
415+
'events', desc=global_default(EVENTS_DESCR, '[scheduler][events]')
416+
):
417+
for item, val in EVENTS_SETTINGS.items():
418+
if isinstance(val, dict):
419+
desc: str = val.pop('desc', '')
420+
else:
421+
desc = val
422+
val = {}
417423
desc = global_default(desc, f"[scheduler][events]{item}")
418424
vdr_type = VDR.V_STRING_LIST
419425
default: Any = Conf.UNSET
@@ -444,7 +450,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
444450
vdr_type = VDR.V_BOOLEAN
445451
elif item.endswith("timeout"):
446452
vdr_type = VDR.V_INTERVAL
447-
Conf(item, vdr_type, default, desc=desc)
453+
Conf(item, vdr_type, default, desc=desc, **val)
448454

449455
Conf('expected task failures', VDR.V_STRING_LIST, desc='''
450456
(For Cylc developers writing a functional tests only)

cylc/flow/config.py

Lines changed: 38 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -542,7 +542,7 @@ def __init__(
542542
self.cfg['scheduling']['special tasks'][s_type] = result
543543

544544
self.process_config_env()
545-
self._check_and_upg_wflow_handler_events()
545+
self._upg_wflow_event_names()
546546

547547
self.mem_log("config.py: before load_graph()")
548548
self.load_graph()
@@ -560,7 +560,7 @@ def __init__(
560560
self.configure_workflow_state_polling_tasks()
561561

562562
for taskdef in self.taskdefs.values():
563-
self._check_task_handler_events(taskdef)
563+
self._check_task_event_names(taskdef)
564564
self._check_task_event_handlers(taskdef)
565565
self._check_special_tasks() # adds to self.implicit_tasks
566566
self._check_explicit_cycling()
@@ -1745,23 +1745,24 @@ def process_config_env(self):
17451745
]
17461746
)
17471747

1748-
def _check_task_handler_events(self, taskdef: 'TaskDef') -> None:
1749-
"""Validate task handler event names."""
1750-
handler_events: Optional[List[str]] = taskdef.rtconfig['events'][
1751-
'handler events'
1752-
]
1753-
if not handler_events:
1754-
return
1755-
invalid = set(handler_events).difference(
1756-
TaskEventsManager.STD_EVENTS,
1757-
taskdef.rtconfig['outputs'],
1758-
)
1759-
if invalid:
1760-
raise WorkflowConfigError(
1761-
"Invalid event name(s) for "
1762-
f"[runtime][{taskdef.name}][events]handler events: "
1763-
+ ', '.join(sorted(invalid))
1748+
def _check_task_event_names(self, taskdef: 'TaskDef') -> None:
1749+
"""Validate task handler/mail event names."""
1750+
for setting in ('handler events', 'mail events'):
1751+
event_names: Optional[List[str]] = taskdef.rtconfig['events'][
1752+
setting
1753+
]
1754+
if not event_names:
1755+
continue
1756+
invalid = set(event_names).difference(
1757+
TaskEventsManager.STD_EVENTS,
1758+
taskdef.rtconfig['outputs'],
17641759
)
1760+
if invalid:
1761+
raise WorkflowConfigError(
1762+
"Invalid event name(s) for "
1763+
f"[runtime][{taskdef.name}][events]{setting}: "
1764+
+ ', '.join(sorted(invalid))
1765+
)
17651766

17661767
def _check_task_event_handlers(self, taskdef: 'TaskDef') -> None:
17671768
"""Check custom event handler templates can be expanded.
@@ -2726,32 +2727,23 @@ def upgrade_clock_triggers(self):
27262727
for seq in taskdef.sequences:
27272728
taskdef.add_xtrig_label(label, seq)
27282729

2729-
def _check_and_upg_wflow_handler_events(self) -> None:
2730-
"""Validate workflow handler events and upgrade any Cylc 7 event names.
2731-
"""
2732-
handler_events: Optional[List[str]] = self.cfg['scheduler']['events'][
2733-
'handler events'
2734-
]
2735-
if not handler_events:
2736-
return
2737-
upgraded: Dict[str, str] = {}
2738-
for i, event in enumerate(handler_events):
2739-
if event in WorkflowEventHandler.EVENTS_DEPRECATED:
2740-
handler_events[i] = upgraded[event] = (
2741-
WorkflowEventHandler.EVENTS_DEPRECATED[event]
2742-
)
2743-
elif event not in WorkflowEventHandler.EVENTS:
2744-
valid = WorkflowEventHandler.EVENTS.copy()
2745-
if cylc.flow.flags.cylc7_back_compat:
2746-
valid += WorkflowEventHandler.EVENTS_DEPRECATED
2747-
raise WorkflowConfigError(
2748-
f"Invalid workflow handler event '{event}'. "
2749-
"[scheduler][events][handler events] must be one of: "
2750-
+ ', '.join(valid)
2730+
def _upg_wflow_event_names(self) -> None:
2731+
"""Upgrade any Cylc 7 workflow handler/mail events names."""
2732+
for setting in ('handler events', 'mail events'):
2733+
event_names: Optional[List[str]] = self.cfg['scheduler']['events'][
2734+
setting
2735+
]
2736+
if not event_names:
2737+
continue
2738+
upgraded: Dict[str, str] = {}
2739+
for i, event in enumerate(event_names):
2740+
if event in WorkflowEventHandler.EVENTS_DEPRECATED:
2741+
event_names[i] = upgraded[event] = (
2742+
WorkflowEventHandler.EVENTS_DEPRECATED[event]
2743+
)
2744+
if upgraded and not cylc.flow.flags.cylc7_back_compat:
2745+
LOG.warning(
2746+
f"{upgrader.DEPR_MSG}\n"
2747+
f" * (8.0.0) [scheduler][events][{setting}] "
2748+
+ ', '.join(f'{k} -> {v}' for k, v in upgraded.items())
27512749
)
2752-
if upgraded and not cylc.flow.flags.cylc7_back_compat:
2753-
LOG.warning(
2754-
f"{upgrader.DEPR_MSG}\n"
2755-
" * (8.0.0) [scheduler][events][handler events] "
2756-
+ ', '.join(f'{k} -> {v}' for k, v in upgraded.items())
2757-
)

cylc/flow/parsec/config.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,9 @@ class ConfigNode(ContextNode):
246246
options:
247247
List of possible options.
248248
TODO: allow this to be a dict with help info
249+
depr_options:
250+
List of deprecated options. These are not displayed in the docs
251+
but are used for backwards compatibility.
249252
default:
250253
The default value.
251254
desc:
@@ -273,7 +276,13 @@ class ConfigNode(ContextNode):
273276
UNSET = '*value unset*'
274277

275278
__slots__ = ContextNode.__slots__ + (
276-
'vdr', 'options', 'default', 'desc', 'display_name', 'meta'
279+
'vdr',
280+
'options',
281+
'depr_options',
282+
'default',
283+
'desc',
284+
'display_name',
285+
'meta',
277286
)
278287

279288
def __init__(
@@ -283,7 +292,8 @@ def __init__(
283292
default: object = UNSET,
284293
options: Optional[list] = None,
285294
desc: Optional[str] = None,
286-
meta: Optional['ConfigNode'] = None
295+
meta: Optional['ConfigNode'] = None,
296+
depr_options: Optional[list] = None,
287297
):
288298
display_name = name
289299
if name.startswith('<'):
@@ -306,7 +316,8 @@ def __init__(
306316
self.display_name = display_name
307317
self.vdr = vdr
308318
self.default = default
309-
self.options = options
319+
self.options = options or []
320+
self.depr_options = depr_options or []
310321
self.desc = dedent(desc).strip() if desc else None
311322
self.meta = meta
312323

cylc/flow/parsec/validate.py

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -230,13 +230,19 @@ def validate(self, cfg_root, spec_root):
230230
# Item is value, coerce according to value type
231231
cfg[key] = self.coercers[specval.vdr](value, keys + [key])
232232
if specval.options:
233-
voptions = specval.options
234-
if (isinstance(cfg[key], list) and
235-
any(val not in voptions for val in cfg[key]) or
236-
not isinstance(cfg[key], list) and
237-
cfg[key] not in voptions):
233+
voptions = {*specval.options, *specval.depr_options}
234+
if isinstance(cfg[key], list):
235+
bad = [
236+
str(i) for i in cfg[key] if i not in voptions
237+
]
238+
if bad:
239+
raise IllegalValueError(
240+
'option', [*keys, key], ', '.join(bad)
241+
)
242+
elif cfg[key] not in voptions:
238243
raise IllegalValueError(
239-
'option', keys + [key], cfg[key])
244+
'option', [*keys, key], cfg[key]
245+
)
240246

241247
__call__ = validate
242248

tests/integration/test_workflow_events.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ def mock_run_event_handlers(self, event, reason=""):
202202
# Configure a long-running shutdown handler.
203203
schd = test_scheduler({
204204
'shutdown handlers': 'sleep 10; echo',
205-
'mail events': [],
205+
'mail events': '',
206206
})
207207

208208
# Set a low process pool timeout value.

0 commit comments

Comments
 (0)