Merge pull request #6922 from cylc/8.5.x-sync

🤖 Merge 8.5.x-sync into master

Author: oliver-sanders

CHANGES.md

@@ -11,6 +11,12 @@ $ towncrier create <PR-number>.<break|feat|fix>.md --content "Short description"
 
 <!-- towncrier release notes start -->
 
+## __cylc-8.5.1 (Released 2025-08-08)__
+
+### 🚀 Enhancements
+
+[#6902](https://github.com/cylc/cylc-flow/pull/6902) - Invalid workflow events in the `mail events` or `handler events` configurations will result in warnings rather than errors.
+
 ## __cylc-8.5.0 (Released 2025-07-24)__
 
 ### 🚀 Enhancements

changes.d/6892.feat.md

@@ -0,0 +1 @@
+Added `--format=json` option to `cylc broadcast` for use with the `--display` option. Deprecated the `--raw` option in favour of `--format=raw`.

cylc/flow/cfgspec/globalcfg.py

@@ -63,6 +63,7 @@
 )
 from cylc.flow.pathutil import SYMLINKABLE_LOCATIONS
 from cylc.flow.platforms import validate_platforms
+from cylc.flow.task_events_mgr import TaskEventsManager as TEM
 from cylc.flow.workflow_events import WorkflowEventHandler
 
 
@@ -171,9 +172,11 @@
         Configure :term:`event handlers` that run when certain workflow
         events occur.
 
-        This section configures *workflow* event handlers; see
-        :cylc:conf:`flow.cylc[runtime][<namespace>][events]` for *task* event
-        handlers.
+        .. admonition:: Not to be confused with
+           :class: tip
+
+           For *task* events, see
+           :cylc:conf:`flow.cylc[runtime][<namespace>][events]`.
 
         Event handlers can be held in the workflow ``bin/`` directory,
         otherwise it is up to you to ensure their location is in ``$PATH``
@@ -194,18 +197,24 @@
 
             .. seealso::
 
-               :ref:`user_guide.scheduler.workflow_events`
+               :ref:`user_guide.scheduler.workflow_events.list`
         ''',
         'options': WorkflowEventHandler.EVENTS.copy(),
         'depr_options': WorkflowEventHandler.EVENTS_DEPRECATED.copy(),
+        'warn_options': True,
     },
     'mail events': {
         'desc': '''
             Specify the workflow events for which notification emails should
             be sent.
+
+            .. seealso::
+
+               :ref:`user_guide.scheduler.workflow_events.list`
         ''',
         'options': WorkflowEventHandler.EVENTS.copy(),
         'depr_options': WorkflowEventHandler.EVENTS_DEPRECATED.copy(),
+        'warn_options': True,
     },
     'startup handlers': f'''
         Handlers to run at scheduler startup.
@@ -546,7 +555,10 @@
 TASK_EVENTS_DESCR = '''
 Configure the task event handling system.
 
-See also :cylc:conf:`flow.cylc[scheduler][events]` for *workflow* events.
+.. admonition:: Not to be confused with
+   :class: tip
+
+   For *workflow* events, see :cylc:conf:`flow.cylc[scheduler][events]`.
 
 Task :term:`event handlers` are scripts to run when task events occur.
 
@@ -576,26 +588,18 @@
         For more information, see
         :ref:`user_guide.runtime.task_event_handling`.
 
-        For workflow events, see
-        :ref:`user_guide.scheduler.workflow_event_handling`.
-
         Example::
 
            echo %(event)s occurred in %(workflow)s >> my-log-file
 
     ''',
-    'execution timeout': '''
-        If a task has not finished after the specified interval, the execution
-        timeout event handler(s) will be called.
-    ''',
-    'handler events': '''
-        A list of events for which :cylc:conf:`[..]handlers` are run.
+    'handler events': f'''
+        :Options: ``{"``, ``".join(TEM.STD_EVENTS)}`` & any custom event
 
-        Specify the events for which the general task event handlers
-        :cylc:conf:`flow.cylc[runtime][<namespace>][events]handlers`
-        should be invoked.
+        A list of events for which :cylc:conf:`[..]handlers` are run.
 
-        See :ref:`user_guide.runtime.task_event_handling` for more information.
+        See :ref:`user_guide.runtime.task_event_handling.list` for more
+        information on task events.
 
         Example::
 
@@ -612,16 +616,25 @@
 
            PT10S, PT1M, PT5M
     ''',
-    'mail events': '''
-        Specify the events for which notification emails should be sent.
+    'mail events': f'''
+        :Options: ``{"``, ``".join(TEM.STD_EVENTS)}`` & any custom event
+
+        A list of events for which notification emails should be sent.
+
+        See :ref:`user_guide.runtime.task_event_handling.list` for more
+        information on task events.
 
         Example::
 
            submission failed, failed
     ''',
+    'execution timeout': '''
+        If a task has not finished after the specified interval, any configured
+        execution timeout event handler(s) will be called.
+    ''',
     'submission timeout': '''
-        If a task has not started after the specified interval, the submission
-        timeout event handler(s) will be called.
+        If a task has not started after the specified interval, any configured
+        submission timeout event handler(s) will be called.
     '''
 }
 
@@ -670,15 +683,36 @@ def comma_sep_section_note(version_changed: str = '') -> str:
 
 
 def short_descr(text: str) -> str:
-    """Get dedented one-paragraph description from long description."""
-    return dedent(text).split('\n\n', 1)[0]
+    r"""Get dedented one-paragraph description from long description.
+
+    Examples:
+        >>> short_descr('foo\n\nbar')
+        'foo'
+
+        >>> short_descr(':Field: Value\n\nfoo\n\nbar')
+        ':Field: Value\n\nfoo'
+
+    """
+    lines = []
+    for line in dedent(text).splitlines():
+        if not line:
+            continue
+        elif line.startswith(':'):
+            lines.append(line)
+        else:
+            lines.append(line)
+            break
+    return '\n\n'.join(lines)
 
 
 def default_for(
     text: str, config_path: str, section: bool = False
 ) -> str:
-    """Get dedented short description and insert a 'Default(s) For' directive
-    that links to this config item's flow.cylc counterpart."""
+    """Return a ":Default For: field for this config.
+
+    Get dedented short description and insert a 'Default(s) For' field
+    that links to this config item's flow.cylc counterpart.
+    """
     directive = f":Default{'s' if section else ''} For:"
     return (
         f"{directive} :cylc:conf:`flow.cylc{config_path}`.\n\n"
@@ -1998,12 +2032,6 @@ def default_for(
             TASK_EVENTS_DESCR, "[runtime][<namespace>][events]", section=True
         ) + "\n\n" + ".. versionadded:: 8.0.0"
     )):
-        Conf('execution timeout', VDR.V_INTERVAL, desc=(
-            default_for(
-                TASK_EVENTS_SETTINGS['execution timeout'],
-                "[runtime][<namespace>][events]execution timeout"
-            )
-        ))
         Conf('handlers', VDR.V_STRING_LIST, desc=(
             default_for(
                 TASK_EVENTS_SETTINGS['handlers'],
@@ -2028,6 +2056,12 @@ def default_for(
                 "[runtime][<namespace>][events]mail events"
             )
         ))
+        Conf('execution timeout', VDR.V_INTERVAL, desc=(
+            default_for(
+                TASK_EVENTS_SETTINGS['execution timeout'],
+                "[runtime][<namespace>][events]execution timeout"
+            )
+        ))
         Conf('submission timeout', VDR.V_INTERVAL, desc=(
             default_for(
                 TASK_EVENTS_SETTINGS['submission timeout'],

cylc/flow/cfgspec/workflow.py

@@ -1646,12 +1646,6 @@ def get_script_common_text(this: str, example: Optional[str] = None):
             with Conf('events', desc=(
                 global_default(TASK_EVENTS_DESCR, "[task events]")
             )):
-                Conf('execution timeout', VDR.V_INTERVAL, desc=(
-                    global_default(
-                        TASK_EVENTS_SETTINGS['execution timeout'],
-                        "[task events]execution timeout"
-                    )
-                ))
                 Conf('handlers', VDR.V_STRING_LIST, None, desc=(
                     global_default(
                         TASK_EVENTS_SETTINGS['handlers'],
@@ -1676,6 +1670,12 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                         "[task events]mail events"
                     )
                 ))
+                Conf('execution timeout', VDR.V_INTERVAL, desc=(
+                    global_default(
+                        TASK_EVENTS_SETTINGS['execution timeout'],
+                        "[task events]execution timeout"
+                    )
+                ))
                 Conf('submission timeout', VDR.V_INTERVAL, desc=(
                     global_default(
                         TASK_EVENTS_SETTINGS['submission timeout'],

cylc/flow/parsec/config.py

@@ -249,6 +249,10 @@ class ConfigNode(ContextNode):
         depr_options:
             List of deprecated options. These are not displayed in the docs
             but are used for backwards compatibility.
+        warn_options:
+            If True, parsec will warn if invalid options are present rather
+            than raising an exception. Any invalid options will be stripped
+            from the config.
         default:
             The default value.
         desc:
@@ -279,6 +283,7 @@ class ConfigNode(ContextNode):
         'vdr',
         'options',
         'depr_options',
+        'warn_options',
         'default',
         'desc',
         'display_name',
@@ -294,6 +299,7 @@ def __init__(
         desc: Optional[str] = None,
         meta: Optional['ConfigNode'] = None,
         depr_options: Optional[list] = None,
+        warn_options: bool = False,
     ):
         display_name = name
         if name.startswith('<'):
@@ -318,6 +324,7 @@ def __init__(
         self.default = default
         self.options = options or []
         self.depr_options = depr_options or []
+        self.warn_options = warn_options
         self.desc = dedent(desc).strip() if desc else None
         self.meta = meta
 

cylc/flow/parsec/validate.py

@@ -33,6 +33,7 @@
 from metomi.isodatetime.parsers import TimePointParser, DurationParser
 from metomi.isodatetime.exceptions import IsodatetimeError, ISO8601SyntaxError
 
+from cylc.flow import LOG
 from cylc.flow.parsec.exceptions import (
     ListValueError, IllegalValueError, IllegalItemError)
 from cylc.flow.subprocctx import SubFuncContext
@@ -236,9 +237,19 @@ def validate(self, cfg_root, spec_root):
                                 str(i) for i in cfg[key] if i not in voptions
                             ]
                             if bad:
-                                raise IllegalValueError(
+                                exc = IllegalValueError(
                                     'option', [*keys, key], ', '.join(bad)
                                 )
+                                if specval.warn_options:
+                                    LOG.warning(
+                                        f'{exc}'
+                                        '\nInvalid items have been removed'
+                                    )
+                                    cfg[key] = [
+                                        x for x in cfg[key] if x not in bad
+                                    ]
+                                else:
+                                    raise exc
                         elif cfg[key] not in voptions:
                             raise IllegalValueError(
                                 'option', [*keys, key], cfg[key]

cylc/flow/remote.py

@@ -84,7 +84,7 @@ def watch_and_kill(proc):
     """Kill proc if my PPID (etc.) changed - e.g. ssh connection dropped."""
     gpa = get_proc_ancestors()
     while True:
-        sleep(0.5)
+        sleep(60)
         if proc.poll() is not None:
             break
         if get_proc_ancestors() != gpa:

cylc/flow/scripts/broadcast.py

@@ -111,6 +111,11 @@
     from optparse import Values
 
 
+RAW_DEPR_MSG = (
+    "DEPRECATED: the --raw option will be removed at Cylc 8.7; "
+    "use --format=raw instead."
+)
+
 REC_ITEM = re.compile(r'^\[([^\]]*)\](.*)$')
 
 MUTATION = '''
@@ -312,12 +317,33 @@ def get_option_parser() -> COP:
         help="Use unicode box characters with -d, -k.",
         action="store_true", default=False, dest="unicode")
 
+    # BACK COMPAT: --raw
+    # From: < 8.5.1
+    # To: 8.5.1
+    # Remove at: 8.7.0
     parser.add_option(
         "-r", "--raw",
-        help="With -d/--display or -k/--display-task, write out "
-             "the broadcast config structure in raw Python form.",
+        help=(
+            "With -d/--display or -k/--display-task, write out "
+            "the broadcast config structure in raw Python form. "
+            f"{RAW_DEPR_MSG}"
+        ),
         action="store_true", default=False, dest="raw")
 
+    parser.add_option(
+        '--format',
+        help=(
+            "With -d/--display or -k/--display-task, write out "
+            "the broadcast config structure in one of the following formats: "
+            "tree, json, or raw (like json but as a Python dictionary). "
+            r"Default: %default."
+        ),
+        action='store',
+        dest='format',
+        choices=('tree', 'json', 'raw'),
+        default='tree',
+    )
+
     return parser
 
 
@@ -356,6 +382,9 @@ async def run(options: 'Values', workflow_id):
 
     }
 
+    if options.raw:
+        ret['stderr'].append(cparse(f"<yellow>{RAW_DEPR_MSG}</yellow>"))
+
     if options.show or options.showtask:
         if options.showtask:
             try:
@@ -369,7 +398,12 @@ async def run(options: 'Values', workflow_id):
         for wflow in result['workflows']:
             settings = wflow['broadcasts']
             padding = get_padding(settings) * ' '
-            if options.raw:
+            if options.format == 'json':
+                import json
+                ret['stdout'].append(
+                    json.dumps(settings, indent=2, sort_keys=True)
+                )
+            elif options.raw or options.format == 'raw':
                 ret['stdout'].append(str(settings))
             else:
                 ret['stdout'].extend(

cylc/flow/task_pool.py

@@ -708,14 +708,12 @@ def load_db_task_action_timers(self, row_idx: int, row: Iterable) -> None:
         if ctx_key == "poll_timer":
             itask = self._get_task_by_id(id_)
             if itask is None:
-                LOG.warning("%(id)s: task not found, skip" % {"id": id_})
                 return
             itask.poll_timer = TaskActionTimer(
                 ctx, delays, num, delay, timeout)
         elif ctx_key[0] == "try_timers":
             itask = self._get_task_by_id(id_)
             if itask is None:
-                LOG.warning("%(id)s: task not found, skip" % {"id": id_})
                 return
             if 'retrying' in ctx_key[1]:
                 if 'submit' in ctx_key[1]: