diff --git a/changes.d/5658.feat.md b/changes.d/5658.feat.md new file mode 100644 index 00000000000..ed33a09b529 --- /dev/null +++ b/changes.d/5658.feat.md @@ -0,0 +1 @@ +New "cylc set" command for setting task prerequisites and outputs. diff --git a/cylc/flow/data_store_mgr.py b/cylc/flow/data_store_mgr.py index ef77105b3a4..7a59e76d7a4 100644 --- a/cylc/flow/data_store_mgr.py +++ b/cylc/flow/data_store_mgr.py @@ -2185,8 +2185,9 @@ def update_workflow(self, reloaded=False): w_delta.n_edge_distance = self.n_edge_distance delta_set = True - if self.schd.pool.main_pool: - pool_points = set(self.schd.pool.main_pool) + if self.schd.pool.active_tasks: + pool_points = set(self.schd.pool.active_tasks) + oldest_point = str(min(pool_points)) if w_data.oldest_active_cycle_point != oldest_point: w_delta.oldest_active_cycle_point = oldest_point diff --git a/cylc/flow/etc/cylc b/cylc/flow/etc/cylc index 48a91c0603a..facf27e4291 100755 --- a/cylc/flow/etc/cylc +++ b/cylc/flow/etc/cylc @@ -2,7 +2,7 @@ # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# +# # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or diff --git a/cylc/flow/flow_mgr.py b/cylc/flow/flow_mgr.py index 41aec947a80..7bb293bfc85 100644 --- a/cylc/flow/flow_mgr.py +++ b/cylc/flow/flow_mgr.py @@ -20,50 +20,164 @@ import datetime from cylc.flow import LOG +from cylc.flow.exceptions import InputError if TYPE_CHECKING: from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager - FlowNums = Set[int] # Flow constants FLOW_ALL = "all" FLOW_NEW = "new" FLOW_NONE = "none" +# For flow-related CLI options: +ERR_OPT_FLOW_VAL = "Flow values must be an integer, or 'all', 'new', or 'none'" +ERR_OPT_FLOW_INT = "Multiple flow options must all be integer valued" +ERR_OPT_FLOW_WAIT = ( + f"--wait is not compatible with --flow={FLOW_NEW} or --flow={FLOW_NONE}" +) + + +def add_flow_opts(parser): + parser.add_option( + "--flow", action="append", dest="flow", metavar="FLOW", + help=f'Assign new tasks to all active flows ("{FLOW_ALL}");' + f' no flow ("{FLOW_NONE}"); a new flow ("{FLOW_NEW}");' + f' or a specific flow (e.g. "2"). The default is "{FLOW_ALL}".' + ' Specific flow numbers can be new or existing.' + ' Reuse the option to assign multiple flow numbers.' + ) + + parser.add_option( + "--meta", metavar="DESCRIPTION", action="store", + dest="flow_descr", default=None, + help=f"description of new flow (with --flow={FLOW_NEW})." + ) + + parser.add_option( + "--wait", action="store_true", default=False, dest="flow_wait", + help="Wait for merge with current active flows before flowing on." + " Note you can use 'cylc set --pre=all' to unset a flow-wait." + ) + + +def validate_flow_opts(options): + """Check validity of flow-related CLI options.""" + if options.flow is None: + # Default to all active flows + options.flow = [FLOW_ALL] + + for val in options.flow: + val = val.strip() + if val in [FLOW_NONE, FLOW_NEW, FLOW_ALL]: + if len(options.flow) != 1: + raise InputError(ERR_OPT_FLOW_INT) + else: + try: + int(val) + except ValueError: + raise InputError(ERR_OPT_FLOW_VAL.format(val)) + + if options.flow_wait and options.flow[0] in [FLOW_NEW, FLOW_NONE]: + raise InputError(ERR_OPT_FLOW_WAIT) + + +def stringify_flow_nums(flow_nums: Set[int], full: bool = False) -> str: + """Return a string representation of a set of flow numbers + + Return: + - "none" for no flow + - "" for the original flow (flows only matter if there are several) + - otherwise e.g. "(flow=1,2,3)" + + Examples: + >>> stringify_flow_nums({}) + '(flows=none)' + + >>> stringify_flow_nums({1}) + '' + + >>> stringify_flow_nums({1}, True) + '(flows=1)' + + >>> stringify_flow_nums({1,2,3}) + '(flows=1,2,3)' + + """ + if not full and flow_nums == {1}: + return "" + else: + return ( + "(flows=" + f"{','.join(str(i) for i in flow_nums) or 'none'}" + ")" + ) + class FlowMgr: """Logic to manage flow counter and flow metadata.""" - def __init__(self, db_mgr: "WorkflowDatabaseManager") -> None: + def __init__( + self, + db_mgr: "WorkflowDatabaseManager", + utc: bool = True + ) -> None: """Initialise the flow manager.""" self.db_mgr = db_mgr self.flows: Dict[int, Dict[str, str]] = {} self.counter: int = 0 + self._timezone = datetime.timezone.utc if utc else None - def get_new_flow(self, description: Optional[str] = None) -> int: - """Increment flow counter, record flow metadata.""" - self.counter += 1 - # record start time to nearest second - now = datetime.datetime.now() - now_sec: str = str( - now - datetime.timedelta(microseconds=now.microsecond)) - description = description or "no description" - self.flows[self.counter] = { - "description": description, - "start_time": now_sec - } - LOG.info( - f"New flow: {self.counter} " - f"({description}) " - f"{now_sec}" - ) - self.db_mgr.put_insert_workflow_flows( - self.counter, - self.flows[self.counter] - ) - return self.counter + def get_flow_num( + self, + flow_num: Optional[int] = None, + meta: Optional[str] = None + ) -> int: + """Return a valid flow number, and record a new flow if necessary. + + If asked for a new flow: + - increment the automatic counter until we find an unused number + + If given a flow number: + - record a new flow if the number is unused + - else return it, as an existing flow number. + + The metadata string is only used if it is a new flow. + + """ + if flow_num is None: + self.counter += 1 + while self.counter in self.flows: + # Skip manually-created out-of-sequence flows. + self.counter += 1 + flow_num = self.counter + + if flow_num in self.flows: + if meta is not None: + LOG.warning( + f'Ignoring flow metadata "{meta}":' + f' {flow_num} is not a new flow' + ) + else: + # Record a new flow. + now_sec = datetime.datetime.now(tz=self._timezone).isoformat( + timespec="seconds" + ) + meta = meta or "no description" + self.flows[flow_num] = { + "description": meta, + "start_time": now_sec + } + LOG.info( + f"New flow: {flow_num} ({meta}) {now_sec}" + ) + self.db_mgr.put_insert_workflow_flows( + flow_num, + self.flows[flow_num] + ) + return flow_num def load_from_db(self, flow_nums: FlowNums) -> None: """Load flow data for scheduler restart. diff --git a/cylc/flow/graph_parser.py b/cylc/flow/graph_parser.py index ad0ec280a3d..3dbab9261b7 100644 --- a/cylc/flow/graph_parser.py +++ b/cylc/flow/graph_parser.py @@ -33,6 +33,7 @@ from cylc.flow.task_id import TaskID from cylc.flow.task_trigger import TaskTrigger from cylc.flow.task_outputs import ( + TASK_OUTPUT_EXPIRED, TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_STARTED, TASK_OUTPUT_FAILED, @@ -41,6 +42,8 @@ TASK_OUTPUT_SUBMIT_FAILED ) from cylc.flow.task_qualifiers import ( + QUAL_FAM_EXPIRE_ALL, + QUAL_FAM_EXPIRE_ANY, QUAL_FAM_SUCCEED_ALL, QUAL_FAM_SUCCEED_ANY, QUAL_FAM_FAIL_ALL, @@ -124,6 +127,8 @@ class GraphParser: # E.g. QUAL_FAM_START_ALL: (TASK_OUTPUT_STARTED, True) simply maps # "FAM:start-all" to "MEMBER:started" and "-all" (all members). fam_to_mem_trigger_map: Dict[str, Tuple[str, bool]] = { + QUAL_FAM_EXPIRE_ALL: (TASK_OUTPUT_EXPIRED, True), + QUAL_FAM_EXPIRE_ANY: (TASK_OUTPUT_EXPIRED, False), QUAL_FAM_START_ALL: (TASK_OUTPUT_STARTED, True), QUAL_FAM_START_ANY: (TASK_OUTPUT_STARTED, False), QUAL_FAM_SUCCEED_ALL: (TASK_OUTPUT_SUCCEEDED, True), @@ -140,6 +145,8 @@ class GraphParser: # Map family pseudo triggers to affected member outputs. fam_to_mem_output_map: Dict[str, List[str]] = { + QUAL_FAM_EXPIRE_ANY: [TASK_OUTPUT_EXPIRED], + QUAL_FAM_EXPIRE_ALL: [TASK_OUTPUT_EXPIRED], QUAL_FAM_START_ANY: [TASK_OUTPUT_STARTED], QUAL_FAM_START_ALL: [TASK_OUTPUT_STARTED], QUAL_FAM_SUCCEED_ANY: [TASK_OUTPUT_SUCCEEDED], @@ -738,6 +745,10 @@ def _set_output_opt( if suicide: return + if output == TASK_OUTPUT_EXPIRED and not optional: + raise GraphParseError( + f"Expired-output {name}:{output} must be optional") + if output == TASK_OUTPUT_FINISHED: # Interpret :finish pseudo-output if optional: diff --git a/cylc/flow/id_match.py b/cylc/flow/id_match.py index bbe087d7fe8..de3fa21cb16 100644 --- a/cylc/flow/id_match.py +++ b/cylc/flow/id_match.py @@ -76,7 +76,7 @@ def filter_ids( - pools: 'List[Pool]', + pool: 'Pool', ids: 'Iterable[str]', *, warn: 'bool' = True, @@ -145,8 +145,6 @@ def filter_ids( if tokens.get(lowest_token.value): break - # This needs to be a set to avoid getting two copies of matched tasks - # in cycle points that appear in both pools: cycles = set() tasks = [] @@ -154,19 +152,18 @@ def filter_ids( if lowest_token == IDTokens.Cycle: cycle = tokens[IDTokens.Cycle.value] cycle_sel = tokens.get(IDTokens.Cycle.value + '_sel') or '*' - for pool in pools: - for icycle, itasks in pool.items(): - if not itasks: - continue - if not point_match(icycle, cycle, pattern_match): - continue - if cycle_sel == '*': + for icycle, itasks in pool.items(): + if not itasks: + continue + if not point_match(icycle, cycle, pattern_match): + continue + if cycle_sel == '*': + cycles.add(icycle) + continue + for itask in itasks.values(): + if match(itask.state.status, cycle_sel): cycles.add(icycle) - continue - for itask in itasks.values(): - if match(itask.state.status, cycle_sel): - cycles.add(icycle) - break + break # filter by task elif lowest_token == IDTokens.Task: # noqa SIM106 @@ -176,36 +173,35 @@ def filter_ids( task = tokens[IDTokens.Task.value] task_sel_raw = tokens.get(IDTokens.Task.value + '_sel') task_sel = task_sel_raw or '*' - for pool in pools: - for icycle, itasks in pool.items(): - if not point_match(icycle, cycle, pattern_match): - continue - for itask in itasks.values(): - if ( - # check cycle selector + for icycle, itasks in pool.items(): + if not point_match(icycle, cycle, pattern_match): + continue + for itask in itasks.values(): + if ( + # check cycle selector + ( ( - ( - # disable cycle_sel if not defined if - # pattern matching is turned off - pattern_match is False - and cycle_sel_raw is None - ) - or match(itask.state.status, cycle_sel) + # disable cycle_sel if not defined if + # pattern matching is turned off + pattern_match is False + and cycle_sel_raw is None ) - # check namespace name - and itask.name_match(task, match_func=match) - # check task selector - and ( - ( - # disable task_sel if not defined if - # pattern matching is turned off - pattern_match is False - and task_sel_raw is None - ) - or match(itask.state.status, task_sel) + or match(itask.state.status, cycle_sel) + ) + # check namespace name + and itask.name_match(task, match_func=match) + # check task selector + and ( + ( + # disable task_sel if not defined if + # pattern matching is turned off + pattern_match is False + and task_sel_raw is None ) - ): - tasks.append(itask) + or match(itask.state.status, task_sel) + ) + ): + tasks.append(itask) else: raise NotImplementedError @@ -226,10 +222,9 @@ def filter_ids( }) ret = _cycles elif out == IDTokens.Task: - for pool in pools: - for icycle in _cycles: - if icycle in pool: - _tasks.extend(pool[icycle].values()) + for icycle in _cycles: + if icycle in pool: + _tasks.extend(pool[icycle].values()) ret = _tasks return ret, _not_matched diff --git a/cylc/flow/network/resolvers.py b/cylc/flow/network/resolvers.py index 5af50b0a4f8..15acb9416ed 100644 --- a/cylc/flow/network/resolvers.py +++ b/cylc/flow/network/resolvers.py @@ -26,7 +26,6 @@ from typing import ( Any, Dict, - Iterable, List, NamedTuple, Optional, @@ -58,7 +57,6 @@ from graphql import ResolveInfo from cylc.flow.data_store_mgr import DataStoreMgr from cylc.flow.scheduler import Scheduler - from cylc.flow.workflow_status import StopMode class TaskMsg(NamedTuple): @@ -706,32 +704,39 @@ async def mutator( 'response': (False, f'No matching workflow in {workflows}')}] w_id = w_ids[0] result = await self._mutation_mapper(command, kwargs, meta) - if result is None: - result = (True, 'Command queued') return [{'id': w_id, 'response': result}] - def _log_command(self, command: str, user: str) -> None: - """Log receipt of command, with user name if not owner.""" - is_owner = user == self.schd.owner - if command == 'put_messages' and is_owner: - # Logging put_messages is overkill. - return - log_msg = f"[command] {command}" - if not is_owner: - log_msg += (f" (issued by {user})") - LOG.info(log_msg) - async def _mutation_mapper( self, command: str, kwargs: Dict[str, Any], meta: Dict[str, Any] - ) -> Optional[Tuple[bool, str]]: - """Map between GraphQL resolvers and internal command interface.""" + ) -> Tuple[bool, str]: + """Map to internal command interface. - self._log_command( - command, - meta.get('auth_user', self.schd.owner) + Some direct methods are in this module. + Others go to the scheduler command queue. + + """ + user = meta.get('auth_user', self.schd.owner) + if user == self.schd.owner: + log_user = "" # don't log user name if owner + else: + log_user = f" from {user}" + + log1 = f'Command "{command}" received{log_user}.' + log2 = ( + f"{command}(" + + ", ".join( + f"{key}={value}" for key, value in kwargs.items()) + + ")" ) + method = getattr(self, command, None) if method is not None: + if ( + command != "put_messages" + or user != self.schd.owner + ): + # Logging task messages as commands is overkill. + LOG.info(f"{log1}\n{log2}") return method(**kwargs) try: @@ -739,11 +744,18 @@ async def _mutation_mapper( except AttributeError: raise ValueError(f"Command '{command}' not found") - self.schd.queue_command( - command, - kwargs + # Queue the command to the scheduler, with a unique command ID + cmd_uuid = str(uuid4()) + LOG.info(f"{log1} ID={cmd_uuid}\n{log2}") + self.schd.command_queue.put( + ( + cmd_uuid, + command, + [], + kwargs, + ) ) - return None + return (True, cmd_uuid) def broadcast( self, @@ -846,107 +858,3 @@ def set_graph_window_extent(self, n_edge_distance): return (True, f'Maximum edge distance set to {n_edge_distance}') else: return (False, 'Edge distance cannot be negative') - - def force_spawn_children( - self, - tasks: Iterable[str], - outputs: Optional[Iterable[str]] = None, - flow_num: Optional[int] = None - ) -> Tuple[bool, str]: - """Spawn children of given task outputs. - - User-facing method name: set_outputs. - - Args: - tasks: List of identifiers or task globs. - outputs: List of outputs to spawn on. - flow_num: Flow number to attribute the outputs. - """ - self.schd.command_queue.put( - ( - "force_spawn_children", - (tasks,), - { - "outputs": outputs, - "flow_num": flow_num - }, - ) - ) - return (True, 'Command queued') - - def stop( - self, - mode: Union[str, 'StopMode'], - cycle_point: Optional[str] = None, - clock_time: Optional[str] = None, - task: Optional[str] = None, - flow_num: Optional[int] = None, - ) -> Tuple[bool, str]: - """Stop the workflow or specific flow from spawning any further. - - Args: - mode: Stop mode to set - cycle_point: Cycle point after which to stop. - clock_time: Wallclock time after which to stop. - task: Stop after this task succeeds. - flow_num: The flow to stop. - ): - - Returns: - outcome: True if command successfully queued. - message: Information about outcome. - - """ - self.schd.command_queue.put(( - "stop", - (), - filter_none({ - 'mode': mode, - 'cycle_point': cycle_point, - 'clock_time': clock_time, - 'task': task, - 'flow_num': flow_num, - }), - ) - ) - return (True, 'Command queued') - - def force_trigger_tasks( - self, - tasks: Iterable[str], - flow: Iterable[str], - flow_wait: bool, - flow_descr: Optional[str] = None, - ): - """Trigger submission of task jobs where possible. - - Args: - tasks (list): - List of identifiers or task globs. - flow (list): - Flow ownership of triggered tasks. - flow_wait (bool): - Wait for flows before continuing - flow_descr (str): - Description of new flow. - - Returns: - tuple: (outcome, message) - outcome (bool) - True if command successfully queued. - message (str) - Information about outcome. - - """ - self.schd.command_queue.put( - ( - "force_trigger_tasks", - (tasks or [],), - { - "flow": flow, - "flow_wait": flow_wait, - "flow_descr": flow_descr - } - ), - ) - return (True, 'Command queued') diff --git a/cylc/flow/network/schema.py b/cylc/flow/network/schema.py index d076882ae73..a1ae2ea26f7 100644 --- a/cylc/flow/network/schema.py +++ b/cylc/flow/network/schema.py @@ -43,7 +43,6 @@ from cylc.flow.id import Tokens from cylc.flow.task_outputs import SORT_ORDERS from cylc.flow.task_state import ( - TASK_OUTPUT_SUCCEEDED, TASK_STATUSES_ORDERED, TASK_STATUS_DESC, TASK_STATUS_WAITING, @@ -1005,6 +1004,14 @@ class Meta: time = Float() +class OutputLabel(String): + """Task output, e.g. "succeeded".""" + + +class PrerequisiteString(String): + """A task prerequisite, e.g. "2040/foo:succeeded".""" + + class XTrigger(ObjectType): class Meta: description = """Task trigger""" @@ -2097,27 +2104,33 @@ class Meta: resolver = partial(mutator, command='remove_tasks') -class SetOutputs(Mutation, TaskMutation): +class SetPrereqsAndOutputs(Mutation, TaskMutation): class Meta: - description = sstrip(''' - Artificially mark task outputs as completed. + description = sstrip(""" + Set task prerequisites or outputs. - This allows you to manually intervene with Cylc's scheduling - algorithm by artificially satisfying outputs of tasks. + By default, set all required outputs for target task(s). - By default this makes tasks appear as if they succeeded. + Setting prerequisites contributes to the task's readiness to run. - Valid for: paused, running workflows. - ''') - resolver = partial(mutator, command='force_spawn_children') + Setting outputs contributes to the task's completion, sets the + corresponding prerequisites of child tasks, and sets any implied + outputs: + - ``started`` implies ``submitted``. + - ``succeeded`` and ``failed`` imply ``started``. + - custom outputs and ``expired`` do not imply any other outputs. + """) + resolver = partial(mutator, command='set') - class Arguments(TaskMutation.Arguments): + class Arguments(TaskMutation.Arguments, FlowMutationArguments): outputs = graphene.List( - String, - default_value=[TASK_OUTPUT_SUCCEEDED], - description='List of task outputs to satisfy.' + OutputLabel, + description='List of task outputs to set complete.' + ) + prerequisites = graphene.List( + PrerequisiteString, + description='List of task prerequisites to set satisfied.' ) - flow_num = Int() class Trigger(Mutation, TaskMutation): @@ -2177,7 +2190,7 @@ class Mutations(ObjectType): poll = _mut_field(Poll) release = _mut_field(Release) remove = _mut_field(Remove) - set_outputs = _mut_field(SetOutputs) + set = _mut_field(SetPrereqsAndOutputs) # noqa A003 trigger = _mut_field(Trigger) # job actions diff --git a/cylc/flow/option_parsers.py b/cylc/flow/option_parsers.py index a60bbb58288..7d760d8293b 100644 --- a/cylc/flow/option_parsers.py +++ b/cylc/flow/option_parsers.py @@ -300,7 +300,7 @@ class CylcOptionParser(OptionParser): action='store_false', dest='log_timestamp', default=True, useif='all'), OptionSettings( - ['--color', '--color'], metavar='WHEN', action='store', + ['--color', '--colour'], metavar='WHEN', action='store', default='auto', choices=['never', 'auto', 'always'], help=( "When to use color/bold text in terminal output." diff --git a/cylc/flow/prerequisite.py b/cylc/flow/prerequisite.py index 31b5db83486..bfe5a50fc46 100644 --- a/cylc/flow/prerequisite.py +++ b/cylc/flow/prerequisite.py @@ -18,6 +18,7 @@ import math import re +from typing import Iterable, Set, TYPE_CHECKING from cylc.flow.cycling.loader import get_point from cylc.flow.exceptions import TriggerExpressionError @@ -27,6 +28,9 @@ ) from cylc.flow.id import quick_relative_detokenise +if TYPE_CHECKING: + from cylc.flow.id import Tokens + class Prerequisite: """The concrete result of an abstract logical trigger expression. @@ -194,20 +198,25 @@ def _conditional_is_satisfied(self): '"%s":\n%s' % (self.get_raw_conditional_expression(), err_msg)) return res - def satisfy_me(self, all_task_outputs): - """Evaluate prerequisite against known outputs. + def satisfy_me(self, outputs: Iterable['Tokens']) -> 'Set[Tokens]': + """Attempt to satisfy me with given outputs. - Updates cache with the evaluation result. + Updates cache with the result. + Return outputs that match. """ - relevant_messages = all_task_outputs & set(self.satisfied) - for message in relevant_messages: - self.satisfied[message] = self.DEP_STATE_SATISFIED + valid = set() + for output in outputs: + prereq = (output['cycle'], output['task'], output['task_sel']) + if prereq not in self.satisfied: + continue + valid.add(output) + self.satisfied[prereq] = self.DEP_STATE_SATISFIED if self.conditional_expression is None: self._all_satisfied = all(self.satisfied.values()) else: self._all_satisfied = self._conditional_is_satisfied() - return relevant_messages + return valid def api_dump(self): """Return list of populated Protobuf data objects.""" diff --git a/cylc/flow/rundb.py b/cylc/flow/rundb.py index 256da3b3129..1dc5db68b0b 100644 --- a/cylc/flow/rundb.py +++ b/cylc/flow/rundb.py @@ -21,7 +21,15 @@ from pprint import pformat import sqlite3 import traceback -from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union +from typing import ( + TYPE_CHECKING, + Iterable, + List, + Set, + Tuple, + Optional, + Union +) from cylc.flow import LOG from cylc.flow.exceptions import PlatformLookupError @@ -776,29 +784,32 @@ def select_task_job_platforms(self): ''' # nosec (table name is code constant) return {i[0] for i in self.connect().execute(stmt)} - def select_submit_nums(self, name, point): - """Select submit_num and flow_nums from task_states table. - - Fetch submit number and flow_nums for spawning tasks. - - Return: {submit_num: (flow_wait, flow_nums)} - - Args: - name: task name - point: task cycle point (str) + def select_prev_instances( + self, name: str, point: str + ) -> List[Tuple[int, bool, Set[int], int]]: + """Select task_states table info about previous instances of a task. + Flow merge results in multiple entries for the same submit number. """ # Ignore bandit false positive: B608: hardcoded_sql_expressions # Not an injection, simply putting the table name in the SQL query # expression as a string constant local to this module. stmt = ( # nosec - r"SELECT flow_nums,submit_num,flow_wait FROM %(name)s" + r"SELECT flow_nums,submit_num,flow_wait,status FROM %(name)s" r" WHERE name==? AND cycle==?" ) % {"name": self.TABLE_TASK_STATES} - ret = {} - for flow_nums_str, submit_num, flow_wait in self.connect().execute( - stmt, (name, point,)): - ret[submit_num] = (flow_wait == 1, deserialise(flow_nums_str)) + ret = [] + for flow_nums_str, submit_num, flow_wait, status in ( + self.connect().execute(stmt, (name, point,)) + ): + ret.append( + ( + submit_num, + flow_wait == 1, + deserialise(flow_nums_str), + status + ) + ) return ret def select_latest_flow_nums(self): diff --git a/cylc/flow/scheduler.py b/cylc/flow/scheduler.py index 9b098d2bbcd..9446af3dc6c 100644 --- a/cylc/flow/scheduler.py +++ b/cylc/flow/scheduler.py @@ -223,7 +223,7 @@ class Scheduler: flow_mgr: FlowMgr # queues - command_queue: 'Queue[Tuple[str, tuple, dict]]' + command_queue: 'Queue[Tuple[str, str, list, dict]]' message_queue: 'Queue[TaskMsg]' ext_trigger_queue: Queue @@ -364,7 +364,6 @@ async def initialise(self): self.data_store_mgr = DataStoreMgr(self) self.broadcast_mgr = BroadcastMgr( self.workflow_db_mgr, self.data_store_mgr) - self.flow_mgr = FlowMgr(self.workflow_db_mgr) self.server = WorkflowRuntimeServer(self) @@ -458,6 +457,8 @@ async def configure(self, params): self.options.cycle_point_tz = ( self.config.cfg['scheduler']['cycle point time zone']) + self.flow_mgr = FlowMgr(self.workflow_db_mgr, self.options.utc_mode) + # Note that daemonization happens after this: self.log_start() @@ -522,7 +523,7 @@ async def configure(self, params): timer.reset() self.timers[event] = timer - if self.is_restart and not self.pool.get_all_tasks(): + if self.is_restart and not self.pool.get_tasks(): # This workflow completed before restart; wait for intervention. with suppress(KeyError): self.timers[self.EVENT_RESTART_TIMEOUT].reset() @@ -632,7 +633,7 @@ async def run_scheduler(self) -> None: self.task_job_mgr.task_remote_mgr.is_restart = True self.task_job_mgr.task_remote_mgr.rsync_includes = ( self.config.get_validated_rsync_includes()) - if self.pool.get_all_tasks(): + if self.pool.get_tasks(): # (If we're not restarting a finished workflow) self.restart_remote_init() self.command_poll_tasks(['*/*']) @@ -874,8 +875,12 @@ def _load_task_run_times(self, row_idx, row): return def process_queued_task_messages(self) -> None: - """Handle incoming task messages for each task proxy.""" + """Process incoming task messages for each task proxy. + + """ messages: 'Dict[str, List[Tuple[Optional[int], TaskMsg]]]' = {} + + # Retrieve queued messages while self.message_queue.qsize(): try: task_msg = self.message_queue.get(block=False) @@ -891,15 +896,15 @@ def process_queued_task_messages(self) -> None: messages[task_id].append( (job, task_msg) ) - # Note on to_poll_tasks: If an incoming message is going to cause a - # reverse change to task state, it is desirable to confirm this by - # polling. + + # Poll tasks for which messages caused a backward state change. to_poll_tasks = [] for itask in self.pool.get_tasks(): message_items = messages.get(itask.identity) if message_items is None: continue should_poll = False + del messages[itask.identity] for submit_num, tm in message_items: if self.task_events_mgr.process_message( itask, tm.severity, tm.message, tm.event_time, @@ -908,19 +913,22 @@ def process_queued_task_messages(self) -> None: should_poll = True if should_poll: to_poll_tasks.append(itask) - self.task_job_mgr.poll_task_jobs( - self.workflow, to_poll_tasks) + if to_poll_tasks: + self.task_job_mgr.poll_task_jobs(self.workflow, to_poll_tasks) + + # Remaining unprocessed messages have no corresponding task proxy. + # For example, if I manually set a running task to succeeded, the + # proxy can be removed, but the orphaned job still sends messages. + for _id, tms in messages.items(): + warn = "Undeliverable task messages received and ignored:" + for _, msg in tms: + warn += f'\n {msg.job_id}: {msg.severity} - "{msg.message}"' + LOG.warning(warn) def get_command_method(self, command_name: str) -> Callable: """Return a command processing method or raise AttributeError.""" return getattr(self, f'command_{command_name}') - def queue_command(self, command: str, kwargs: dict) -> None: - self.command_queue.put(( - command, - tuple(kwargs.values()), {} - )) - async def process_command_queue(self) -> None: """Process queued commands.""" qsize = self.command_queue.qsize() @@ -928,17 +936,15 @@ async def process_command_queue(self) -> None: return LOG.debug(f"Processing {qsize} queued command(s)") while True: + uuid: str + name: str + args: list + kwargs: dict try: - command = self.command_queue.get(False) - name, args, kwargs = command + uuid, name, args, kwargs = self.command_queue.get(False) except Empty: break - args_string = ', '.join(str(a) for a in args) - kwargs_string = ', '.join( - f"{key}={value}" for key, value in kwargs.items() - ) - sep = ', ' if kwargs_string and args_string else '' - cmdstr = f"{name}({args_string}{sep}{kwargs_string})" + msg = f'Command "{name}" ' + '{result}' + f'. ID={uuid}' try: fcn = self.get_command_method(name) n_warnings: Optional[int] @@ -953,16 +959,20 @@ async def process_command_queue(self) -> None: not isinstance(exc, CommandFailedError) ): LOG.error(traceback.format_exc()) - LOG.error(f"Command failed: {cmdstr}\n{exc}") + LOG.error( + msg.format(result="failed") + f"\n{exc}" + ) else: if n_warnings: LOG.info( - f"Command actioned with {n_warnings} warning(s): " - f"{cmdstr}" + msg.format( + result=f"actioned with {n_warnings} warnings" + ) ) else: - LOG.info(f"Command actioned: {cmdstr}") + LOG.info(msg.format(result="actioned")) self.is_updated = True + self.command_queue.task_done() def info_get_graph_raw(self, cto, ctn, grouping=None): @@ -1025,9 +1035,9 @@ def _set_stop(self, stop_mode: Optional[StopMode] = None) -> None: self.stop_mode = stop_mode self.update_data_store() - def command_release(self, task_globs: Iterable[str]) -> int: + def command_release(self, tasks: Iterable[str]) -> int: """Release held tasks.""" - return self.pool.release_held_tasks(task_globs) + return self.pool.release_held_tasks(tasks) def command_release_hold_point(self) -> None: """Release all held tasks and unset workflow hold after cycle point, @@ -1039,17 +1049,17 @@ def command_resume(self) -> None: """Resume paused workflow.""" self.resume_workflow() - def command_poll_tasks(self, items: List[str]) -> int: + def command_poll_tasks(self, tasks: Iterable[str]) -> int: """Poll pollable tasks or a task or family if options are provided.""" if self.config.run_mode('simulation'): return 0 - itasks, _, bad_items = self.pool.filter_task_proxies(items) + itasks, _, bad_items = self.pool.filter_task_proxies(tasks) self.task_job_mgr.poll_task_jobs(self.workflow, itasks) return len(bad_items) - def command_kill_tasks(self, items: List[str]) -> int: + def command_kill_tasks(self, tasks: Iterable[str]) -> int: """Kill all tasks or a task/family if options are provided.""" - itasks, _, bad_items = self.pool.filter_task_proxies(items) + itasks, _, bad_items = self.pool.filter_task_proxies(tasks) if self.config.run_mode('simulation'): for itask in itasks: if itask.state(*TASK_STATUSES_ACTIVE): @@ -1059,9 +1069,9 @@ def command_kill_tasks(self, items: List[str]) -> int: self.task_job_mgr.kill_task_jobs(self.workflow, itasks) return len(bad_items) - def command_hold(self, task_globs: Iterable[str]) -> int: + def command_hold(self, tasks: Iterable[str]) -> int: """Hold specified tasks.""" - return self.pool.hold_tasks(task_globs) + return self.pool.hold_tasks(tasks) def command_set_hold_point(self, point: str) -> None: """Hold all tasks after the specified cycle point.""" @@ -1078,18 +1088,18 @@ def command_pause(self) -> None: self.pause_workflow() @staticmethod - def command_set_verbosity(lvl: Union[int, str]) -> None: + def command_set_verbosity(level: Union[int, str]) -> None: """Set workflow verbosity.""" try: - lvl = int(lvl) + lvl = int(level) LOG.setLevel(lvl) except (TypeError, ValueError) as exc: raise CommandFailedError(exc) cylc.flow.flags.verbosity = log_level_to_verbosity(lvl) - def command_remove_tasks(self, items) -> int: + def command_remove_tasks(self, tasks: Iterable[str]) -> int: """Remove tasks.""" - return self.pool.remove_tasks(items) + return self.pool.remove_tasks(tasks) async def command_reload_workflow(self) -> None: """Reload workflow configuration.""" @@ -1704,6 +1714,7 @@ async def main_loop(self) -> None: tinit = time() # Useful for debugging core scheduler issues: + # import logging # self.pool.log_task_pool(logging.CRITICAL) if self.incomplete_ri_map: self.manage_remote_init() @@ -1740,7 +1751,7 @@ async def main_loop(self) -> None: if self.xtrigger_mgr.do_housekeeping: self.xtrigger_mgr.housekeep(self.pool.get_tasks()) - self.pool.set_expired_tasks() + self.pool.clock_expire_tasks() self.release_queued_tasks() if ( @@ -2069,9 +2080,11 @@ def check_auto_shutdown(self): if itask.state( TASK_STATUS_PREPARING, TASK_STATUS_SUBMITTED, - TASK_STATUS_RUNNING - ) - or ( + TASK_STATUS_RUNNING, + ) or ( + # This is because runahead limit gets truncated + # to stop_point if there is one, so tasks spawned + # beyond the stop_point must be runahead limited. itask.state(TASK_STATUS_WAITING) and not itask.state.is_runahead ) @@ -2098,7 +2111,7 @@ def pause_workflow(self, msg: Optional[str] = None) -> None: if self.is_paused: LOG.info("Workflow is already paused") return - _msg = "PAUSING the workflow now" + _msg = "Pausing the workflow" if msg: _msg += f': {msg}' LOG.info(_msg) @@ -2128,18 +2141,38 @@ def resume_workflow(self, quiet: bool = False) -> None: self.workflow_db_mgr.put_workflow_paused(False) self.update_data_store() - def command_force_trigger_tasks(self, items, flow, flow_wait, flow_descr): + def command_force_trigger_tasks( + self, + tasks: Iterable[str], + flow: List[str], + flow_wait: bool = False, + flow_descr: Optional[str] = None + ): """Manual task trigger.""" return self.pool.force_trigger_tasks( - items, flow, flow_wait, flow_descr) + tasks, flow, flow_wait, flow_descr) - def command_force_spawn_children(self, items, outputs, flow_num): + def command_set( + self, + tasks: List[str], + flow: List[str], + outputs: Optional[List[str]] = None, + prerequisites: Optional[List[str]] = None, + flow_wait: bool = False, + flow_descr: Optional[str] = None + ): """Force spawn task successors. - User-facing method name: set_outputs. - + Note, the "outputs" and "prerequisites" arguments might not be + populated in the mutation arguments so must provide defaults here. """ - return self.pool.force_spawn_children(items, outputs, flow_num) + if outputs is None: + outputs = [] + if prerequisites is None: + prerequisites = [] + return self.pool.set_prereqs_and_outputs( + tasks, outputs, prerequisites, flow, flow_wait, flow_descr + ) def _update_profile_info(self, category, amount, amount_format="%s"): """Update the 1, 5, 15 minute dt averages for a given category.""" diff --git a/cylc/flow/scripts/completion_server.py b/cylc/flow/scripts/completion_server.py index 1677adb0d1e..da1ef14556a 100644 --- a/cylc/flow/scripts/completion_server.py +++ b/cylc/flow/scripts/completion_server.py @@ -40,6 +40,8 @@ # Which provide possible values to the completion functions. import asyncio +from contextlib import suppress +import inspect import os from pathlib import Path import select @@ -50,6 +52,7 @@ from packaging.specifiers import SpecifierSet from cylc.flow.cfgspec.glbl_cfg import glbl_cfg +from cylc.flow.exceptions import CylcError from cylc.flow.id import tokenise, IDTokens, Tokens from cylc.flow.network.scan import scan from cylc.flow.option_parsers import CylcOptionParser as COP @@ -193,7 +196,12 @@ async def complete_cylc(_root: str, *items: str) -> t.List[str]: if ret is not None: return ret if previous and previous.startswith('-'): - ret = await complete_option_value(command, previous, partial) + ret = await complete_option_value( + command, + previous, + partial, + items=items, + ) if ret is not None: return ret @@ -256,10 +264,11 @@ async def complete_option( async def complete_option_value( command: str, option: str, - partial: t.Optional[str] = None + partial: t.Optional[str] = None, + items: t.Optional[t.Iterable[str]] = None, ) -> t.Optional[t.List[str]]: """Complete values for --options.""" - vals = await list_option_values(command, option, partial) + vals = await list_option_values(command, option, partial, items=items) if vals is not None: return complete(partial, vals) return None @@ -331,9 +340,21 @@ async def list_option_values( command: str, option: str, partial: t.Optional[str] = '', + items: t.Optional[t.Iterable[str]] = None, ) -> t.Optional[t.List[str]]: """List values for an option in a Cylc command. + Args: + command: + The Cylc sub-command. + option: + The --option to list possible values for. + partial: + The part of the command the user is completing. + items: + The CLI context, i.e. everything that has been typed on the CLI + before the partial. + E.G. --flow ['all', 'new', 'none'] """ if option in OPTION_MAP: @@ -341,7 +362,22 @@ async def list_option_values( if not list_option: # do not perform completion for this option return [] - return await list_option(None, partial) + kwargs = {} + if 'tokens_list' in inspect.getfullargspec(list_option).args: + # the function requires information about tokens already specified + # on the CLI + # (e.g. the workflow//cycle/task the command is operating on) + tokens_list = [] + for item in items or []: + # pull out things from the command which look like IDs + if '//' in item: + with suppress(ValueError): + tokens_list.append(Tokens(item)) + continue + with suppress(ValueError): + tokens_list.append(Tokens(item, relative=True)) + kwargs['tokens_list'] = tokens_list + return await list_option(partial, **kwargs) return None @@ -412,7 +448,6 @@ async def list_resources(_partial: str) -> t.List[str]: async def list_dir( - _workflow: t.Optional[str], partial: t.Optional[str] ) -> t.List[str]: """List an arbitrary dir on the filesystem. @@ -459,7 +494,6 @@ def list_rel_dir(path: Path, base: Path) -> t.List[str]: async def list_flows( - _workflow: t.Optional[str], _partial: t.Optional[str] ) -> t.List[str]: """List values for the --flow option.""" @@ -467,13 +501,96 @@ async def list_flows( async def list_colours( - _workflow: t.Optional[str], _partial: t.Optional[str] ) -> t.List[str]: """List values for the --color option.""" return ['never', 'auto', 'always'] +async def list_outputs( + _partial: t.Optional[str], + tokens_list: t.Optional[t.List[Tokens]], +): + """List task outputs.""" + return (await _list_prereqs_and_outputs(tokens_list))[1] + + +async def list_prereqs( + _partial: t.Optional[str], + tokens_list: t.Optional[t.List[Tokens]], +): + """List task prerequisites.""" + return (await _list_prereqs_and_outputs(tokens_list))[0] + ['all'] + + +async def _list_prereqs_and_outputs( + tokens_list: t.Optional[t.List[Tokens]], +) -> t.Tuple[t.List[str], t.List[str]]: + """List task prerequisites and outputs. + + Returns: + tuple - (prereqs, outputs) + + """ + if not tokens_list: + # no context information available on the CLI + # we can't list prereqs/outputs + return ([], []) + + # dynamic import for this relatively unlikely case to avoid slowing down + # server startup unnecessarily + from cylc.flow.network.client_factory import get_client + from cylc.flow.scripts.show import prereqs_and_outputs_query + from types import SimpleNamespace + + workflows: t.Dict[str, t.List[Tokens]] = {} + current_workflow = None + for tokens in tokens_list: + workflow = tokens['workflow'] + task = tokens['task'] + if workflow: + workflows.setdefault(workflow, []) + current_workflow = workflow + if current_workflow and task: + workflows[current_workflow].append(tokens.task) + + clients = {} + for workflow in workflows: + with suppress(CylcError): + clients[workflow] = get_client(workflow) + + if not workflows: + return ([], []) + + json: dict = {} + await asyncio.gather(*( + prereqs_and_outputs_query( + workflow, + workflows[workflow], + pclient, + SimpleNamespace(json=True), + json, + ) + for workflow, pclient in clients.items() + )) + + if not json: + return ([], []) + return ( + [ + f"{cond['taskId']}:{cond['reqState']}" + for value in json.values() + for prerequisite in value['prerequisites'] + for cond in prerequisite['conditions'] + ], + [ + output['label'] + for value in json.values() + for output in value['outputs'] + ], + ) + + # non-exhaustive list of Cylc commands which take non-workflow arguments COMMAND_MAP: t.Dict[str, t.Optional[t.Callable]] = { # register commands which have special positional arguments @@ -512,6 +629,8 @@ async def list_colours( '--flow': list_flows, '--colour': list_colours, '--color': list_colours, + '--out': list_outputs, + '--pre': list_prereqs, # options for which we should not attempt to complete values for '--rm': None, '--run-name': None, @@ -527,7 +646,7 @@ async def list_colours( } -def cli_detokenise(tokens: Tokens) -> str: +def cli_detokenise(tokens: Tokens, relative=False) -> str: """Format tokens for use on the command line. I.E. add the trailing slash[es] onto the end. @@ -535,9 +654,13 @@ def cli_detokenise(tokens: Tokens) -> str: if tokens.is_null: # shouldn't happen but prevents possible error return '' + if relative: + id_ = tokens.relative_id + else: + id_ = tokens.id if tokens.lowest_token == IDTokens.Workflow.value: - return f'{tokens.id}//' - return f'{tokens.id}/' + return f'{id_}//' + return f'{id_}/' def next_token(tokens: Tokens) -> t.Optional[str]: diff --git a/cylc/flow/scripts/cylc.py b/cylc/flow/scripts/cylc.py index 700a70f8d7b..cabcec553e9 100644 --- a/cylc/flow/scripts/cylc.py +++ b/cylc/flow/scripts/cylc.py @@ -293,7 +293,9 @@ def get_version(long=False): 'register': 'cylc register has been removed; use cylc install or cylc play', 'reset': - 'cylc reset has been replaced by cylc set-outputs', + 'cylc reset has been replaced by cylc set', + 'set-outputs': + 'cylc set-outputs (cylc 8.0-8.2) has been replaced by cylc set', 'restart': 'cylc run & cylc restart have been replaced by cylc play', 'review': @@ -311,6 +313,8 @@ def get_version(long=False): 'cylc submit has been removed', 'start': 'cylc start & cylc restart have been replaced by cylc play', + 'set-verbosity': + 'cylc set-verbosity has been replaced by cylc verbosity', 'warranty': 'cylc warranty has been replaced by cylc help license', } diff --git a/cylc/flow/scripts/dump.py b/cylc/flow/scripts/dump.py index d3ffda59fce..f891c943d77 100755 --- a/cylc/flow/scripts/dump.py +++ b/cylc/flow/scripts/dump.py @@ -18,23 +18,22 @@ """cylc dump [OPTIONS] ARGS -Print information about a running workflow. +Print state of a running workflow including tasks in the active (n=0) window. -This command can provide information about active tasks, e.g. running or queued -tasks. For more detailed view of the workflow see `cylc tui` or `cylc gui`. - -For command line monitoring: -* `cylc tui` -* `watch cylc dump WORKFLOW_ID` works for small simple workflows +For a more detailed view of the workflow see `cylc tui` or `cylc gui`. For more information about a specific task, such as the current state of its prerequisites and outputs, see 'cylc show'. +For command line monitoring: +* `cylc tui` +* `watch cylc dump -t WORKFLOW_ID` works for small simple workflows + Examples: # Display the state of all active tasks, sorted by cycle point: $ cylc dump --tasks --sort WORKFLOW_ID | grep running - # Display the state of all active in a particular cycle point: + # Display the state of all active tasks in a particular cycle point: $ cylc dump -t WORKFLOW_ID | grep 2010082406 """ diff --git a/cylc/flow/scripts/message.py b/cylc/flow/scripts/message.py index 8e16eebe776..0727d51f387 100755 --- a/cylc/flow/scripts/message.py +++ b/cylc/flow/scripts/message.py @@ -17,16 +17,26 @@ # along with this program. If not, see . r"""cylc message [OPTIONS] -- ARGS -Record task messages. - -Send messages to: -- The job stdout/stderr. -- The job status file, if there is one. -- The scheduler, if communication is possible. - -Jobs use this command to record and report status such as success and -failure. Applications run by jobs can use this command to report messages -and to report registered task outputs. +Command to record and send messages from task jobs back to the scheduler. + +Messages are sent to: +- job stdout/stderr +- the job status file, if there is one +- the scheduler, if communication is possible + +Jobs automatically use this command to record and report job status: started +(running) and success or failure. + +Applications run by jobs can use this command to report custom messages and +and registered task output messages. + +For custom outputs, use the task message not the associated output name: +[runtime] + [[my-task]] + # ... + [[[outputs]]] + # = + x = "file x completed and archived" Messages can be specified as arguments. A '-' indicates that the command should read messages from STDIN. When reading from STDIN, multiple messages are diff --git a/cylc/flow/scripts/set.py b/cylc/flow/scripts/set.py new file mode 100755 index 00000000000..88e9a177c1b --- /dev/null +++ b/cylc/flow/scripts/set.py @@ -0,0 +1,429 @@ +#!/usr/bin/env python3 + +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""cylc set [OPTIONS] ARGS + +Command to manually set task prerequisites and outputs in running workflows. + +By default, it sets all required outputs (note "succeeded" may be optional). + +Setting task prerequisites: + - contributes to the task's readiness to run, and + - promotes it to the scheduler's active task pool + +Note --pre=all also promotes parentless tasks (with no task-prerequisites) to +the active pool where clock and xtriggers become active. This is needed to +start a new flow that continues to future cycle points, if you need the first +parentless tasks in the new flow to wait on clock or xtriggers before running. + +Setting task outputs: + - contributes to a task's completion, and + - spawns downstream tasks that depend on those outputs + +Note setting final outputs (succeeded, failed, expired) also sets task state. +Setting the started and submitted outputs spawns downstream tasks that depend +on them but does not affect task state, because there is no running job. + +Implied outputs are set automatically: + - started implies submitted + - succeeded and failed imply started + - custom outputs and expired do not imply other outputs + +For custom outputs, use the output names not the associated task messages: +[runtime] + [[my-task]] + # ... + [[[outputs]]] + # = + x = "file x completed and archived" + +CLI Completion: + Cylc can auto-complete prerequisites and outputs for active tasks if you + specify the task in the command before attempting TAB-completion. + +Examples: + # complete all required outputs of 3/bar: + $ cylc set my_workflow//3/bar + # or: + $ cylc set --out=required my_workflow//3/bar + + # complete the succeeded output of 3/bar: + $ cylc set --out=succeeded my_workflow//3/bar + + # satisfy the 3/foo:succeeded prerequisite of 3/bar: + $ cylc set --pre=3/foo my_workflow//3/bar + # or: + $ cylc set --pre=3/foo:succeeded my_workflow//3/bar + + # satisfy all prerequisites (if any) of 3/bar and promote it to + # the active window (and start checking its xtriggers, if any): + $ cylc set --pre=all my_workflow//3/bar + + # complete the "file1" custom output of 3/bar: + $ cylc set --out=file1 my_workflow//3/bar + + # satisfy the "3/bar:file1" prerequisite of 3/qux: + $ cylc set --pre=3/bar:file1 my_workflow//3/qux + + # set multiple outputs at once: + $ cylc set --out=a --out=b,c my_workflow//3/bar + + # set multiple prerequisites at once: + $ cylc set --pre=3/foo:x --pre=3/foo:y,3/foo:z my_workflow//3/bar + +""" + +from functools import partial +from typing import TYPE_CHECKING, List, Optional + +from cylc.flow.exceptions import InputError +from cylc.flow.network.client_factory import get_client +from cylc.flow.network.multi import call_multi +from cylc.flow.option_parsers import ( + FULL_ID_MULTI_ARG_DOC, + CylcOptionParser as COP, +) +from cylc.flow.id import Tokens +from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED +from cylc.flow.terminal import cli_function +from cylc.flow.flow_mgr import ( + add_flow_opts, + validate_flow_opts +) + + +if TYPE_CHECKING: + from optparse import Values + + +MUTATION = ''' +mutation ( + $wFlows: [WorkflowID]!, + $tasks: [NamespaceIDGlob]!, + $prerequisites: [PrerequisiteString], + $outputs: [OutputLabel], + $flow: [Flow!], + $flowWait: Boolean, + $flowDescr: String, +) { + set ( + workflows: $wFlows, + tasks: $tasks, + prerequisites: $prerequisites, + outputs: $outputs, + flow: $flow, + flowWait: $flowWait, + flowDescr: $flowDescr + ) { + result + } +} +''' + + +SELECTOR_ERROR = ( + 'Use "--output={1}" to specify outputs, not "{0}:{1}"' +) + + +def get_option_parser() -> COP: + parser = COP( + __doc__, + comms=True, + multitask=True, + multiworkflow=True, + argdoc=[FULL_ID_MULTI_ARG_DOC], + ) + + parser.add_option( + "-o", "--out", "--output", metavar="OUTPUT(s)", + help=( + "Complete task outputs. For multiple outputs re-use the" + " option, or give a comma-separated list of outputs, or" + ' use "--out=required" to complete all required outputs.' + " OUTPUT format: trigger names as used in the graph." + ), + action="append", default=None, dest="outputs" + ) + + parser.add_option( + "-p", "--pre", "--prerequisite", metavar="PREREQUISITE(s)", + help=( + "Satisfy task prerequisites. For multiple prerequisites" + " re-use the option, or give a comma-separated list, or" + ' use "--pre=all" to satisfy all prerequisites, if any.' + " PREREQUISITE format: 'cycle/task[:OUTPUT]', where" + " :OUTPUT defaults to :succeeded." + ), + action="append", default=None, dest="prerequisites" + ) + + add_flow_opts(parser) + return parser + + +def validate_prereq(prereq: str) -> Optional[str]: + """Return prereq (with :succeeded) if valid, else None. + + Format: cycle/task[:output] + + Examples: + >>> validate_prereq('1/foo:succeeded') + '1/foo:succeeded' + + >>> validate_prereq('1/foo') + '1/foo:succeeded' + + >>> validate_prereq('all') + 'all' + + # Error: + >>> validate_prereq('fish') + + """ + try: + tokens = Tokens(prereq, relative=True) + except ValueError: + return None + if ( + tokens["cycle"] == prereq + and prereq != "all" + ): + # Error: --pre= other than "all" + return None + + if prereq != "all" and tokens["task_sel"] is None: + prereq += f":{TASK_OUTPUT_SUCCEEDED}" + + return prereq + + +def split_opts(options: List[str]): + """Return list from multi-use and comma-separated options. + + Examples: + # --out='a,b,c' + >>> split_opts(['a,b,c']) + ['a', 'b', 'c'] + + # --out='a' --out='a,b' + >>> split_opts(['a', 'b,c']) + ['a', 'b', 'c'] + + # --out='a' --out='a,b' + >>> split_opts(['a', 'a,b']) + ['a', 'b'] + + # --out=' a ' + >>> split_opts([' a ']) + ['a'] + + # --out='a, b, c , d' + >>> split_opts(['a, b, c , d']) + ['a', 'b', 'c', 'd'] + + """ + return sorted({ + item.strip() + for option in (options or []) + for item in option.strip().split(',') + }) + + +def get_prereq_opts(prereq_options: List[str]): + """Convert prerequisites to a flat list with output selectors. + + Examples: + # Set multiple at once: + >>> get_prereq_opts(['1/foo:bar', '2/foo:baz,3/foo:qux']) + ['1/foo:bar', '2/foo:baz', '3/foo:qux'] + + # --pre=all + >>> get_prereq_opts(["all"]) + ['all'] + + # implicit ":succeeded" + >>> get_prereq_opts(["1/foo"]) + ['1/foo:succeeded'] + + # Error: invalid format: + >>> get_prereq_opts(["fish"]) + Traceback (most recent call last): + ... + InputError: + + # Error: invalid format: + >>> get_prereq_opts(["1/foo::bar"]) + Traceback (most recent call last): + ... + InputError: + + # Error: "all" must be used alone: + >>> get_prereq_opts(["all", "2/foo:baz"]) + Traceback (most recent call last): + ... + InputError: + + """ + prereqs = split_opts(prereq_options) + if not prereqs: + return [] + + prereqs2 = [] + bad: List[str] = [] + for pre in prereqs: + p = validate_prereq(pre) + if p is not None: + prereqs2.append(p) + else: + bad.append(pre) + if bad: + raise InputError( + "Use prerequisite format /:output\n" + "\n ".join(bad) + ) + + if len(prereqs2) > 1: # noqa SIM102 (anticipates "cylc set --pre=cycle") + if "all" in prereqs: + raise InputError("--pre=all must be used alone") + + return prereqs2 + + +def get_output_opts(output_options: List[str]): + """Convert outputs options to a flat list, and validate. + + Examples: + Good: + >>> get_output_opts(['a', 'b,c']) + ['a', 'b', 'c'] + >>> get_output_opts(["required"]) # "required" is explicit default + [] + + Bad: + >>> get_output_opts(["required", "a"]) # "required" must be used alone + Traceback (most recent call last): + InputError: + # Error: "required" must be used alone + >>> get_output_opts(["waiting"]) # cannot "reset" to waiting + Traceback (most recent call last): + InputError: + + """ + outputs = split_opts(output_options) + + # If "required" is explicit just ditch it (same as the default) + if not outputs or outputs == ["required"]: + return [] + + if "required" in outputs: + raise InputError("--out=required must be used alone") + if "waiting" in outputs: + raise InputError( + "Tasks can not be set to waiting, use a new flow to re-run" + ) + + return outputs + + +def validate_opts(output_opt: List[str], prereq_opt: List[str]): + """Check global option consistency + + Examples: + >>> validate_opts(["a"], None) # OK + + >>> validate_opts(None, ["1/a:failed"]) #OK + + >>> validate_opts(["a"], ["1/a:failed"]) + Traceback (most recent call last): + ... + InputError: + + """ + if output_opt and prereq_opt: + raise InputError("Use --prerequisite or --output, not both.") + + +def validate_tokens(tokens_list): + """Check the cycles/tasks provided. + + This checks that cycle/task selectors have not been provided in the IDs. + + Examples: + Good: + >>> validate_tokens([Tokens('w//c')]) + >>> validate_tokens([Tokens('w//c/t')]) + + Bad: + >>> validate_tokens([Tokens('w//c:s')]) + Traceback (most recent call last): + cylc.flow.exceptions.InputError + >>> validate_tokens([Tokens('w//c/t:s')]) + Traceback (most recent call last): + cylc.flow.exceptions.InputError + + """ + for tokens in tokens_list: + if tokens['cycle_sel']: + raise InputError(SELECTOR_ERROR.format( + tokens['cycle'], + tokens['cycle_sel'], + )) + if tokens['task_sel']: + raise InputError(SELECTOR_ERROR.format( + tokens['task'], + tokens['task_sel'], + )) + + +async def run( + options: 'Values', + workflow_id: str, + *tokens_list +) -> None: + validate_tokens(tokens_list) + + pclient = get_client(workflow_id, timeout=options.comms_timeout) + + mutation_kwargs = { + 'request_string': MUTATION, + 'variables': { + 'wFlows': [workflow_id], + 'tasks': [ + tokens.relative_id_with_selectors + for tokens in tokens_list + ], + 'outputs': get_output_opts(options.outputs), + 'prerequisites': get_prereq_opts(options.prerequisites), + 'flow': options.flow, + 'flowWait': options.flow_wait, + 'flowDescr': options.flow_descr + } + } + + await pclient.async_request('graphql', mutation_kwargs) + + +@cli_function(get_option_parser) +def main(parser: COP, options: 'Values', *ids) -> None: + validate_opts(options.outputs, options.prerequisites) + validate_flow_opts(options) + call_multi( + partial(run, options), + *ids, + ) diff --git a/cylc/flow/scripts/set_outputs.py b/cylc/flow/scripts/set_outputs.py deleted file mode 100755 index b45ddbff268..00000000000 --- a/cylc/flow/scripts/set_outputs.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 - -# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -"""cylc set-outputs [OPTIONS] ARGS - -Artificially satisfy task outputs. - -Mark task outputs as completed and spawn downstream tasks that depend on those -outputs. By default it marks tasks as succeeded. - -This allows you to manually intervene with Cylc's scheduling algorithm by -artificially satisfying outputs of tasks. - -If a flow number is given, the child tasks will start (or continue) that flow. - -Examples: - # For example, for the following dependency graph: - R1 = ''' - a => b & c => d - foo:x => bar => baz - ''' - - # spawn 1/b and 1/c, but 1/d will not subsequently run - $ cylc set-outputs my_workflow//1/a - - # spawn 1/b and 1/c as flow 2, followed by 1/d - $ cylc set-outputs --flow=2 my_workflow//1/a - - # spawn 1/bar as flow 3, followed by 1/baz - $ cylc set-outputs --flow=3 --output=x my_workflow//1/foo - -Use --output multiple times to spawn off of several outputs at once. -""" - -from functools import partial -from typing import TYPE_CHECKING - -from cylc.flow.network.client_factory import get_client -from cylc.flow.network.multi import call_multi -from cylc.flow.option_parsers import ( - FULL_ID_MULTI_ARG_DOC, - CylcOptionParser as COP, -) -from cylc.flow.terminal import cli_function - -if TYPE_CHECKING: - from optparse import Values - -MUTATION = ''' -mutation ( - $wFlows: [WorkflowID]!, - $tasks: [NamespaceIDGlob]!, - $outputs: [String], - $flowNum: Int, -) { - setOutputs ( - workflows: $wFlows, - tasks: $tasks, - outputs: $outputs, - flowNum: $flowNum, - ) { - result - } -} -''' - - -def get_option_parser() -> COP: - parser = COP( - __doc__, - comms=True, - multitask=True, - multiworkflow=True, - argdoc=[FULL_ID_MULTI_ARG_DOC], - ) - - parser.add_option( - "-o", "--output", metavar="OUTPUT", - help="Set OUTPUT (default \"succeeded\") completed.", - action="append", default=None, dest="outputs") - - parser.add_option( - "-f", "--flow", metavar="FLOW", - help="Number of the flow to attribute the outputs.", - action="store", default=None, dest="flow_num") - - return parser - - -async def run(options: 'Values', workflow_id: str, *tokens_list) -> None: - pclient = get_client(workflow_id, timeout=options.comms_timeout) - - mutation_kwargs = { - 'request_string': MUTATION, - 'variables': { - 'wFlows': [workflow_id], - 'tasks': [ - tokens.relative_id_with_selectors - for tokens in tokens_list - ], - 'outputs': options.outputs, - 'flowNum': options.flow_num - } - } - - await pclient.async_request('graphql', mutation_kwargs) - - -@cli_function(get_option_parser) -def main(parser: COP, options: 'Values', *ids) -> None: - call_multi( - partial(run, options), - *ids, - ) diff --git a/cylc/flow/scripts/trigger.py b/cylc/flow/scripts/trigger.py index 3e4a3da96f7..9d27fc38bd1 100755 --- a/cylc/flow/scripts/trigger.py +++ b/cylc/flow/scripts/trigger.py @@ -44,7 +44,6 @@ from functools import partial from typing import TYPE_CHECKING -from cylc.flow.exceptions import InputError from cylc.flow.network.client_factory import get_client from cylc.flow.network.multi import call_multi from cylc.flow.option_parsers import ( @@ -52,20 +51,16 @@ CylcOptionParser as COP, ) from cylc.flow.terminal import cli_function -from cylc.flow.flow_mgr import FLOW_NONE, FLOW_NEW, FLOW_ALL +from cylc.flow.flow_mgr import ( + add_flow_opts, + validate_flow_opts +) + if TYPE_CHECKING: from optparse import Values -ERR_OPT_FLOW_VAL = "Flow values must be integer, 'all', 'new', or 'none'" -ERR_OPT_FLOW_INT = "Multiple flow options must all be integer valued" -ERR_OPT_FLOW_META = "Metadata is only for new flows" -ERR_OPT_FLOW_WAIT = ( - f"--wait is not compatible with --flow={FLOW_NEW} or --flow={FLOW_NONE}" -) - - MUTATION = ''' mutation ( $wFlows: [WorkflowID]!, @@ -95,49 +90,10 @@ def get_option_parser() -> COP: multiworkflow=True, argdoc=[FULL_ID_MULTI_ARG_DOC], ) - - parser.add_option( - "--flow", action="append", dest="flow", metavar="FLOW", - help=f"Assign the triggered task to all active flows ({FLOW_ALL});" - f" no flow ({FLOW_NONE}); a new flow ({FLOW_NEW});" - f" or a specific flow (e.g. 2). The default is {FLOW_ALL}." - " Reuse the option to assign multiple specific flows." - ) - - parser.add_option( - "--meta", metavar="DESCRIPTION", action="store", - dest="flow_descr", default=None, - help=f"description of triggered flow (with --flow={FLOW_NEW})." - ) - - parser.add_option( - "--wait", action="store_true", default=False, dest="flow_wait", - help="Wait for merge with current active flows before flowing on." - ) - + add_flow_opts(parser) return parser -def _validate(options): - """Check validity of flow-related options.""" - for val in options.flow: - val = val.strip() - if val in [FLOW_NONE, FLOW_NEW, FLOW_ALL]: - if len(options.flow) != 1: - raise InputError(ERR_OPT_FLOW_INT) - else: - try: - int(val) - except ValueError: - raise InputError(ERR_OPT_FLOW_VAL.format(val)) - - if options.flow_descr and options.flow != [FLOW_NEW]: - raise InputError(ERR_OPT_FLOW_META) - - if options.flow_wait and options.flow[0] in [FLOW_NEW, FLOW_NONE]: - raise InputError(ERR_OPT_FLOW_WAIT) - - async def run(options: 'Values', workflow_id: str, *tokens_list): pclient = get_client(workflow_id, timeout=options.comms_timeout) @@ -154,18 +110,13 @@ async def run(options: 'Values', workflow_id: str, *tokens_list): 'flowDescr': options.flow_descr, } } - await pclient.async_request('graphql', mutation_kwargs) @cli_function(get_option_parser) def main(parser: COP, options: 'Values', *ids: str): """CLI for "cylc trigger".""" - - if options.flow is None: - options.flow = [FLOW_ALL] # default to all active flows - _validate(options) - + validate_flow_opts(options) call_multi( partial(run, options), *ids, diff --git a/cylc/flow/scripts/set_verbosity.py b/cylc/flow/scripts/verbosity.py similarity index 90% rename from cylc/flow/scripts/set_verbosity.py rename to cylc/flow/scripts/verbosity.py index aa40d01345c..0479625c951 100755 --- a/cylc/flow/scripts/set_verbosity.py +++ b/cylc/flow/scripts/verbosity.py @@ -16,13 +16,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -"""cylc set-verbosity [OPTIONS] ARGS +"""cylc verbosity [OPTIONS] ARGS -Change the logging severity level of a running workflow. +Set the logging severity level of a running scheduler. -Only messages at or above the chosen severity level will be logged; for -example, if you choose WARNING, only warnings and critical messages will be -logged. +Messages at or above the chosen level are logged. If you choose +WARNING (say) only WARNING and CRITICAL messages will be logged. """ from functools import partial diff --git a/cylc/flow/task_events_mgr.py b/cylc/flow/task_events_mgr.py index 04cbac361af..a7f16e04498 100644 --- a/cylc/flow/task_events_mgr.py +++ b/cylc/flow/task_events_mgr.py @@ -76,12 +76,18 @@ TASK_STATUS_SUBMIT_FAILED, TASK_STATUS_RUNNING, TASK_STATUS_FAILED, + TASK_STATUS_EXPIRED, TASK_STATUS_SUCCEEDED, TASK_STATUS_WAITING ) from cylc.flow.task_outputs import ( - TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, - TASK_OUTPUT_FAILED, TASK_OUTPUT_SUBMIT_FAILED) + TASK_OUTPUT_EXPIRED, + TASK_OUTPUT_SUBMITTED, + TASK_OUTPUT_STARTED, + TASK_OUTPUT_SUCCEEDED, + TASK_OUTPUT_FAILED, + TASK_OUTPUT_SUBMIT_FAILED +) from cylc.flow.wallclock import ( get_current_time_string, get_seconds_as_interval_string as intvl_as_str @@ -186,11 +192,11 @@ def log_task_job_activity(ctx, workflow, point, name, submit_num=None): try: with open(os.path.expandvars(job_activity_log), "ab") as handle: handle.write((ctx_str + '\n').encode()) - except IOError as exc: - # This happens when there is no job directory, e.g. if job host - # selection command causes an submission failure, there will be no job - # directory. In this case, just send the information to the log. - LOG.exception(exc) + except IOError: + # This happens when there is no job directory. E.g., if a job host + # selection command causes a submission failure, or if a waiting task + # expires before a job log directory is otherwise needed. + # (Don't log the exception content, it looks like a bug). LOG.info(ctx_str) if ctx.cmd and ctx.ret_code: LOG.error(ctx_str) @@ -407,6 +413,7 @@ class TaskEventsManager(): EVENT_RETRY = "retry" EVENT_STARTED = TASK_OUTPUT_STARTED EVENT_SUBMITTED = TASK_OUTPUT_SUBMITTED + EVENT_EXPIRED = TASK_OUTPUT_EXPIRED EVENT_SUBMIT_FAILED = "submission failed" EVENT_SUBMIT_RETRY = "submission retry" EVENT_SUCCEEDED = TASK_OUTPUT_SUCCEEDED @@ -609,6 +616,7 @@ def process_message( event_time: Optional[str] = None, flag: str = FLAG_INTERNAL, submit_num: Optional[int] = None, + forced: bool = False ) -> Optional[bool]: """Parse a task message and update task state. @@ -646,13 +654,14 @@ def process_message( submit_num: The submit number of the task relevant for the message. If not specified, use latest submit number. + forced: + If this message is due to manual completion or not (cylc set) Return: None: in normal circumstances. True: if polling is required to confirm a reversal of status. """ - # Log messages if event_time is None: event_time = get_current_time_string() @@ -666,7 +675,8 @@ def process_message( self.reset_inactivity_timer_func() if not self._process_message_check( - itask, severity, message, event_time, flag, submit_num): + itask, severity, message, event_time, flag, submit_num, forced + ): return None # always update the workflow state summary for latest message @@ -682,75 +692,80 @@ def process_message( # Satisfy my output, if possible, and spawn children. # (first remove signal: failed/EXIT -> failed) + # Complete the corresponding task output, if there is one. msg0 = message.split('/')[0] - completed_trigger = itask.state.outputs.set_msg_trg_completion( - message=msg0, is_completed=True) - self.data_store_mgr.delta_task_output(itask, msg0) - - # Check the `started` event has not been missed e.g. due to - # polling delay - if (message not in [self.EVENT_SUBMITTED, self.EVENT_SUBMIT_FAILED, - self.EVENT_STARTED] - and not itask.state.outputs.is_completed(TASK_OUTPUT_STARTED)): - self.setup_event_handlers( - itask, self.EVENT_STARTED, f'job {self.EVENT_STARTED}') - self.spawn_func(itask, TASK_OUTPUT_STARTED) + if message.startswith(ABORT_MESSAGE_PREFIX): + msg0 = TASK_OUTPUT_FAILED + + completed_output = None + if msg0 not in [TASK_OUTPUT_SUBMIT_FAILED, TASK_OUTPUT_FAILED]: + completed_output = itask.state.outputs.set_msg_trg_completion( + message=msg0, is_completed=True) + if completed_output: + self.data_store_mgr.delta_task_output(itask, msg0) + + for implied in ( + itask.state.outputs.get_incomplete_implied(msg0) + ): + # Set submitted and/or started first, if skipped. + # (whether by forced set, or missed message). + LOG.info(f"[{itask}] setting implied output: {implied}") + self.process_message( + itask, INFO, implied, event_time, + self.FLAG_INTERNAL, submit_num, forced + ) + if message == self.EVENT_STARTED: if ( flag == self.FLAG_RECEIVED and itask.state.is_gt(TASK_STATUS_RUNNING) ): + # Already running. return True - if itask.state.status == TASK_STATUS_PREPARING: - # The started message must have arrived before the submitted - # one, so assume that a successful submission occurred and act - # accordingly. Note the submitted message is internal, whereas - # the started message comes in on the network. - self._process_message_submitted(itask, event_time) - self.spawn_func(itask, TASK_OUTPUT_SUBMITTED) - - self._process_message_started(itask, event_time) - self.spawn_func(itask, TASK_OUTPUT_STARTED) + self._process_message_started(itask, event_time, forced) + self.spawn_children(itask, TASK_OUTPUT_STARTED) elif message == self.EVENT_SUCCEEDED: - self._process_message_succeeded(itask, event_time) - self.spawn_func(itask, TASK_OUTPUT_SUCCEEDED) + self._process_message_succeeded(itask, event_time, forced) + self.spawn_children(itask, TASK_OUTPUT_SUCCEEDED) + + elif message == self.EVENT_EXPIRED: + self._process_message_expired(itask, event_time, forced) + self.spawn_children(itask, TASK_OUTPUT_EXPIRED) + elif message == self.EVENT_FAILED: if ( flag == self.FLAG_RECEIVED and itask.state.is_gt(TASK_STATUS_FAILED) ): + # Already failed. return True if self._process_message_failed( - itask, event_time, self.JOB_FAILED): - self.spawn_func(itask, TASK_OUTPUT_FAILED) + itask, event_time, self.JOB_FAILED, forced + ): + self.spawn_children(itask, TASK_OUTPUT_FAILED) + elif message == self.EVENT_SUBMIT_FAILED: if ( flag == self.FLAG_RECEIVED and itask.state.is_gt(TASK_STATUS_SUBMIT_FAILED) ): + # Already submit-failed return True if self._process_message_submit_failed( - itask, - event_time, - submit_num + itask, event_time, submit_num, forced ): - self.spawn_func(itask, TASK_OUTPUT_SUBMIT_FAILED) + self.spawn_children(itask, TASK_OUTPUT_SUBMIT_FAILED) + elif message == self.EVENT_SUBMITTED: if ( flag == self.FLAG_RECEIVED - and itask.state.is_gt(TASK_STATUS_SUBMITTED) + and itask.state.is_gte(TASK_STATUS_SUBMITTED) ): + # Already submitted. return True - if ( - itask.state.status == TASK_STATUS_PREPARING - or itask.tdef.run_mode == 'simulation' - ): - # If not in the preparing state we already assumed and handled - # job submission under the started event above... - # (sim mode does not have the job prep state) - self._process_message_submitted(itask, event_time) - self.spawn_func(itask, TASK_OUTPUT_SUBMITTED) + self._process_message_submitted(itask, event_time, forced) + self.spawn_children(itask, TASK_OUTPUT_SUBMITTED) # ... but either way update the job ID in the job proxy (it only # comes in via the submission message). @@ -767,27 +782,34 @@ def process_message( flag == self.FLAG_RECEIVED and itask.state.is_gt(TASK_STATUS_FAILED) ): + # Already failed. return True signal = message[len(FAIL_MESSAGE_PREFIX):] self._db_events_insert(itask, "signaled", signal) self.workflow_db_mgr.put_update_task_jobs( itask, {"run_signal": signal}) if self._process_message_failed( - itask, event_time, self.JOB_FAILED): - self.spawn_func(itask, TASK_OUTPUT_FAILED) + itask, event_time, self.JOB_FAILED, forced + ): + self.spawn_children(itask, TASK_OUTPUT_FAILED) + elif message.startswith(ABORT_MESSAGE_PREFIX): # Task aborted with message if ( flag == self.FLAG_RECEIVED and itask.state.is_gt(TASK_STATUS_FAILED) ): + # Already failed. return True aborted_with = message[len(ABORT_MESSAGE_PREFIX):] self._db_events_insert(itask, "aborted", message) self.workflow_db_mgr.put_update_task_jobs( itask, {"run_signal": aborted_with}) - if self._process_message_failed(itask, event_time, aborted_with): - self.spawn_func(itask, TASK_OUTPUT_FAILED) + if self._process_message_failed( + itask, event_time, aborted_with, forced + ): + self.spawn_children(itask, TASK_OUTPUT_FAILED) + elif message.startswith(VACATION_MESSAGE_PREFIX): # Task job pre-empted into a vacation state self._db_events_insert(itask, "vacated", message) @@ -796,8 +818,8 @@ def process_message( itask.try_timers[TimerFlags.SUBMISSION_RETRY].num = 0 itask.job_vacated = True # Believe this and change state without polling (could poll?). - if itask.state_reset(TASK_STATUS_SUBMITTED): - itask.state_reset(is_queued=False) + if itask.state_reset(TASK_STATUS_SUBMITTED, forced=forced): + itask.state_reset(is_queued=False, forced=forced) self.data_store_mgr.delta_task_state(itask) self.data_store_mgr.delta_task_queued(itask) self._reset_job_timers(itask) @@ -806,11 +828,15 @@ def process_message( # system, we should probably aim to remove support for job vacation # instead. Otherwise, we should have: # self.setup_event_handlers(itask, 'vacated', message) - elif completed_trigger: - # Message of an as-yet unreported custom task output. + + elif completed_output: + # Message of a custom task output. # No state change. - self.setup_event_handlers(itask, completed_trigger, message) - self.spawn_func(itask, msg0) + # Log completion of o (not needed for standard outputs) + LOG.info(f"[{itask}] completed output {completed_output}") + self.setup_event_handlers(itask, completed_output, message) + self.spawn_children(itask, msg0) + else: # Unhandled messages. These include: # * general non-output/progress messages @@ -820,9 +846,11 @@ def process_message( LOG.debug(f"[{itask}] unhandled: {message}") self._db_events_insert( itask, (f"message {lseverity}"), message) + if lseverity in self.NON_UNIQUE_EVENTS: itask.non_unique_events.update({lseverity: 1}) self.setup_event_handlers(itask, lseverity, message) + return None def _process_message_check( @@ -833,13 +861,17 @@ def _process_message_check( event_time: str, flag: str, submit_num: int, + forced: bool = False ) -> bool: """Helper for `.process_message`. See `.process_message` for argument list Check whether to process/skip message. - Return True if `.process_message` should contine, False otherwise. + Return True if `.process_message` should continue, False otherwise. """ + if itask.transient or forced: + return True + if self.timestamp: timestamp = f" at {event_time}" else: @@ -1253,10 +1285,11 @@ def _retry_task(self, itask, wallclock_time, submit_retry=False): os.getenv("CYLC_WORKFLOW_RUN_DIR") ) itask.state.add_xtrigger(label) + if itask.state_reset(TASK_STATUS_WAITING): self.data_store_mgr.delta_task_state(itask) - def _process_message_failed(self, itask, event_time, message): + def _process_message_failed(self, itask, event_time, message, forced): """Helper for process_message, handle a failed message. Return True if no retries (hence go to the failed state). @@ -1273,14 +1306,20 @@ def _process_message_failed(self, itask, event_time, message): "time_run_exit": event_time, }) if ( - TimerFlags.EXECUTION_RETRY not in itask.try_timers + forced + or TimerFlags.EXECUTION_RETRY not in itask.try_timers or itask.try_timers[TimerFlags.EXECUTION_RETRY].next() is None ): # No retry lined up: definitive failure. - if itask.state_reset(TASK_STATUS_FAILED): + no_retries = True + if itask.state_reset(TASK_STATUS_FAILED, forced=forced): self.setup_event_handlers(itask, self.EVENT_FAILED, message) self.data_store_mgr.delta_task_state(itask) - no_retries = True + itask.state.outputs.set_msg_trg_completion( + message=TASK_OUTPUT_FAILED, is_completed=True) + self.data_store_mgr.delta_task_output( + itask, TASK_OUTPUT_FAILED) + self.data_store_mgr.delta_task_state(itask) else: # There is an execution retry lined up. timer = itask.try_timers[TimerFlags.EXECUTION_RETRY] @@ -1292,7 +1331,7 @@ def _process_message_failed(self, itask, event_time, message): self._reset_job_timers(itask) return no_retries - def _process_message_started(self, itask, event_time): + def _process_message_started(self, itask, event_time, forced): """Helper for process_message, handle a started message.""" if itask.job_vacated: itask.job_vacated = False @@ -1303,7 +1342,7 @@ def _process_message_started(self, itask, event_time): itask.set_summary_time('started', event_time) self.workflow_db_mgr.put_update_task_jobs(itask, { "time_run": itask.summary['started_time_string']}) - if itask.state_reset(TASK_STATUS_RUNNING): + if itask.state_reset(TASK_STATUS_RUNNING, forced=forced): self.setup_event_handlers( itask, self.EVENT_STARTED, f'job {self.EVENT_STARTED}') self.data_store_mgr.delta_task_state(itask) @@ -1313,8 +1352,23 @@ def _process_message_started(self, itask, event_time): if TimerFlags.SUBMISSION_RETRY in itask.try_timers: itask.try_timers[TimerFlags.SUBMISSION_RETRY].num = 0 - def _process_message_succeeded(self, itask, event_time): - """Helper for process_message, handle a succeeded message.""" + def _process_message_expired(self, itask, event_time, forced): + """Helper for process_message, handle task expiry.""" + if not itask.state_reset(TASK_STATUS_EXPIRED, forced=forced): + return + self.data_store_mgr.delta_task_state(itask) + self.data_store_mgr.delta_task_queued(itask) + self.setup_event_handlers( + itask, + self.EVENT_EXPIRED, + "Task expired: will not submit job." + ) + + def _process_message_succeeded(self, itask, event_time, forced): + """Helper for process_message, handle a succeeded message. + + Ignore forced. + """ job_tokens = itask.tokens.duplicate(job=str(itask.submit_num)) self.data_store_mgr.delta_job_time(job_tokens, 'finished', event_time) @@ -1329,13 +1383,15 @@ def _process_message_succeeded(self, itask, event_time): itask.tdef.elapsed_times.append( itask.summary['finished_time'] - itask.summary['started_time']) - if itask.state_reset(TASK_STATUS_SUCCEEDED): + if itask.state_reset(TASK_STATUS_SUCCEEDED, forced=forced): self.setup_event_handlers( itask, self.EVENT_SUCCEEDED, f"job {self.EVENT_SUCCEEDED}") self.data_store_mgr.delta_task_state(itask) self._reset_job_timers(itask) - def _process_message_submit_failed(self, itask, event_time, submit_num): + def _process_message_submit_failed( + self, itask, event_time, submit_num, forced + ): """Helper for process_message, handle a submit-failed message. Return True if no retries (hence go to the submit-failed state). @@ -1350,16 +1406,21 @@ def _process_message_submit_failed(self, itask, event_time, submit_num): }) itask.summary['submit_method_id'] = None if ( - TimerFlags.SUBMISSION_RETRY not in itask.try_timers + forced + or TimerFlags.SUBMISSION_RETRY not in itask.try_timers or itask.try_timers[TimerFlags.SUBMISSION_RETRY].next() is None ): # No submission retry lined up: definitive failure. # See github #476. no_retries = True - if itask.state_reset(TASK_STATUS_SUBMIT_FAILED): + if itask.state_reset(TASK_STATUS_SUBMIT_FAILED, forced=forced): self.setup_event_handlers( itask, self.EVENT_SUBMIT_FAILED, f'job {self.EVENT_SUBMIT_FAILED}') + itask.state.outputs.set_msg_trg_completion( + message=TASK_OUTPUT_SUBMIT_FAILED, is_completed=True) + self.data_store_mgr.delta_task_output( + itask, TASK_OUTPUT_SUBMIT_FAILED) self.data_store_mgr.delta_task_state(itask) else: # There is a submission retry lined up. @@ -1372,18 +1433,18 @@ def _process_message_submit_failed(self, itask, event_time, submit_num): # Register newly submit-failed job with the database and datastore. job_tokens = itask.tokens.duplicate(job=str(itask.submit_num)) - self._insert_task_job(itask, event_time, self.JOB_SUBMIT_FAIL_FLAG) + self._insert_task_job( + itask, event_time, self.JOB_SUBMIT_FAIL_FLAG, forced=forced) self.data_store_mgr.delta_job_state( job_tokens, TASK_STATUS_SUBMIT_FAILED ) - self._reset_job_timers(itask) return no_retries def _process_message_submitted( - self, itask: 'TaskProxy', event_time: str + self, itask: 'TaskProxy', event_time: str, forced: bool ) -> None: """Helper for process_message, handle a submit-succeeded message.""" with suppress(KeyError): @@ -1399,10 +1460,11 @@ def _process_message_submitted( if itask.tdef.run_mode == 'simulation': # Simulate job started as well. itask.set_summary_time('started', event_time) - if itask.state_reset(TASK_STATUS_RUNNING): + if itask.state_reset(TASK_STATUS_RUNNING, forced=forced): self.data_store_mgr.delta_task_state(itask) itask.state.outputs.set_completion(TASK_OUTPUT_STARTED, True) self.data_store_mgr.delta_task_output(itask, TASK_OUTPUT_STARTED) + else: # Unset started and finished times in case of resubmission. itask.set_summary_time('started') @@ -1411,8 +1473,8 @@ def _process_message_submitted( # The job started message can (rarely) come in before the # submit command returns - in which case do not go back to # 'submitted'. - if itask.state_reset(TASK_STATUS_SUBMITTED): - itask.state_reset(is_queued=False) + if itask.state_reset(TASK_STATUS_SUBMITTED, forced=forced): + itask.state_reset(is_queued=False, forced=forced) self.setup_event_handlers( itask, self.EVENT_SUBMITTED, @@ -1424,7 +1486,8 @@ def _process_message_submitted( # Register the newly submitted job with the database and datastore. # Do after itask has changed state - self._insert_task_job(itask, event_time, self.JOB_SUBMIT_SUCCESS_FLAG) + self._insert_task_job( + itask, event_time, self.JOB_SUBMIT_SUCCESS_FLAG, forced=forced) job_tokens = itask.tokens.duplicate(job=str(itask.submit_num)) self.data_store_mgr.delta_job_time( job_tokens, @@ -1448,7 +1511,8 @@ def _insert_task_job( self, itask: 'TaskProxy', event_time: str, - submit_status: int + submit_status: int, + forced: bool = False ): """Insert a new job proxy into the datastore. @@ -1461,7 +1525,12 @@ def _insert_task_job( # itask.jobs appends for automatic retries (which reuse the same task # proxy) but a retriggered task that was not already in the pool will # not see previous submissions (so can't use itask.jobs[submit_num-1]). - job_conf = itask.jobs[-1] + # And transient tasks, used for setting outputs and spawning children, + # do not submit jobs. + if itask.tdef.run_mode == "simulation" or forced: + job_conf = {"submit_num": 0} + else: + job_conf = itask.jobs[-1] # insert job into data store self.data_store_mgr.insert_job( @@ -1710,6 +1779,10 @@ def _get_handler_template_variables( def _reset_job_timers(self, itask): """Set up poll timer and timeout for task.""" + + if itask.transient: + return + if not itask.state(*TASK_STATUSES_ACTIVE): # Reset, task not active itask.timeout = None @@ -1875,3 +1948,9 @@ def reset_bad_hosts(self): f'{self.bad_hosts}' ) self.bad_hosts.clear() + + def spawn_children(self, itask, output): + # update DB task outputs + self.workflow_db_mgr.put_update_task_outputs(itask) + # spawn child-tasks + self.spawn_func(itask, output) diff --git a/cylc/flow/task_outputs.py b/cylc/flow/task_outputs.py index 4da21f62722..644b7b0dd3c 100644 --- a/cylc/flow/task_outputs.py +++ b/cylc/flow/task_outputs.py @@ -15,6 +15,7 @@ # along with this program. If not, see . """Task output message manager and constants.""" +from typing import List # Standard task output strings, used for triggering. TASK_OUTPUT_EXPIRED = "expired" @@ -69,7 +70,7 @@ class TaskOutputs: def __init__(self, tdef): self._by_message = {} self._by_trigger = {} - self._required = set() + self._required = {} # trigger: message # Add outputs from task def. for trigger, (message, required) in tdef.outputs.items(): @@ -93,7 +94,7 @@ def _add(self, message, trigger, is_completed=False, required=False): self._by_message[message] = [trigger, message, is_completed] self._by_trigger[trigger] = self._by_message[message] if required: - self._required.add(trigger) + self._required[trigger] = message def set_completed_by_msg(self, message): """For flow trigger --wait: set completed outputs from the DB.""" @@ -114,7 +115,7 @@ def exists(self, message=None, trigger=None): return False def get_all(self): - """Return an iterator for all outputs.""" + """Return an iterator for all output messages.""" return sorted(self._by_message.values(), key=self.msg_sort_key) def get_completed(self): @@ -140,6 +141,16 @@ def has_custom_triggers(self): """Return True if it has any custom triggers.""" return any(key not in SORT_ORDERS for key in self._by_trigger) + def _get_custom_triggers(self, required: bool = False) -> List[str]: + """Return list of all, or required, custom trigger messages.""" + custom = [ + out[1] for trg, out in self._by_trigger.items() + if trg not in SORT_ORDERS + ] + if required: + custom = [out for out in custom if out in self._required.values()] + return custom + def get_not_completed(self): """Return all not-completed output messages.""" ret = [] @@ -251,6 +262,30 @@ def _get_item(self, message, trigger): else: return self._by_message[message] + def get_incomplete_implied(self, output: str) -> List[str]: + """Return an ordered list of incomplete implied outputs. + + Use to determined implied outputs to complete automatically. + + Implied outputs are necessarily earlier outputs. + + - started implies submitted + - succeeded and failed imply started + - custom outputs and expired do not imply other outputs + + """ + implied: List[str] = [] + + if output in [TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED]: + # Finished, so it must have submitted and started. + implied = [TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_STARTED] + + elif output == TASK_OUTPUT_STARTED: + # It must have submitted. + implied = [TASK_OUTPUT_SUBMITTED] + + return [out for out in implied if not self.is_completed(out)] + @staticmethod def is_valid_std_name(name): """Check name is a valid standard output name.""" @@ -264,3 +299,18 @@ def msg_sort_key(item): except ValueError: ind = 999 return (ind, item[_MESSAGE] or '') + + @staticmethod + def output_sort_key(item): + """Compare by output order. + + Examples: + + >>> this = TaskOutputs.output_sort_key + >>> sorted(['finished', 'started', 'custom'], key=this) + ['started', 'custom', 'finished'] + """ + if item in TASK_OUTPUTS: + return TASK_OUTPUTS.index(item) + # Sort custom outputs after started. + return TASK_OUTPUTS.index(TASK_OUTPUT_STARTED) + .5 diff --git a/cylc/flow/task_pool.py b/cylc/flow/task_pool.py index db79be4fa5d..a1528ec963b 100644 --- a/cylc/flow/task_pool.py +++ b/cylc/flow/task_pool.py @@ -19,7 +19,6 @@ from contextlib import suppress from collections import Counter import json -from time import time from typing import ( Dict, Iterable, @@ -62,20 +61,28 @@ TASK_STATUS_RUNNING, TASK_STATUS_SUCCEEDED, TASK_STATUS_FAILED, - TASK_OUTPUT_EXPIRED, - TASK_OUTPUT_FAILED, - TASK_OUTPUT_SUCCEEDED, - TASK_OUTPUT_SUBMIT_FAILED, ) +from cylc.flow.task_trigger import TaskTrigger from cylc.flow.util import ( serialise, deserialise ) from cylc.flow.wallclock import get_current_time_string from cylc.flow.platforms import get_platform +from cylc.flow.task_outputs import ( + TASK_OUTPUT_SUCCEEDED, + TASK_OUTPUT_EXPIRED, + TASK_OUTPUT_FAILED, + TASK_OUTPUT_SUBMIT_FAILED, +) from cylc.flow.task_queues.independent import IndepQueueManager -from cylc.flow.flow_mgr import FLOW_ALL, FLOW_NONE, FLOW_NEW +from cylc.flow.flow_mgr import ( + stringify_flow_nums, + FLOW_ALL, + FLOW_NONE, + FLOW_NEW +) if TYPE_CHECKING: from cylc.flow.config import WorkflowConfig @@ -86,6 +93,7 @@ from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager from cylc.flow.flow_mgr import FlowMgr, FlowNums + Pool = Dict['PointBase', Dict[str, TaskProxy]] @@ -110,7 +118,6 @@ def __init__( self.stop_point = config.stop_point or config.final_point self.workflow_db_mgr: 'WorkflowDatabaseManager' = workflow_db_mgr self.task_events_mgr: 'TaskEventsManager' = task_events_mgr - # TODO this is ugly: self.task_events_mgr.spawn_func = self.spawn_on_output self.data_store_mgr: 'DataStoreMgr' = data_store_mgr self.flow_mgr: 'FlowMgr' = flow_mgr @@ -120,12 +127,10 @@ def __init__( self._prev_runahead_sequence_points: Optional[Set['PointBase']] = None self.runahead_limit_point: Optional['PointBase'] = None - self.main_pool: Pool = {} - self.hidden_pool: Pool = {} - self._main_pool_list: List[TaskProxy] = [] - self._hidden_pool_list: List[TaskProxy] = [] - self.main_pool_changed = False - self.hidden_pool_changed = False + # Tasks in the active window of the workflow. + self.active_tasks: Pool = {} + self._active_tasks_list: List[TaskProxy] = [] + self.active_tasks_changed = False self.tasks_removed = False self.hold_point: Optional['PointBase'] = None @@ -169,20 +174,17 @@ def stop_task_done(self): def _swap_out(self, itask): """Swap old task for new, during reload.""" - if itask.identity in self.hidden_pool.get(itask.point, set()): - self.hidden_pool[itask.point][itask.identity] = itask - self.hidden_pool_changed = True - elif itask.identity in self.main_pool.get(itask.point, set()): - self.main_pool[itask.point][itask.identity] = itask - self.main_pool_changed = True + if itask.identity in self.active_tasks.get(itask.point, set()): + self.active_tasks[itask.point][itask.identity] = itask + self.active_tasks_changed = True def load_from_point(self): """Load the task pool for the workflow start point. Add every parentless task out to the runahead limit. """ - flow_num = self.flow_mgr.get_new_flow( - f"original flow from {self.config.start_point}") + flow_num = self.flow_mgr.get_flow_num( + meta=f"original flow from {self.config.start_point}") self.compute_runahead() for name in self.task_name_list: tdef = self.config.get_taskdef(name) @@ -211,34 +213,14 @@ def db_add_new_flow_rows(self, itask: TaskProxy) -> None: self.workflow_db_mgr.put_insert_task_outputs(itask) def add_to_pool(self, itask) -> None: - """Add a task to the hidden (if not satisfied) or main task pool. + """Add a task to the pool.""" - If the task already exists in the hidden pool and is satisfied, move it - to the main pool. - """ - if itask.is_task_prereqs_not_done() and not itask.is_manual_submit: - # Add to hidden pool if not satisfied. - self.hidden_pool.setdefault(itask.point, {}) - self.hidden_pool[itask.point][itask.identity] = itask - self.hidden_pool_changed = True - LOG.debug(f"[{itask}] added to hidden task pool") - else: - # Add to main pool. - # First remove from hidden pool if necessary. - try: - del self.hidden_pool[itask.point][itask.identity] - except KeyError: - pass - else: - self.hidden_pool_changed = True - if not self.hidden_pool[itask.point]: - del self.hidden_pool[itask.point] - self.main_pool.setdefault(itask.point, {}) - self.main_pool[itask.point][itask.identity] = itask - self.main_pool_changed = True - LOG.debug(f"[{itask}] added to main task pool") + self.active_tasks.setdefault(itask.point, {}) + self.active_tasks[itask.point][itask.identity] = itask + self.active_tasks_changed = True + LOG.info(f"[{itask}] added to active task pool") - self.create_data_store_elements(itask) + self.create_data_store_elements(itask) if itask.tdef.max_future_prereq_offset is not None: # (Must do this once added to the pool). @@ -267,8 +249,8 @@ def release_runahead_tasks(self): Return True if any tasks are released, else False. Call when RH limit changes. """ - if not self.main_pool or not self.runahead_limit_point: - # (At start-up main pool might not exist yet) + if not self.active_tasks or not self.runahead_limit_point: + # (At start-up task pool might not exist yet) return False released = False @@ -277,7 +259,7 @@ def release_runahead_tasks(self): # tasks can cause the task pool to change size during iteration. release_me = [ itask - for point, itask_id_map in self.main_pool.items() + for point, itask_id_map in self.active_tasks.items() for itask in itask_id_map.values() if point <= self.runahead_limit_point if itask.state.is_runahead @@ -285,11 +267,12 @@ def release_runahead_tasks(self): for itask in release_me: self.rh_release_and_queue(itask) - self.spawn_to_rh_limit( - itask.tdef, - itask.tdef.next_point(itask.point), - itask.flow_nums - ) + if itask.flow_nums: + self.spawn_to_rh_limit( + itask.tdef, + itask.tdef.next_point(itask.point), + itask.flow_nums + ) released = True return released @@ -304,6 +287,10 @@ def compute_runahead(self, force=False) -> bool: * The max future offset might have changed. * The runahead limit config or task pool might have changed (reload). + This is a collective task pool computation. Call it once at the end + of a group operation such as removal of multiple tasks (not after + every individual task operation). + Start from earliest point with unfinished tasks. Partially satisfied and incomplete tasks count too because they still need to run. @@ -324,7 +311,7 @@ def compute_runahead(self, force=False) -> bool: base_point: Optional['PointBase'] = None # First get the runahead base point. - if not self.main_pool: + if not self.active_tasks: # Find the earliest sequence point beyond the workflow start point. base_point = min( point @@ -430,7 +417,7 @@ def compute_runahead(self, force=False) -> bool: def update_flow_mgr(self): flow_nums_seen = set() - for itask in self.get_all_tasks(): + for itask in self.get_tasks(): flow_nums_seen.update(itask.flow_nums) self.flow_mgr.load_from_db(flow_nums_seen) @@ -546,9 +533,7 @@ def load_db_task_pool_for_restart(self, row_idx, row): sat = {} for prereq_name, prereq_cycle, prereq_output, satisfied in ( self.workflow_db_mgr.pri_dao.select_task_prerequisites( - cycle, - name, - flow_nums, + cycle, name, flow_nums, ) ): # Prereq satisfaction as recorded in the DB. @@ -636,14 +621,14 @@ def load_db_task_action_timers(self, row_idx: int, row: Iterable) -> None: return LOG.info("+ %s/%s %s" % (cycle, name, ctx_key)) if ctx_key == "poll_timer": - itask = self._get_main_task_by_id(id_) + itask = self._get_task_by_id(id_) if itask is None: LOG.warning("%(id)s: task not found, skip" % {"id": id_}) return itask.poll_timer = TaskActionTimer( ctx, delays, num, delay, timeout) elif ctx_key[0] == "try_timers": - itask = self._get_main_task_by_id(id_) + itask = self._get_task_by_id(id_) if itask is None: LOG.warning("%(id)s: task not found, skip" % {"id": id_}) return @@ -706,20 +691,25 @@ def rh_release_and_queue(self, itask) -> None: # (otherwise waiting on xtriggers etc.) self.queue_task(itask) - def _get_spawned_or_merged_task( - self, point: 'PointBase', name: str, flow_nums: 'FlowNums' + def get_or_spawn_task( + self, + point: 'PointBase', + name: str, + flow_nums: 'FlowNums', + flow_wait: bool = False ) -> Optional[TaskProxy]: - """Return new or existing task point/name with merged flow_nums""" - taskid = Tokens(cycle=str(point), task=name).relative_id - ntask = ( - self._get_hidden_task_by_id(taskid) - or self._get_main_task_by_id(taskid) + """Return new or existing task point/name with merged flow_nums. + + It does not add a spawned task proxy to the pool. + """ + ntask = self._get_task_by_id( + Tokens(cycle=str(point), task=name).relative_id ) if ntask is None: # ntask does not exist: spawn it in the flow. - ntask = self.spawn_task(name, point, flow_nums) + ntask = self.spawn_task(name, point, flow_nums, flow_wait) else: - # ntask already exists (n=0 or incomplete): merge flows. + # ntask already exists (n=0): merge flows. self.merge_flows(ntask, flow_nums) return ntask # may be None @@ -733,7 +723,7 @@ def spawn_to_rh_limit(self, tdef, point, flow_nums) -> None: self.compute_runahead() while point is not None and (point <= self.runahead_limit_point): if tdef.is_parentless(point): - ntask = self._get_spawned_or_merged_task( + ntask = self.get_or_spawn_task( point, tdef.name, flow_nums ) if ntask is not None: @@ -741,42 +731,43 @@ def spawn_to_rh_limit(self, tdef, point, flow_nums) -> None: self.rh_release_and_queue(ntask) point = tdef.next_point(point) - # Once more (for the rh-limited task: don't rh release it!) - if point is not None and tdef.is_parentless(point): - ntask = self._get_spawned_or_merged_task( + # Once more for the runahead-limited task (don't release it). + self.spawn_if_parentless(tdef, point, flow_nums) + + def spawn_if_parentless(self, tdef, point, flow_nums): + """Spawn a task if parentless, regardless of runahead limit.""" + if flow_nums and point is not None and tdef.is_parentless(point): + ntask = self.get_or_spawn_task( point, tdef.name, flow_nums ) if ntask is not None: self.add_to_pool(ntask) - def remove(self, itask, reason=""): - """Remove a task from the pool (e.g. after a reload).""" - self.tasks_removed = True - msg = "task proxy removed" - if reason: - msg += f" ({reason})" + def remove(self, itask, reason=None): + """Remove a task from the pool.""" - try: - del self.hidden_pool[itask.point][itask.identity] - except KeyError: - pass - else: - # e.g. for suicide of partially satisfied task - self.hidden_pool_changed = True - if not self.hidden_pool[itask.point]: - del self.hidden_pool[itask.point] - LOG.debug(f"[{itask}] {msg}") - self.task_queue_mgr.remove_task(itask) - return + if itask.state.is_runahead and itask.flow_nums: + # If removing a parentless runahead-limited task + # auto-spawn its next instance first. + self.spawn_if_parentless( + itask.tdef, + itask.tdef.next_point(itask.point), + itask.flow_nums + ) + if reason is None: + msg = "task completed" + else: + msg = f"removed ({reason})" try: - del self.main_pool[itask.point][itask.identity] + del self.active_tasks[itask.point][itask.identity] except KeyError: pass else: - self.main_pool_changed = True - if not self.main_pool[itask.point]: - del self.main_pool[itask.point] + self.tasks_removed = True + self.active_tasks_changed = True + if not self.active_tasks[itask.point]: + del self.active_tasks[itask.point] self.task_queue_mgr.remove_task(itask) if itask.tdef.max_future_prereq_offset is not None: self.set_max_future_offset() @@ -787,66 +778,38 @@ def remove(self, itask, reason=""): # Event-driven final update of task_states table. # TODO: same for datastore (still updated by scheduler loop) self.workflow_db_mgr.put_update_task_state(itask) - LOG.debug(f"[{itask}] {msg}") + LOG.info(f"[{itask}] {msg}") del itask - def get_all_tasks(self) -> List[TaskProxy]: - """Return a list of all task proxies.""" - return self.get_hidden_tasks() + self.get_tasks() - def get_tasks(self) -> List[TaskProxy]: - """Return a list of task proxies in the main pool.""" + """Return a list of task proxies in the task pool.""" # Cached list only for use internally in this method. - if self.main_pool_changed: - self.main_pool_changed = False - self._main_pool_list = [] - for _, itask_id_map in self.main_pool.items(): + if self.active_tasks_changed: + self.active_tasks_changed = False + self._active_tasks_list = [] + for _, itask_id_map in self.active_tasks.items(): for __, itask in itask_id_map.items(): - self._main_pool_list.append(itask) - return self._main_pool_list - - def get_hidden_tasks(self) -> List[TaskProxy]: - """Return a list of task proxies in the hidden pool.""" - # Cached list only for use internally in this method. - if self.hidden_pool_changed: - self.hidden_pool_changed = False - self._hidden_pool_list = [] - for itask_id_maps in self.hidden_pool.values(): - self._hidden_pool_list.extend(list(itask_id_maps.values())) - return self._hidden_pool_list + self._active_tasks_list.append(itask) + return self._active_tasks_list def get_tasks_by_point(self) -> 'Dict[PointBase, List[TaskProxy]]': """Return a map of task proxies by cycle point.""" point_itasks = {} - for point, itask_id_map in self.main_pool.items(): + for point, itask_id_map in self.active_tasks.items(): point_itasks[point] = list(itask_id_map.values()) - for point, itask_id_map in self.hidden_pool.items(): - if point not in point_itasks: - point_itasks[point] = list(itask_id_map.values()) - else: - point_itasks[point] += list(itask_id_map.values()) - return point_itasks def get_task(self, point, name) -> Optional[TaskProxy]: """Retrieve a task from the pool.""" rel_id = f'{point}/{name}' - for pool in (self.main_pool, self.hidden_pool): - tasks = pool.get(point) - if tasks and rel_id in tasks: - return tasks[rel_id] - return None - - def _get_hidden_task_by_id(self, id_: str) -> Optional[TaskProxy]: - """Return runahead pool task by ID if it exists, or None.""" - for itask_ids in list(self.hidden_pool.values()): - with suppress(KeyError): - return itask_ids[id_] + tasks = self.active_tasks.get(point) + if tasks and rel_id in tasks: + return tasks[rel_id] return None - def _get_main_task_by_id(self, id_: str) -> Optional[TaskProxy]: - """Return main pool task by ID if it exists, or None.""" - for itask_ids in list(self.main_pool.values()): + def _get_task_by_id(self, id_: str) -> Optional[TaskProxy]: + """Return pool task by ID if it exists, or None.""" + for itask_ids in self.active_tasks.values(): with suppress(KeyError): return itask_ids[id_] return None @@ -897,8 +860,8 @@ def release_queued_tasks(self): for itask in released: itask.state_reset(is_queued=False) - itask.waiting_on_job_prep = True self.data_store_mgr.delta_task_queued(itask) + itask.waiting_on_job_prep = True if cylc.flow.flags.cylc7_back_compat: # Cylc 7 Back Compat: spawn downstream to cause Cylc 7 style @@ -913,7 +876,7 @@ def release_queued_tasks(self): def get_min_point(self): """Return the minimum cycle point currently in the pool.""" - cycles = list(self.main_pool) + cycles = list(self.active_tasks) minc = None if cycles: minc = min(cycles) @@ -962,7 +925,7 @@ def reload_taskdefs(self, config: 'WorkflowConfig') -> None: self.config.adopt_orphans(orphans) LOG.info("Reloading task definitions.") - tasks = self.get_all_tasks() + tasks = self.get_tasks() # Log tasks orphaned by a reload but not currently in the task pool. for name in orphans: if name not in (itask.tdef.name for itask in tasks): @@ -1016,9 +979,12 @@ def reload_taskdefs(self, config: 'WorkflowConfig') -> None: self.config.runtime['descendants'] ) + if self.compute_runahead(): + self.release_runahead_tasks() + # Now queue all tasks that are ready to run for itask in self.get_tasks(): - # Recreate data store elements from main pool. + # Recreate data store elements from task pool. self.create_data_store_elements(itask) if itask.state.is_queued: # Already queued @@ -1045,7 +1011,7 @@ def set_stop_point(self, stop_point: 'PointBase') -> bool: ): self.runahead_limit_point = stop_point # Now handle existing waiting tasks (e.g. xtriggered). - for itask in self.get_all_tasks(): + for itask in self.get_tasks(): if ( itask.point > stop_point and itask.state(TASK_STATUS_WAITING) @@ -1074,8 +1040,7 @@ def can_stop(self, stop_mode): and itask.state(*TASK_STATUSES_ACTIVE) and not itask.state.kill_failed ) - # we don't need to check for preparing tasks because they will be - # reset to waiting on restart + # preparing tasks get reset to waiting on restart for itask in self.get_tasks() ) @@ -1135,7 +1100,7 @@ def log_incomplete_tasks(self) -> bool: return False def log_unsatisfied_prereqs(self) -> bool: - """Log unsatisfied prerequisites in the hidden pool. + """Log unsatisfied prerequisites in the pool. Return True if any, ignoring: - prerequisites beyond the stop point @@ -1143,17 +1108,21 @@ def log_unsatisfied_prereqs(self) -> bool: (can be caused by future triggers) """ unsat: Dict[str, List[str]] = {} - for itask in self.get_hidden_tasks(): + for itask in self.get_tasks(): task_point = itask.point if self.stop_point and task_point > self.stop_point: continue - for pre in itask.state.get_unsatisfied_prerequisites(): - point, name, output = pre + for point, task, msg in ( + itask.state.get_unsatisfied_prerequisites() + ): if get_point(point) > self.stop_point: continue if itask.identity not in unsat: unsat[itask.identity] = [] - unsat[itask.identity].append(f"{point}/{name}:{output}") + unsat[itask.identity].append( + f"{point}/{task}:" + f"{self.config.get_taskdef(task).get_output(msg)}" + ) if unsat: LOG.warning( "Partially satisfied prerequisites:\n" @@ -1180,6 +1149,8 @@ def is_stalled(self) -> bool: ) or ( itask.state(TASK_STATUS_WAITING) and not itask.state.is_runahead + # (avoid waiting pre-spawned absolute-triggered tasks:) + and not itask.is_task_prereqs_not_done() ) for itask in self.get_tasks() ): return False @@ -1205,7 +1176,7 @@ def release_held_active_task(self, itask: TaskProxy) -> None: def set_hold_point(self, point: 'PointBase') -> None: """Set the point after which all tasks must be held.""" self.hold_point = point - for itask in self.get_all_tasks(): + for itask in self.get_tasks(): if itask.point > point: self.hold_active_task(itask) self.workflow_db_mgr.put_workflow_hold_cycle_point(point) @@ -1249,7 +1220,7 @@ def release_held_tasks(self, items: Iterable[str]) -> int: def release_hold_point(self) -> None: """Unset the workflow hold point and release all held active tasks.""" self.hold_point = None - for itask in self.get_all_tasks(): + for itask in self.get_tasks(): self.release_held_active_task(itask) self.tasks_to_hold.clear() self.workflow_db_mgr.put_tasks_to_hold(self.tasks_to_hold) @@ -1263,26 +1234,29 @@ def check_abort_on_task_fails(self): return self.abort_task_failed def spawn_on_output(self, itask, output, forced=False): - """Spawn and update itask's children, remove itask if finished. + """Spawn child-tasks of given output, into the pool. - Also set a the abort-on-task-failed flag if necessary. + Remove the parent task from the pool if complete. - If not flowing on: - - update existing children but don't spawn new ones - - unless forced (manual command): spawn but with no flow number + Called by task event manager on receiving output messages, and after + forced setting of task outputs (in this case the parent task could + be transient, i.e. not in the pool). + + Also set the abort-on-task-failed flag if necessary. + + If not flowing on, update existing children but don't spawn new ones + (unless manually forced to spawn with no flow number). If an absolute output is completed update the store of completed abs outputs, and update the prerequisites of every instance of the child - in the pool. (And in self.spawn() use the store of completed abs - outputs to satisfy any tasks with abs prerequisites). + in the pool. (The self.spawn method uses the store of completed abs + outputs to satisfy any tasks with absolute prerequisites). Args: - tasks: List of identifiers or task globs. - outputs: List of outputs to spawn on. - forced: If True this is a manual spawn command. + output: output to spawn on. + forced: True if called from manual set task command """ - self.workflow_db_mgr.put_update_task_outputs(itask) if ( output == TASK_OUTPUT_FAILED and self.expected_failed_tasks is not None @@ -1297,6 +1271,12 @@ def spawn_on_output(self, itask, output, forced=False): suicide = [] for c_name, c_point, is_abs in children: + + if itask.flow_wait: + LOG.warning( + f"[{itask}] not spawning on {output}: flow wait requested") + continue + if is_abs: self.abs_outputs_done.add( (str(itask.point), itask.tdef.name, output)) @@ -1308,17 +1288,15 @@ def spawn_on_output(self, itask, output, forced=False): cycle=str(c_point), task=c_name, ).relative_id - c_task = ( - self._get_hidden_task_by_id(c_taskid) - or self._get_main_task_by_id(c_taskid) - ) + + c_task = self._get_task_by_id(c_taskid) + if c_task is not None and c_task != itask: # (Avoid self-suicide: A => !A) self.merge_flows(c_task, itask.flow_nums) elif ( c_task is None and (itask.flow_nums or forced) - and not itask.flow_wait ): # If child is not in the pool already, and parent belongs to a # flow (so it can spawn children), and parent is not waiting @@ -1337,11 +1315,8 @@ def spawn_on_output(self, itask, output, forced=False): else: tasks = [c_task] for t in tasks: - t.state.satisfy_me({ - (str(itask.point), itask.tdef.name, output) - }) + t.satisfy_me([itask.tokens.duplicate(task_sel=output)]) self.data_store_mgr.delta_task_prerequisite(t) - # Add it to the hidden pool or move it to the main pool. self.add_to_pool(t) if t.point <= self.runahead_limit_point: @@ -1364,17 +1339,14 @@ def spawn_on_output(self, itask, output, forced=False): msg += " suiciding while active" self.remove(c_task, msg) - if not forced and output in [ - # final task statuses - TASK_OUTPUT_SUCCEEDED, - TASK_OUTPUT_EXPIRED, - TASK_OUTPUT_FAILED, - TASK_OUTPUT_SUBMIT_FAILED, - ]: - self.remove_if_complete(itask) + self.remove_if_complete(itask, output) + + def remove_if_complete( + self, itask: TaskProxy, output: Optional[str] = None + ) -> bool: + """Remove a finished task if required outputs are complete. - def remove_if_complete(self, itask): - """Remove finished itask if required outputs are complete. + Return True if removed else False. Cylc 8: - if complete: @@ -1388,32 +1360,48 @@ def remove_if_complete(self, itask): else (failed): - retain and recompute runahead (C7 failed tasks don't count toward runahead limit) + """ - ret = False + if not itask.state(*TASK_STATUSES_FINAL): + # can't be complete + return False + + if itask.identity == self.stop_task_id: + self.stop_task_finished = True + if cylc.flow.flags.cylc7_back_compat: + ret = False if not itask.state(TASK_STATUS_FAILED, TASK_OUTPUT_SUBMIT_FAILED): - self.remove(itask, 'finished') + self.remove(itask) ret = True + # Recompute runahead either way; failed tasks don't count in C7. if self.compute_runahead(): self.release_runahead_tasks() - else: - incomplete = itask.state.outputs.get_incomplete() - if incomplete: - # Retain as incomplete. + return ret + + if itask.state(TASK_STATUS_EXPIRED): + self.remove(itask, "expired") + if self.compute_runahead(): + self.release_runahead_tasks() + return True + + incomplete = itask.state.outputs.get_incomplete() + if incomplete: + # Keep incomplete tasks in the pool. + if output in TASK_STATUSES_FINAL: + # Log based on the output, not the state, to avoid warnings + # due to use of "cylc set" to set internal outputs on an + # already-finished task. LOG.warning( f"[{itask}] did not complete required outputs:" f" {incomplete}" ) - else: - # Remove as completed. - self.remove(itask, 'finished') - ret = True - if itask.identity == self.stop_task_id: - self.stop_task_finished = True - if self.compute_runahead(): - self.release_runahead_tasks() + return False - return ret + self.remove(itask) + if self.compute_runahead(): + self.release_runahead_tasks() + return True def spawn_on_all_outputs( self, itask: TaskProxy, completed_only: bool = False @@ -1448,10 +1436,7 @@ def spawn_on_all_outputs( cycle=str(c_point), task=c_name, ).relative_id - c_task = ( - self._get_hidden_task_by_id(c_taskid) - or self._get_main_task_by_id(c_taskid) - ) + c_task = self._get_task_by_id(c_taskid) if c_task is not None: # already spawned continue @@ -1461,9 +1446,9 @@ def spawn_on_all_outputs( # not spawnable continue if completed_only: - c_task.state.satisfy_me({ - (str(itask.point), itask.tdef.name, output) - }) + c_task.satisfy_me( + [itask.tokens.duplicate(task_sel=output)] + ) self.data_store_mgr.delta_task_prerequisite(c_task) self.add_to_pool(c_task) if ( @@ -1472,12 +1457,13 @@ def spawn_on_all_outputs( ): self.rh_release_and_queue(c_task) - def can_spawn(self, name: str, point: 'PointBase') -> bool: - """Return True if the task with the given name & point is within - various workflow limits.""" + def can_be_spawned(self, name: str, point: 'PointBase') -> bool: + """Return True if a point/name is within graph bounds.""" + if name not in self.config.taskdefs: LOG.debug('No task definition %s', name) return False + # Don't spawn outside of graph limits. # TODO: is it possible for initial_point to not be defined?? # (see also the similar check + log message in scheduler.py) @@ -1487,63 +1473,187 @@ def can_spawn(self, name: str, point: 'PointBase') -> bool: LOG.debug( 'Not spawning %s/%s: before initial cycle point', point, name) return False - elif self.config.final_point and point > self.config.final_point: + + if self.config.final_point and point > self.config.final_point: # Only happens on manual trigger beyond FCP LOG.debug( 'Not spawning %s/%s: beyond final cycle point', point, name) return False + + # Is it on-sequence and within recurrence bounds. + if not self.config.get_taskdef(name).is_valid_point(point): + LOG.warning( + self.ERR_PREFIX_TASK_NOT_ON_SEQUENCE.format( + name, point + ) + ) + return False + return True + def _get_task_history( + self, name: str, point: 'PointBase', flow_nums: Set[int] + ) -> Tuple[int, str, bool]: + """Get history of previous submits for this task.""" + + info = self.workflow_db_mgr.pri_dao.select_prev_instances( + name, str(point) + ) + try: + submit_num: int = max(s[0] for s in info) + except ValueError: + # never spawned before in any flow + submit_num = 0 + + prev_status = TASK_STATUS_WAITING + prev_flow_wait = False + + for _snum, f_wait, old_fnums, status in info: + if set.intersection(flow_nums, old_fnums): + # matching flows + prev_status = status + prev_flow_wait = f_wait + if prev_status in TASK_STATUSES_FINAL: + # task finished + break + # Else continue: there may be multiple entries with flow + # overlap due to merges (they'll have have same snum and + # f_wait); keep going to find the finished one, if any. + + return submit_num, prev_status, prev_flow_wait + + def _load_historical_outputs(self, itask): + """Load a task's historical outputs from the DB.""" + info = self.workflow_db_mgr.pri_dao.select_task_outputs( + itask.tdef.name, str(itask.point)) + if not info: + # task never ran before + self.db_add_new_flow_rows(itask) + else: + for outputs_str, fnums in info.items(): + if itask.flow_nums.intersection(fnums): + for msg in json.loads(outputs_str): + itask.state.outputs.set_completed_by_msg(msg) + def spawn_task( self, name: str, point: 'PointBase', flow_nums: Set[int], force: bool = False, - is_manual_submit: bool = False, flow_wait: bool = False, ) -> Optional[TaskProxy]: - """Spawn point/name. Return the spawned task, or None. + """Return task proxy if not completed in this flow, or if forced. + + If finished previously with flow wait, just try to spawn children. + + Note finished tasks may be incomplete, but we don't automatically + re-run incomplete tasks in the same flow. + + For every task spawned, we need a DB lookup for submit number, + and flow-wait. - Force arg used in manual triggering. """ - if not self.can_spawn(name, point): + if not self.can_be_spawned(name, point): return None - # Get submit number by flow_nums {flow_nums: submit_num, ...} - snums = self.workflow_db_mgr.pri_dao.select_submit_nums( - name, str(point) + submit_num, prev_status, prev_flow_wait = ( + self._get_task_history(name, point, flow_nums) ) - try: - submit_num = max(s for s in snums.keys()) - except ValueError: - # Task never spawned in any flow. - submit_num = 0 - flow_wait_done = False - for f_wait, old_fnums in snums.values(): - # Flow_nums of previous instances. - if ( - not force and - set.intersection(flow_nums, old_fnums) - ): - if f_wait: - flow_wait_done = f_wait - break - # To avoid "conditional reflow" with (e.g.) "foo | bar => baz". - LOG.warning( - f"Task {point}/{name} already spawned in {flow_nums}" - ) + itask = self._get_task_proxy_db_outputs( + point, + self.config.get_taskdef(name), + flow_nums, + status=prev_status, + submit_num=submit_num, + flow_wait=flow_wait, + ) + if itask is None: + return None + + if prev_status in TASK_STATUSES_FINAL: + # Task finished previously. + msg = f"[{point}/{name}:{prev_status}] already finished" + if itask.is_complete(): + msg += " and completed" + itask.transient = True + else: + # revive as incomplete. + msg += " incomplete" + + LOG.info( + f"{msg} {stringify_flow_nums(flow_nums, full=True)})" + ) + if prev_flow_wait: + self._spawn_after_flow_wait(itask) + + if itask.transient and not force: return None - # Spawn if on-sequence and within recurrence bounds. - taskdef = self.config.get_taskdef(name) - if not taskdef.is_valid_point(point): - LOG.warning( - self.ERR_PREFIX_TASK_NOT_ON_SEQUENCE.format( - taskdef.name, point + # (else not previously finishedr, so run it) + + if not itask.transient: + if (name, point) in self.tasks_to_hold: + LOG.info(f"[{itask}] holding (as requested earlier)") + self.hold_active_task(itask) + elif self.hold_point and itask.point > self.hold_point: + # Hold if beyond the workflow hold point + LOG.info( + f"[{itask}] holding (beyond workflow " + f"hold point: {self.hold_point})" ) - ) + self.hold_active_task(itask) + + # Don't add to pool if it depends on a task beyond the stop point. + # "foo; foo[+P1] & bar => baz" + # Here, in the final cycle bar wants to spawn baz, but that would + # stall because baz also depends on foo after the final point. + if self.stop_point and itask.point <= self.stop_point: + for pct in itask.state.prerequisites_get_target_points(): + if pct > self.stop_point: + LOG.warning( + f"[{itask}] not spawned: a prerequisite is beyond" + f" the workflow stop point ({self.stop_point})" + ) + return None + + # Satisfy any absolute triggers. + if ( + itask.tdef.has_abs_triggers + and itask.state.prerequisites_are_not_all_satisfied() + ): + itask.satisfy_me([ + Tokens(cycle=cycle, task=task, task_sel=output) + for cycle, task, output in self.abs_outputs_done + ]) + + self.db_add_new_flow_rows(itask) + return itask + + def _spawn_after_flow_wait(self, itask: TaskProxy) -> None: + LOG.info(f"[{itask}] spawning outputs after flow-wait") + self.spawn_on_all_outputs(itask, completed_only=True) + # update flow wait status in the DB + itask.flow_wait = False + # itask.flow_nums = orig_fnums + self.workflow_db_mgr.put_update_task_flow_wait(itask) + return None + + def _get_task_proxy_db_outputs( + self, + point: 'PointBase', + taskdef: 'TaskDef', + flow_nums: 'FlowNums', + status: str = TASK_STATUS_WAITING, + flow_wait: bool = False, + transient: bool = False, + is_manual_submit: bool = False, + submit_num: int = 0, + ) -> Optional['TaskProxy']: + """Spawn a task, update outputs from DB.""" + + if not self.can_be_spawned(taskdef.name, point): return None itask = TaskProxy( @@ -1551,112 +1661,256 @@ def spawn_task( taskdef, point, flow_nums, - submit_num=submit_num, - is_manual_submit=is_manual_submit, + status=status, flow_wait=flow_wait, + submit_num=submit_num, + transient=transient, + is_manual_submit=is_manual_submit ) - if (name, point) in self.tasks_to_hold: - LOG.info(f"[{itask}] holding (as requested earlier)") - self.hold_active_task(itask) - elif self.hold_point and itask.point > self.hold_point: - # Hold if beyond the workflow hold point - LOG.info( - f"[{itask}] holding (beyond workflow " - f"hold point: {self.hold_point})" - ) - self.hold_active_task(itask) - - if self.stop_point and itask.point <= self.stop_point: - future_trigger_overrun = False - for pct in itask.state.prerequisites_get_target_points(): - if pct > self.stop_point: - future_trigger_overrun = True - break - if future_trigger_overrun: - LOG.warning( - f"[{itask}] won't run: depends on a task beyond " - f"the stop point ({self.stop_point})" - ) - - # Satisfy any absolute triggers. - if ( - itask.tdef.has_abs_triggers and - itask.state.prerequisites_are_not_all_satisfied() - ): - itask.state.satisfy_me(self.abs_outputs_done) - - if flow_wait_done: - for outputs_str, fnums in ( - self.workflow_db_mgr.pri_dao.select_task_outputs( - itask.tdef.name, str(itask.point)) - ).items(): - if flow_nums.intersection(fnums): - for msg in json.loads(outputs_str): - itask.state.outputs.set_completed_by_msg(msg) - break - LOG.info(f"{itask} spawning on outputs after flow wait") - self.spawn_on_all_outputs(itask, completed_only=True) + if itask is None: return None - LOG.debug(f"[{itask}] spawned") - self.db_add_new_flow_rows(itask) + # Update it with outputs that were already completed. + info = self.workflow_db_mgr.pri_dao.select_task_outputs( + itask.tdef.name, str(itask.point)) + if not info: + # (Note still need this if task not run before) + self.db_add_new_flow_rows(itask) + for outputs_str, fnums in info.items(): + if flow_nums.intersection(fnums): + for msg in json.loads(outputs_str): + itask.state.outputs.set_completed_by_msg(msg) return itask - def force_spawn_children( + def _standardise_prereqs( + self, prereqs: 'List[str]' + ) -> 'Dict[Tokens, str]': + """Convert prerequisites to a map of task messages: outputs. + + (So satsify_me logs failures) + + """ + _prereqs = {} + for prereq in prereqs: + pre = Tokens(prereq, relative=True) + # add implicit "succeeded"; convert "succeed" to "succeeded" etc. + output = TaskTrigger.standardise_name( + pre['task_sel'] or TASK_OUTPUT_SUCCEEDED) + # Convert outputs to task messages. + try: + msg = self.config.get_taskdef( + pre['task'] + ).outputs[output][0] + cycle = standardise_point_string(pre['cycle']) + except KeyError: + # The task does not have this output. + LOG.warning( + f"output {pre.relative_id_with_selectors} not found") + continue + except WorkflowConfigError as exc: + LOG.warning( + f'Invalid prerequisite task name:\n{exc.args[0]}') + except PointParsingError as exc: + LOG.warning( + f'Invalid prerequisite cycle point:\n{exc.args[0]}') + else: + _prereqs[pre.duplicate(task_sel=msg, cycle=cycle)] = prereq + return _prereqs + + def _standardise_outputs( + self, point: 'PointBase', tdef: 'TaskDef', outputs: List[str] + ) -> List[str]: + """Convert output names to task output messages.""" + _outputs = [] + for out in outputs: + # convert "succeed" to "succeeded" etc. + output = TaskTrigger.standardise_name(out) + try: + msg = tdef.outputs[output][0] + except KeyError: + LOG.warning(f"output {point}/{tdef.name}:{output} not found") + continue + _outputs.append(msg) + return _outputs + + def set_prereqs_and_outputs( self, items: Iterable[str], - outputs: Optional[List[str]] = None, - flow_num: Optional[int] = None + outputs: List[str], + prereqs: List[str], + flow: List[str], + flow_wait: bool = False, + flow_descr: Optional[str] = None ): - """Spawn downstream children of given outputs, on user command. + """Set prerequisites or outputs of target tasks. + + Default: set all required outputs. + + Set prerequisites: + - spawn the task (if not spawned) + - update its prerequisites + + Set outputs: + - update task outputs in the DB + - (implied outputs are handled by the event manager) + - spawn children of the outputs (if not spawned) + - update the child prerequisites + + Task matching restrictions (for now): + - globs (cycle and name) only match in the pool + - future tasks must be specified individually + - family names are not expanded to members - User-facing command name: set_outputs. Creates a transient parent just - for the purpose of spawning children. + + Uses a transient task proxy to spawn children. (Even if parent was + previously spawned in this flow its children might not have been). + + Note transient tasks are a subset of forced tasks (you can + force-trigger a task that is already in the pool). + + A forced output cannot cause a state change to submitted or running, + but it can complete a task so that it doesn't need to run. Args: - items: Identifiers for matching task definitions, each with the - form "point/name". - outputs: List of outputs to spawn on - flow_num: Flow number to attribute the outputs + items: task ID match patterns + prereqs: prerequisites to set + outputs: outputs to set + flow: flow numbers for spawned or merged tasks + flow_wait: wait for flows to catch up before continuing + flow_descr: description of new flow + + """ + flow_nums = self._get_flow_nums(flow, flow_descr) + if flow_nums is None: + # Illegal flow command opts + return + + # Get matching pool tasks and future task definitions. + itasks, future_tasks, unmatched = self.filter_task_proxies( + items, + future=True, + warn=False, + ) + + for itask in itasks: + # Existing task proxies. + self.merge_flows(itask, flow_nums) + if prereqs: + self._set_prereqs_itask(itask, prereqs, flow_nums) + else: + self._set_outputs_itask(itask, outputs) + + for name, point in future_tasks: + tdef = self.config.get_taskdef(name) + if prereqs: + self._set_prereqs_tdef( + point, tdef, prereqs, flow_nums, flow_wait) + else: + trans = self._get_task_proxy_db_outputs( + point, tdef, flow_nums, + flow_wait=flow_wait, transient=True + ) + if trans is not None: + self._set_outputs_itask(trans, outputs) + + if self.compute_runahead(): + self.release_runahead_tasks() + + def _set_outputs_itask( + self, + itask: 'TaskProxy', + outputs: List[str], + ) -> None: + """Set requested outputs on a task proxy and spawn children.""" + + if not outputs: + outputs = itask.tdef.get_required_output_messages() + else: + outputs = self._standardise_outputs( + itask.point, itask.tdef, outputs) + + outputs = sorted(outputs, key=itask.state.outputs.output_sort_key) + for output in outputs: + if itask.state.outputs.is_completed(output): + LOG.info(f"output {itask.identity}:{output} completed already") + continue + self.task_events_mgr.process_message( + itask, logging.INFO, output, forced=True) + + if not itask.state(TASK_STATUS_WAITING): + # Can't be runahead limited or queued. + itask.state_reset(is_runahead=False, is_queued=False) + self.task_queue_mgr.remove_task(itask) + self.data_store_mgr.delta_task_queued(itask) + + self.data_store_mgr.delta_task_state(itask) + self.data_store_mgr.delta_task_outputs(itask) + self.workflow_db_mgr.put_update_task_state(itask) + self.workflow_db_mgr.put_update_task_outputs(itask) + + def _set_prereqs_itask( + self, + itask: 'TaskProxy', + prereqs: 'List[str]', + flow_nums: 'Set[int]', + ) -> bool: + """Set prerequisites on a task proxy. + + Prerequisite format: "cycle/task:output" or "all". + + Return True if any prereqs are valid, else False. """ - outputs = outputs or [TASK_OUTPUT_SUCCEEDED] - if flow_num is None: - flow_nums = None + if prereqs == ["all"]: + itask.state.set_all_satisfied() else: - flow_nums = {flow_num} + # Attempt to set the given presrequisites. + # Log any that aren't valid for the task. + presus = self._standardise_prereqs(prereqs) + unmatched = itask.satisfy_me(list(presus.keys())) + for task_msg in unmatched: + LOG.warning( + f"{itask.identity} does not depend on" + f' "{presus[task_msg]}"' + ) + if len(unmatched) == len(prereqs): + # No prereqs matched. + return False + if ( + self.runahead_limit_point is not None + and itask.point <= self.runahead_limit_point + ): + self.rh_release_and_queue(itask) + self.data_store_mgr.delta_task_prerequisite(itask) + return True - n_warnings, task_items = self.match_taskdefs(items) - for (_, point), taskdef in sorted(task_items.items()): - # This the parent task: - itask = TaskProxy( - self.tokens, - taskdef, - point, - flow_nums=flow_nums, - ) - # Spawn children of selected outputs. - for trig, out, _ in itask.state.outputs.get_all(): - if trig in outputs: - LOG.info(f"[{itask}] Forced spawning on {out}") - self.spawn_on_output(itask, out, forced=True) + def _set_prereqs_tdef( + self, point, taskdef, prereqs, flow_nums, flow_wait + ): + """Spawn a future task and set prerequisites on it.""" + + itask = self.spawn_task(taskdef.name, point, flow_nums, flow_wait) + if itask is None: + return + if self._set_prereqs_itask(itask, prereqs, flow_nums): + self.add_to_pool(itask) def _get_active_flow_nums(self) -> Set[int]: - """Return all active, or most recent previous, flow numbers. + """Return active flow numbers. + + If there are no active flows (e.g. on restarting a completed workflow) + return the most recent active flows. - If there are any active flows, return all active flow numbers. - Otherwise (e.g. on restarting a completed workflow) return - the flow numbers of the most recent previous active task. """ fnums = set() - for itask in self.get_all_tasks(): + for itask in self.get_tasks(): fnums.update(itask.flow_nums) if not fnums: fnums = self.workflow_db_mgr.pri_dao.select_latest_flow_nums() return fnums def remove_tasks(self, items): - """Remove tasks from the pool.""" + """Remove tasks from the pool (forced by command).""" itasks, _, bad_items = self.filter_task_proxies(items) for itask in itasks: self.remove(itask, 'request') @@ -1664,127 +1918,138 @@ def remove_tasks(self, items): self.release_runahead_tasks() return len(bad_items) - def force_trigger_tasks( - self, items: Iterable[str], - flow: List[str], - flow_wait: bool = False, - flow_descr: Optional[str] = None - ) -> int: - """Manual task triggering. - - Don't get a new flow number for existing n=0 tasks (e.g. incomplete - tasks). These can carry on in the original flow if retriggered. - - Queue the task if not queued, otherwise release it to run. - - """ + def _get_flow_nums( + self, + flow: List[str], + meta: Optional[str] = None, + ) -> Optional[Set[int]]: + """Get correct flow numbers given user command options.""" if set(flow).intersection({FLOW_ALL, FLOW_NEW, FLOW_NONE}): if len(flow) != 1: LOG.warning( f'The "flow" values {FLOW_ALL}, {FLOW_NEW} & {FLOW_NONE}' ' cannot be used in combination with integer flow numbers.' ) - return 0 + return None if flow[0] == FLOW_ALL: flow_nums = self._get_active_flow_nums() elif flow[0] == FLOW_NEW: - flow_nums = {self.flow_mgr.get_new_flow(flow_descr)} + flow_nums = {self.flow_mgr.get_flow_num(meta=meta)} elif flow[0] == FLOW_NONE: flow_nums = set() else: try: - flow_nums = {int(n) for n in flow} + flow_nums = { + self.flow_mgr.get_flow_num( + flow_num=int(n), meta=meta + ) + for n in flow + } except ValueError: LOG.warning( - f"Trigger ignored, illegal flow values {flow}" + f"Ignoring command: illegal flow values {flow}" ) - return 0 + return None + return flow_nums + + def _force_trigger(self, itask): + """Assumes task is in the pool""" + # TODO is this flag still needed, and consistent with "cylc set"? + itask.is_manual_submit = True + itask.reset_try_timers() + if itask.state_reset(TASK_STATUS_WAITING): + # (could also be unhandled failed) + self.data_store_mgr.delta_task_state(itask) + # (No need to set prerequisites satisfied here). + if itask.state.is_runahead: + # Release from runahead, and queue it. + self.rh_release_and_queue(itask) + self.spawn_to_rh_limit( + itask.tdef, + itask.tdef.next_point(itask.point), + itask.flow_nums + ) + else: + # De-queue it to run now. + self.task_queue_mgr.force_release_task(itask) - # n_warnings, task_items = self.match_taskdefs(items) - itasks, future_tasks, unmatched = self.filter_task_proxies( - items, - future=True, - warn=False, + def force_trigger_tasks( + self, items: Iterable[str], + flow: List[str], + flow_wait: bool = False, + flow_descr: Optional[str] = None + ): + """Force a task to trigger (user command). + + Always run the task, even if a previous run was flow-waited. + + If the task did not run before in the flow: + - run it, and spawn on outputs unless flow-wait is set. + (but load the previous outputs from the DB) + + Else if the task ran before in the flow: + - load previous outputs + If the previous run was not flow-wait + - run it, and try to spawn on outputs + Else if the previous run was flow-wait: + - just spawn (if not already spawned in this flow) + unless flow-wait is set. + + """ + # Get flow numbers for the tasks to be triggered. + flow_nums = self._get_flow_nums(flow, flow_descr) + if flow_nums is None: + return + + # Get matching tasks proxies, and matching future task IDs. + existing_tasks, future_ids, unmatched = self.filter_task_proxies( + items, future=True, warn=False, ) - # Spawn future tasks. - for name, point in future_tasks: - # (Flow values already validated by the trigger client). - itask = self.spawn_task( - name, + # Trigger existing tasks. + for itask in existing_tasks: + if itask.state(TASK_STATUS_PREPARING, *TASK_STATUSES_ACTIVE): + LOG.warning(f"[{itask}] ignoring trigger - already active") + continue + self._force_trigger(itask) + + # Spawn and trigger future tasks. + for name, point in future_ids: + + if not self.can_be_spawned(name, point): + continue + + submit_num, _prev_status, prev_fwait = self._get_task_history( + name, point, flow_nums) + + itask = TaskProxy( + self.tokens, + self.config.get_taskdef(name), point, flow_nums, - force=True, - is_manual_submit=True, - flow_wait=flow_wait + flow_wait=flow_wait, + submit_num=submit_num, ) if itask is None: continue - itasks.append(itask) - # Trigger matched tasks if not already active. - for itask in itasks: - if itask.state(TASK_STATUS_PREPARING, *TASK_STATUSES_ACTIVE): - LOG.warning(f"[{itask}] ignoring trigger - already active") - continue - itask.is_manual_submit = True - itask.reset_try_timers() - # (If None, spawner reports cycle bounds errors). - if itask.state_reset(TASK_STATUS_WAITING): - # (could also be unhandled failed) - self.data_store_mgr.delta_task_state(itask) - # (No need to set prerequisites satisfied here). - self.add_to_pool(itask) # move from hidden if necessary. - if itask.state.is_runahead: - # Release from runahead, and queue it. - self.rh_release_and_queue(itask) - self.spawn_to_rh_limit( - itask.tdef, - itask.tdef.next_point(itask.point), - itask.flow_nums - ) - else: - # De-queue it to run now. - self.task_queue_mgr.force_release_task(itask) + self.db_add_new_flow_rows(itask) - return len(unmatched) - - def set_expired_tasks(self): - res = False - for itask in self.get_tasks(): - if self._set_expired_task(itask): - res = True - return res + if prev_fwait: + # update completed outputs from the DB + self._load_historical_outputs(itask) - def _set_expired_task(self, itask): - """Check if task has expired. Set state and event handler if so. + # run it (or run it again for incomplete flow-wait) + self.add_to_pool(itask) + self._force_trigger(itask) - Return True if task has expired. - """ - if ( - not itask.state( - TASK_STATUS_WAITING, - is_held=False - ) - or itask.tdef.expiration_offset is None - ): - return False - if itask.expire_time is None: - itask.expire_time = ( - itask.get_point_as_seconds() + - itask.get_offset_as_seconds(itask.tdef.expiration_offset)) - if time() > itask.expire_time: - msg = 'Task expired (skipping job).' - LOG.warning(f"[{itask}] {msg}") - self.task_events_mgr.setup_event_handlers(itask, "expired", msg) - # TODO succeeded and expired states are useless due to immediate - # removal under all circumstances (unhandled failed is still used). - if itask.state_reset(TASK_STATUS_EXPIRED, is_held=False): - self.data_store_mgr.delta_task_state(itask) - self.data_store_mgr.delta_task_held(itask) - self.remove(itask, 'expired') - return True - return False + def clock_expire_tasks(self): + """Expire any tasks past their clock-expiry time.""" + for itask in self.get_tasks(): + if not itask.clock_expire(): + continue + self.task_events_mgr.process_message( + itask, logging.WARNING, TASK_OUTPUT_EXPIRED) def task_succeeded(self, id_): """Return True if task with id_ is in the succeeded state.""" @@ -1802,7 +2067,7 @@ def stop_flow(self, flow_num): Remove the flow number from every task in the pool, and remove any task with no remaining flow numbers if it is not already active. """ - for itask in self.get_all_tasks(): + for itask in self.get_tasks(): try: itask.flow_nums.remove(flow_num) except KeyError: @@ -1813,25 +2078,20 @@ def stop_flow(self, flow_num): *TASK_STATUSES_ACTIVE, TASK_STATUS_PREPARING) and not itask.flow_nums ): + # Don't spawn successor if the task is parentless. self.remove(itask, "flow stopped") + if self.compute_runahead(): + self.release_runahead_tasks() + def log_task_pool(self, log_lvl=logging.DEBUG): - """Log content of task and prerequisite pools in debug mode.""" - for pool, name in [ - (self.get_tasks(), "Main"), - (self.get_hidden_tasks(), "Hidden") - ]: - if pool: - LOG.log( - log_lvl, - f"{name} pool:\n" - + "\n".join( - f"* {itask} status={itask.state.status}" - f" runahead={itask.state.is_runahead}" - f" queued={itask.state.is_queued}" - for itask in pool - ) - ) + """Log content of task pool, for debugging.""" + LOG.log( + log_lvl, + "\n".join( + f"* {itask}" for itask in self.get_tasks() + ) + ) def filter_task_proxies( self, @@ -1860,7 +2120,7 @@ def filter_task_proxies( """ matched, unmatched = filter_ids( - [self.main_pool, self.hidden_pool], + self.active_tasks, ids, warn=warn, ) diff --git a/cylc/flow/task_proxy.py b/cylc/flow/task_proxy.py index 0e7fe1d8868..6afb6a2087e 100644 --- a/cylc/flow/task_proxy.py +++ b/cylc/flow/task_proxy.py @@ -19,6 +19,7 @@ from collections import Counter from copy import copy from fnmatch import fnmatchcase +from time import time from typing import ( Any, Callable, @@ -34,9 +35,23 @@ from metomi.isodatetime.timezone import get_local_time_zone from cylc.flow import LOG +from cylc.flow.flow_mgr import stringify_flow_nums from cylc.flow.platforms import get_platform from cylc.flow.task_action_timer import TimerFlags -from cylc.flow.task_state import TaskState, TASK_STATUS_WAITING +from cylc.flow.task_outputs import ( + TASK_OUTPUT_FAILED, + TASK_OUTPUT_EXPIRED, + TASK_OUTPUT_SUCCEEDED, + TASK_OUTPUT_SUBMIT_FAILED +) +from cylc.flow.task_state import ( + TaskState, + TASK_STATUS_WAITING, + TASK_STATUS_EXPIRED, + TASK_STATUS_SUCCEEDED, + TASK_STATUS_SUBMIT_FAILED, + TASK_STATUS_FAILED +) from cylc.flow.taskdef import generate_graph_children from cylc.flow.wallclock import get_unix_time_from_time_string as str2time from cylc.flow.cycling.iso8601 import ( @@ -46,10 +61,10 @@ ) if TYPE_CHECKING: - from cylc.flow.id import Tokens from cylc.flow.cycling import PointBase from cylc.flow.task_action_timer import TaskActionTimer from cylc.flow.taskdef import TaskDef + from cylc.flow.id import Tokens class TaskProxy: @@ -145,6 +160,9 @@ class TaskProxy: .waiting_on_job_prep: True whilst task is awaiting job prep, reset to False once the preparation has completed. + .transient: + This is a transient proxy - not to be added to the task pool, but + used e.g. to spawn children, or to get task-specific information. Args: tdef: The definition object of this task. @@ -186,6 +204,7 @@ class TaskProxy: 'tokens', 'try_timers', 'waiting_on_job_prep', + 'transient' ] def __init__( @@ -201,6 +220,7 @@ def __init__( is_manual_submit: bool = False, flow_wait: bool = False, data_mode: bool = False, + transient: bool = False ) -> None: self.tdef = tdef @@ -246,6 +266,8 @@ def __init__( else: self.platform = get_platform() + self.transient = transient + self.job_vacated = False self.poll_timer: Optional['TaskActionTimer'] = None self.timeout: Optional[float] = None @@ -266,16 +288,31 @@ def __init__( else: self.graph_children = generate_graph_children(tdef, self.point) + if self.tdef.expiration_offset is not None: + self.expire_time = ( + self.get_point_as_seconds() + + self.get_offset_as_seconds( + self.tdef.expiration_offset + ) + ) + def __repr__(self) -> str: return f"<{self.__class__.__name__} '{self.tokens}'>" def __str__(self) -> str: - """Stringify with tokens, state, submit_num, and flow_nums.""" + """Stringify with tokens, state, submit_num, and flow_nums. + + Don't print submit number for pre job-prep states. + + Format: "//{}:status". + """ + id_ = self.identity + if self.transient: + return f"{id_}{stringify_flow_nums(self.flow_nums)}" + if not self.state(TASK_STATUS_WAITING, TASK_STATUS_EXPIRED): + id_ += f"/{self.submit_num:02d}" return ( - f"{self.identity} " - f"{self.state} " - f"job:{self.submit_num:02d}" - f" flows:{','.join(str(i) for i in self.flow_nums) or 'none'}" + f"{id_}{stringify_flow_nums(self.flow_nums)}:{self.state}" ) def copy_to_reload_successor(self, reload_successor, check_output): @@ -478,12 +515,78 @@ def merge_flows(self, flow_nums: Set) -> None: def state_reset( self, status=None, is_held=None, is_queued=None, is_runahead=None, - silent=False + silent=False, forced=False ) -> bool: - """Set new state and log the change. Return whether it changed.""" + """Set new state and log the change. Return whether it changed. + + """ before = str(self) - if self.state.reset(status, is_held, is_queued, is_runahead): - if not silent: + + if status == TASK_STATUS_EXPIRED: + is_queued = False + is_runahead = False + + if self.state.reset( + status, is_held, is_queued, is_runahead, forced + ): + if not silent and not self.transient: LOG.info(f"[{before}] => {self.state}") return True + return False + + def satisfy_me( + self, task_messages: 'List[Tokens]' + ) -> 'Set[Tokens]': + """Try to satisfy my prerequisites with given output messages. + + The task output messages are of the form "cycle/task:message" + Log a warning for messages that I don't depend on. + + Return a set of unmatched task messages. + + """ + used = self.state.satisfy_me(task_messages) + return set(task_messages) - used + + def clock_expire(self) -> bool: + """Return True if clock expire time is up, else False.""" + if ( + self.expire_time is None # expiry not configured + or self.state(TASK_STATUS_EXPIRED) # already expired + or time() < self.expire_time # not time yet + ): + return False + return True + + def is_finished(self) -> bool: + """Return True if a final state achieved.""" + return ( + self.state( + TASK_STATUS_EXPIRED, + TASK_STATUS_SUBMIT_FAILED, + TASK_STATUS_FAILED, + TASK_STATUS_SUCCEEDED + ) + ) + + def is_complete(self) -> bool: + """Return True if complete or expired.""" + return ( + self.state(TASK_STATUS_EXPIRED) + or not self.state.outputs.is_incomplete() + ) + + def set_state_by_outputs(self) -> None: + """Set state according to which final output is completed.""" + for output in ( + TASK_OUTPUT_EXPIRED, TASK_OUTPUT_SUBMIT_FAILED, + TASK_OUTPUT_FAILED, TASK_OUTPUT_SUCCEEDED + ): + if self.state.outputs.is_completed(output, output): + # This assumes status and output strings are the same: + self.state_reset( + status=output, + silent=True, is_queued=False, is_runahead=False + ) + break diff --git a/cylc/flow/task_qualifiers.py b/cylc/flow/task_qualifiers.py index 68558f76975..eae93ec1ff0 100644 --- a/cylc/flow/task_qualifiers.py +++ b/cylc/flow/task_qualifiers.py @@ -44,6 +44,8 @@ QUAL_FAM_SUBMIT_ANY = "submit-any" QUAL_FAM_SUBMIT_FAIL_ALL = "submit-fail-all" QUAL_FAM_SUBMIT_FAIL_ANY = "submit-fail-any" +QUAL_FAM_EXPIRE_ALL = "expire-all" +QUAL_FAM_EXPIRE_ANY = "expire-any" # alternative (shorthand) qualifiers ALT_QUALIFIERS = { diff --git a/cylc/flow/task_state.py b/cylc/flow/task_state.py index 9dea7e9af28..ebb3dbc985b 100644 --- a/cylc/flow/task_state.py +++ b/cylc/flow/task_state.py @@ -1,6 +1,6 @@ # THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. # Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# + # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or @@ -17,14 +17,14 @@ """Task state related logic.""" -from typing import List +from typing import List, Iterable, Set, TYPE_CHECKING from cylc.flow.prerequisite import Prerequisite -from cylc.flow.task_outputs import ( - TaskOutputs, - TASK_OUTPUT_EXPIRED, TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUBMIT_FAILED, - TASK_OUTPUT_STARTED, TASK_OUTPUT_SUCCEEDED, TASK_OUTPUT_FAILED) +from cylc.flow.task_outputs import TaskOutputs from cylc.flow.wallclock import get_current_time_string +if TYPE_CHECKING: + from cylc.flow.id import Tokens + # Task status names and meanings. # Held back from job submission due to un-met prerequisites: @@ -197,8 +197,10 @@ class TaskState: xtriggers as {trigger (str): satisfied (boolean), ...}. ._is_satisfied (boolean): Are prerequisites satisfied? + Reset None to force re-evaluation when a prereq gets satisfied. ._suicide_is_satisfied (boolean): Are prerequisites to trigger suicide satisfied? + Reset None to force re-evaluation when a prereq gets satisfied. """ # Memory optimization - constrain possible attributes to this list. @@ -308,13 +310,23 @@ def __call__( ) ) - def satisfy_me(self, all_task_outputs): - """Attempt to get my prerequisites satisfied.""" - for prereqs in [self.prerequisites, self.suicide_prerequisites]: - for prereq in prereqs: - if prereq.satisfy_me(all_task_outputs): - self._is_satisfied = None - self._suicide_is_satisfied = None + def satisfy_me( + self, + outputs: Iterable['Tokens'] + ) -> Set['Tokens']: + """Try to satisfy my prerequisites with given outputs. + + Return which outputs I actually depend on. + """ + valid: Set[Tokens] = set() + for prereq in (*self.prerequisites, *self.suicide_prerequisites): + yep = prereq.satisfy_me(outputs) + if yep: + valid = valid.union(yep) + continue + self._is_satisfied = None + self._suicide_is_satisfied = None + return valid def xtriggers_all_satisfied(self): """Return True if all xtriggers are satisfied.""" @@ -324,6 +336,12 @@ def external_triggers_all_satisfied(self): """Return True if all external triggers are satisfied.""" return all(self.external_triggers.values()) + def set_all_satisfied(self): + """Set all my prerequisites satisfied.""" + for p in self.prerequisites: + p.set_satisfied() + self._is_satisfied = True + def prerequisites_all_satisfied(self): """Return True if (non-suicide) prerequisites are fully satisfied.""" if self._is_satisfied is None: @@ -380,12 +398,10 @@ def get_resolved_dependencies(self): ) def reset( - self, status=None, is_held=None, is_queued=None, is_runahead=None): - """Change status, and manipulate outputs and prerequisites accordingly. - - Outputs are manipulated on manual state reset to reflect the new task - status. Since spawn-on-demand implementation, state reset is only used - for internal state changes. + self, status=None, is_held=None, is_queued=None, is_runahead=None, + forced=False + ): + """Change status. Args: status (str): @@ -393,11 +409,19 @@ def reset( is_held (bool): Set the task to be held or not, or None to leave this property unchanged. + forced (bool): + If called as a result of a forced change (via "cylc set") Returns: - returns: whether state change or not (bool) + Whether state changed or not (bool) """ + req = status + + if forced and req in [TASK_STATUS_SUBMITTED, TASK_STATUS_RUNNING]: + # Can't force change to an active state because there's no job. + return False + current_status = ( self.status, self.is_held, @@ -414,39 +438,18 @@ def reset( # no change - do nothing return False - # perform the actual state change + # perform the state change self.status, self.is_held, self.is_queued, self.is_runahead = ( requested_status ) self.time_updated = get_current_time_string() self.is_updated = True - - if is_held: - # only reset task outputs if not setting task to held - # https://github.com/cylc/cylc-flow/pull/2116 - return True - self.kill_failed = False - # Set standard outputs in accordance with task state. if status is None: # NOTE: status is None if the task is being released status = self.status - if status_leq(status, TASK_STATUS_SUBMITTED): - self.outputs.set_all_incomplete() - self.outputs.set_completion( - TASK_OUTPUT_EXPIRED, status == TASK_STATUS_EXPIRED) - self.outputs.set_completion( - TASK_OUTPUT_SUBMITTED, status_geq(status, TASK_STATUS_SUBMITTED)) - self.outputs.set_completion( - TASK_OUTPUT_STARTED, status_geq(status, TASK_STATUS_RUNNING)) - self.outputs.set_completion( - TASK_OUTPUT_SUBMIT_FAILED, status == TASK_STATUS_SUBMIT_FAILED) - self.outputs.set_completion( - TASK_OUTPUT_SUCCEEDED, status == TASK_STATUS_SUCCEEDED) - self.outputs.set_completion( - TASK_OUTPUT_FAILED, status == TASK_STATUS_FAILED) return True @@ -455,6 +458,11 @@ def is_gt(self, status): return (TASK_STATUSES_ORDERED.index(self.status) > TASK_STATUSES_ORDERED.index(status)) + def is_gte(self, status): + """"Return True if self.status >= status.""" + return (TASK_STATUSES_ORDERED.index(self.status) >= + TASK_STATUSES_ORDERED.index(status)) + def _add_prerequisites(self, point, tdef): """Add task prerequisites.""" # Triggers for sequence_i only used if my cycle point is a diff --git a/cylc/flow/taskdef.py b/cylc/flow/taskdef.py index a4ae681e2d9..1da5101306b 100644 --- a/cylc/flow/taskdef.py +++ b/cylc/flow/taskdef.py @@ -22,13 +22,14 @@ import cylc.flow.flags from cylc.flow.exceptions import TaskDefError from cylc.flow.task_id import TaskID -from cylc.flow.task_state import ( +from cylc.flow.task_outputs import ( + TASK_OUTPUT_EXPIRED, TASK_OUTPUT_SUBMITTED, TASK_OUTPUT_SUBMIT_FAILED, TASK_OUTPUT_SUCCEEDED, - TASK_OUTPUT_FAILED + TASK_OUTPUT_FAILED, + SORT_ORDERS ) -from cylc.flow.task_outputs import SORT_ORDERS if TYPE_CHECKING: from cylc.flow.cycling import PointBase @@ -178,6 +179,13 @@ def add_output(self, output, message): # optional/required is None until defined by the graph self.outputs[output] = (message, None) + def get_output(self, message): + """Return output name corresponding to task message.""" + for name, (msg, _) in self.outputs.items(): + if msg == message: + return name + raise KeyError(f"Unknown task output message: {message}") + def _add_std_outputs(self): """Add the standard outputs.""" # optional/required is None until defined by the graph @@ -186,10 +194,14 @@ def _add_std_outputs(self): def set_required_output(self, output, required): """Set outputs to required or optional.""" - # (Note outputs and associated messages already defined.) + # (Note outputs and associated messages are already defined.) message, _ = self.outputs[output] self.outputs[output] = (message, required) + def get_required_output_messages(self): + """Return list of required outputs (as task messages).""" + return [msg for (msg, req) in self.outputs.values() if req] + def tweak_outputs(self): """Output consistency checking and tweaking.""" @@ -204,6 +216,9 @@ def tweak_outputs(self): ): self.set_required_output(TASK_OUTPUT_SUCCEEDED, True) + # Expired must be optional + self.set_required_output(TASK_OUTPUT_EXPIRED, False) + # In Cylc 7 back compat mode, make all success outputs required. if cylc.flow.flags.cylc7_back_compat: for output in [ diff --git a/cylc/flow/tui/data.py b/cylc/flow/tui/data.py index 04c3d7220b1..90344e2462b 100644 --- a/cylc/flow/tui/data.py +++ b/cylc/flow/tui/data.py @@ -110,6 +110,7 @@ 'kill', 'trigger', 'poll', + 'set', ], 'job': [ 'kill', diff --git a/cylc/flow/unicode_rules.py b/cylc/flow/unicode_rules.py index 4c71e190ce3..4608559798d 100644 --- a/cylc/flow/unicode_rules.py +++ b/cylc/flow/unicode_rules.py @@ -23,6 +23,7 @@ _TASK_NAME_PREFIX, ) from cylc.flow.task_qualifiers import TASK_QUALIFIERS +from cylc.flow.task_state import TASK_STATUSES_ORDERED ENGLISH_REGEX_MAP = { r'\w': 'alphanumeric', @@ -350,8 +351,8 @@ class TaskOutputValidator(UnicodeRuleChecker): not_starts_with('_cylc'), # blacklist keywords not_equals('required', 'optional', 'all'), - # blacklist built-in task qualifiers - not_equals(*TASK_QUALIFIERS), + # blacklist built-in task qualifiers and statuses (e.g. "waiting") + not_equals(*sorted({*TASK_QUALIFIERS, *TASK_STATUSES_ORDERED})), ] diff --git a/cylc/flow/workflow_db_mgr.py b/cylc/flow/workflow_db_mgr.py index 0b98d0662d9..128cfd45126 100644 --- a/cylc/flow/workflow_db_mgr.py +++ b/cylc/flow/workflow_db_mgr.py @@ -424,13 +424,34 @@ def put_update_task_state(self, itask): "time_updated": itask.state.time_updated, "status": itask.state.status, "flow_wait": itask.flow_wait, - "is_manual_submit": itask.is_manual_submit + "is_manual_submit": itask.is_manual_submit, + } + where_args = { + "cycle": str(itask.point), + "name": itask.tdef.name, + "flow_nums": serialise(itask.flow_nums), + } + # Note tasks_states table rows are for latest submit_num only + # (not one row per submit). + self.db_updates_map.setdefault(self.TABLE_TASK_STATES, []) + self.db_updates_map[self.TABLE_TASK_STATES].append( + (set_args, where_args)) + + def put_update_task_flow_wait(self, itask): + """Update flow_wait status of a task, in the task_states table. + + NOTE the task_states table is normally updated along with the task pool + table. This method is only needed as a final update for a non-pool task + that just spawned its children after a flow wait. + """ + set_args = { + "time_updated": itask.state.time_updated, + "flow_wait": itask.flow_wait, } where_args = { "cycle": str(itask.point), "name": itask.tdef.name, "flow_nums": serialise(itask.flow_nums), - "submit_num": itask.submit_num, } self.db_updates_map.setdefault(self.TABLE_TASK_STATES, []) self.db_updates_map[self.TABLE_TASK_STATES].append( @@ -454,7 +475,7 @@ def put_task_pool(self, pool: 'TaskPool') -> None: # This should already be done by self.put_task_event_timers above: # self.db_deletes_map[self.TABLE_TASK_ACTION_TIMERS].append({}) self.db_deletes_map[self.TABLE_TASK_TIMEOUT_TIMERS].append({}) - for itask in pool.get_all_tasks(): + for itask in pool.get_tasks(): for prereq in itask.state.prerequisites: for (p_cycle, p_name, p_output), satisfied_state in ( prereq.satisfied.items() @@ -508,7 +529,8 @@ def put_task_pool(self, pool: 'TaskPool') -> None: "time_updated": itask.state.time_updated, "submit_num": itask.submit_num, "try_num": itask.get_try_num(), - "status": itask.state.status + "status": itask.state.status, + "is_manual_submit": itask.is_manual_submit, } where_args = { "cycle": str(itask.point), diff --git a/setup.cfg b/setup.cfg index 705856f8a47..45a66d81c88 100644 --- a/setup.cfg +++ b/setup.cfg @@ -190,11 +190,11 @@ cylc.command = remove = cylc.flow.scripts.remove:main report-timings = cylc.flow.scripts.report_timings:main [report-timings] scan = cylc.flow.scripts.scan:cli - set-verbosity = cylc.flow.scripts.set_verbosity:main show = cylc.flow.scripts.show:main - set-outputs = cylc.flow.scripts.set_outputs:main + set = cylc.flow.scripts.set:main stop = cylc.flow.scripts.stop:main subscribe = cylc.flow.scripts.subscribe:main + verbosity = cylc.flow.scripts.verbosity:main workflow-state = cylc.flow.scripts.workflow_state:main tui = cylc.flow.scripts.tui:main trigger = cylc.flow.scripts.trigger:main diff --git a/tests/conftest.py b/tests/conftest.py index 8ed3e210fc6..4207a10f165 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -112,7 +112,7 @@ def _log_filter( if (name is None or name == log_name) and (level is None or level == log_level) and (contains is None or contains in log_message) - and (regex is None or re.match(regex, log_message)) + and (regex is None or re.search(regex, log_message)) and (exact_match is None or exact_match == log_message) ] return _log_filter diff --git a/tests/flakyfunctional/cylc-poll/03-poll-all/flow.cylc b/tests/flakyfunctional/cylc-poll/03-poll-all/flow.cylc index 5e21a978fea..709e3bb16e6 100644 --- a/tests/flakyfunctional/cylc-poll/03-poll-all/flow.cylc +++ b/tests/flakyfunctional/cylc-poll/03-poll-all/flow.cylc @@ -41,7 +41,7 @@ cylc poll "${CYLC_WORKFLOW_ID}//*" cylc__job__poll_grep_workflow_log \ - "${CYLC_TASK_CYCLE_POINT}/submit_hold preparing .* => submitted" + "${CYLC_TASK_CYCLE_POINT}/submit_hold/01:preparing.* => submitted" st_file="${CYLC_WORKFLOW_RUN_DIR}/log/job/${CYLC_TASK_CYCLE_POINT}/submit_hold/NN/job.status" pkill -g "$(awk -F= '$1 == "CYLC_JOB_ID" {print $2}' "${st_file}")" diff --git a/tests/flakyfunctional/cylc-poll/16-execution-time-limit.t b/tests/flakyfunctional/cylc-poll/16-execution-time-limit.t index 830a4f71aff..c437867251d 100755 --- a/tests/flakyfunctional/cylc-poll/16-execution-time-limit.t +++ b/tests/flakyfunctional/cylc-poll/16-execution-time-limit.t @@ -58,14 +58,14 @@ __PYTHON__ LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Test logging of the "next job poll" message when task starts. TEST_NAME="${TEST_NAME_BASE}-log-entry" -LINE="$(grep '\[1/foo\ .* execution timeout=None, polling intervals=' "${LOG}")" +LINE="$(grep '\[1/foo.* execution timeout=None, polling intervals=' "${LOG}")" run_ok "${TEST_NAME}" grep -q 'health: execution timeout=None, polling intervals=' <<< "${LINE}" # Determine poll times. PREDICTED_POLL_TIME=$(time_offset \ "$(cut -d ' ' -f 1 <<< "${LINE}")" \ "PT10S") # PT5S time limit + PT5S polling interval ACTUAL_POLL_TIME=$(sed -n \ - 's|\(.*\) DEBUG - \[1/foo running .* (polled)failed .*|\1|p' "${LOG}") + 's|\(.*\) DEBUG - \[1/foo.* (polled)failed .*|\1|p' "${LOG}") # Test execution timeout polling. # Main loop is roughly 1 second, but integer rounding may give an apparent 2 diff --git a/tests/flakyfunctional/hold-release/14-hold-kill/flow.cylc b/tests/flakyfunctional/hold-release/14-hold-kill/flow.cylc index 7b82bd3e15c..9a2c51a88fd 100644 --- a/tests/flakyfunctional/hold-release/14-hold-kill/flow.cylc +++ b/tests/flakyfunctional/hold-release/14-hold-kill/flow.cylc @@ -9,7 +9,7 @@ echo '# killing "sleeper"' cylc kill "${CYLC_WORKFLOW_ID}//1/sleeper" cylc__job__poll_grep_workflow_log -E \ - '1/sleeper\ waiting\(held\) .* job killed' + '1/sleeper:waiting\(held\).* job killed' sleep 10 # sleep, should still be held after 10 seconds cylc dump -s -t "${CYLC_WORKFLOW_ID}" >'cylc-dump.out' diff --git a/tests/flakyfunctional/hold-release/15-hold-after/flow.cylc b/tests/flakyfunctional/hold-release/15-hold-after/flow.cylc index 71cb8d14b25..fe9e2067a34 100644 --- a/tests/flakyfunctional/hold-release/15-hold-after/flow.cylc +++ b/tests/flakyfunctional/hold-release/15-hold-after/flow.cylc @@ -23,7 +23,7 @@ script = cylc hold --after '20140101T12' "${CYLC_WORKFLOW_ID}" [[stopper]] script = """ - cylc__job__poll_grep_workflow_log -E '20140101T1200Z/bar .* \(received\)succeeded' + cylc__job__poll_grep_workflow_log -E '20140101T1200Z/bar/01.* \(received\)succeeded' cylc stop "${CYLC_WORKFLOW_ID}" """ [[[job]]] diff --git a/tests/functional/authentication/00-shared-fs.t b/tests/functional/authentication/00-shared-fs.t index 0e8b268b36e..dcfc6242648 100755 --- a/tests/functional/authentication/00-shared-fs.t +++ b/tests/functional/authentication/00-shared-fs.t @@ -41,8 +41,8 @@ WORKFLOW_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Note: double poll existence of workflow log on workflow host and then localhost to # avoid any issues with unstable mounting of the shared file system. poll ssh -oBatchMode=yes -n "${CYLC_TEST_HOST}" test -e "${WORKFLOW_LOG}" -poll_grep_workflow_log -E '19700101T0000Z/t1 submitted .* => running' -poll_grep_workflow_log -E '19700101T0000Z/t1 running .* => failed' +poll_grep_workflow_log -E '19700101T0000Z/t1/01:submitted.* => running' +poll_grep_workflow_log -E '19700101T0000Z/t1/01:running.* => failed' run_ok "${TEST_NAME_BASE}-broadcast" \ cylc broadcast -n 't1' -s '[environment]CYLC_TEST_VAR_FOO=foo' "${WORKFLOW_NAME}" diff --git a/tests/functional/broadcast/07-timeout/flow.cylc b/tests/functional/broadcast/07-timeout/flow.cylc index 46bbc749fe9..055f09e729b 100644 --- a/tests/functional/broadcast/07-timeout/flow.cylc +++ b/tests/functional/broadcast/07-timeout/flow.cylc @@ -18,7 +18,7 @@ [[timeout]] script = """ cylc__job__poll_grep_workflow_log -E \ - "${CYLC_TASK_ID} .* execution timeout after PT1S" + "${CYLC_TASK_ID}.* execution timeout after PT1S" """ [[[events]]] execution timeout = PT1M diff --git a/tests/functional/cli/02-now.t b/tests/functional/cli/02-now.t deleted file mode 100755 index f6bed459275..00000000000 --- a/tests/functional/cli/02-now.t +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. -# Copyright (C) NIWA & British Crown (Met Office) & Contributors. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -#------------------------------------------------------------------------------- -# Test "cylc play --icp=now WORKFLOW". -# And "cylc play --icp=next(...) WORKFLOW" and "cylc play --icp=previous(...) WORKFLOW" -# And restart. - -. "$(dirname "$0")/test_header" -set_test_number 10 -init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' -[scheduler] - [[events]] - abort on stall timeout = true - stall timeout = PT0S - abort on inactivity timeout = true - inactivity timeout = PT1M -[scheduling] - [[graph]] - R1 = foo -[runtime] - [[foo]] - script = cylc__job__wait_cylc_message_started; cylc stop --now --now "${CYLC_WORKFLOW_ID}" -__FLOW_CONFIG__ - -run_ok "${TEST_NAME_BASE}-validate" cylc validate --icp='now' "${WORKFLOW_NAME}" - -for ICP in 'now' 'next(T00)' 'previous(T00)'; do - workflow_run_ok "${TEST_NAME_BASE}-run-icp-${ICP}" \ - cylc play --debug --no-detach --icp="${ICP}" "${WORKFLOW_NAME}" - - workflow_run_ok "${TEST_NAME_BASE}-restart-icp-${ICP}" \ - cylc play --debug --no-detach "${WORKFLOW_NAME}" - - sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM task_pool' >'task_pool.out' - cmp_ok 'task_pool.out' <'/dev/null' - # TODO - is this test still useful? - # Consider checking the task succeeded in the task_states table, instead. - # pre-SoD: - # sqlite3 "${WORKFLOW_RUN_DIR}/log/db" 'SELECT * FROM task_pool' >'task_pool.out' - # cmp_ok 'task_pool.out' <<__OUT__ - # ${MY_CYCLE}|foo|1|succeeded|0 - # __OUT__ - - delete_db -done - -purge -exit diff --git a/tests/functional/cli/03-set-verbosity.t b/tests/functional/cli/03-verbosity.t similarity index 89% rename from tests/functional/cli/03-set-verbosity.t rename to tests/functional/cli/03-verbosity.t index 7938f91cab7..b3103fd67d3 100755 --- a/tests/functional/cli/03-set-verbosity.t +++ b/tests/functional/cli/03-verbosity.t @@ -15,13 +15,13 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- -# Test "cylc set-verbosity" +# Test "cylc verbosity" . "$(dirname "$0")/test_header" set_test_number 4 # Test illegal log level TEST_NAME="${TEST_NAME_BASE}-bad" -run_fail "$TEST_NAME" cylc set-verbosity duck quack +run_fail "$TEST_NAME" cylc verbosity duck quack grep_ok 'InputError: Illegal logging level, duck' "${TEST_NAME}.stderr" # Test good log level @@ -42,8 +42,8 @@ init_workflow "${TEST_NAME_BASE}" << '__FLOW__' echo "CYLC_DEBUG: $CYLC_DEBUG" [[ "$CYLC_DEBUG" != 'true' ]] - cylc set-verbosity DEBUG "$CYLC_WORKFLOW_ID" - cylc__job__poll_grep_workflow_log 'Command actioned: set_verbosity' + cylc verbosity DEBUG "$CYLC_WORKFLOW_ID" + cylc__job__poll_grep_workflow_log 'Command "set_verbosity" actioned' """ [[getter]] script = """ diff --git a/tests/functional/clock-expire/00-basic/flow.cylc b/tests/functional/clock-expire/00-basic/flow.cylc index afeb7df356d..3605ca885e8 100644 --- a/tests/functional/clock-expire/00-basic/flow.cylc +++ b/tests/functional/clock-expire/00-basic/flow.cylc @@ -23,7 +23,7 @@ Skip a daily post-processing workflow if the 'copy' task has expired.""" # behind "now + 1 day". This makes the first two 'copy' tasks expire. [[graph]] P1D = """ - model[-P1D] => model => copy => proc + model[-P1D] => model => copy? => proc copy:expired? => !proc """ [runtime] diff --git a/tests/functional/cylc-cat-log/05-remote-tail.t b/tests/functional/cylc-cat-log/05-remote-tail.t index f3a787b5675..e75a87df862 100755 --- a/tests/functional/cylc-cat-log/05-remote-tail.t +++ b/tests/functional/cylc-cat-log/05-remote-tail.t @@ -41,7 +41,7 @@ $SCP "${PWD}/bin/my-tailer.sh" \ # Run detached. workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" #------------------------------------------------------------------------------- -poll_grep_workflow_log -E '1/foo preparing .* => submitted' +poll_grep_workflow_log -E '1/foo/01:preparing.* => submitted' # cylc cat-log -m 't' tail-follows a file, so needs to be killed. # Send interrupt signal to tail command after 15 seconds. TEST_NAME="${TEST_NAME_BASE}-cat-log" diff --git a/tests/functional/cylc-kill/03-simulation.t b/tests/functional/cylc-kill/03-simulation.t index f9152a84a1f..dadfeede123 100755 --- a/tests/functional/cylc-kill/03-simulation.t +++ b/tests/functional/cylc-kill/03-simulation.t @@ -28,7 +28,7 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" cylc play --debug -m simulation "${WORKFLOW_NAME}" >/dev/null 2>&1 # wait for simulated job start -poll_grep_workflow_log "1/foo .* running" -E +poll_grep_workflow_log "1/foo.* running" -E # kill it run_ok killer cylc kill "${WORKFLOW_NAME}//1/foo" @@ -37,6 +37,6 @@ run_ok killer cylc kill "${WORKFLOW_NAME}//1/foo" poll_grep_workflow_log "INFO - DONE" # check the sim job was kiled -grep_workflow_log_ok killed "1/foo .* failed" -E +grep_workflow_log_ok killed "1/foo.* failed" -E purge diff --git a/tests/functional/cylc-message/02-multi.t b/tests/functional/cylc-message/02-multi.t index eab3148cde4..17a0e3b4189 100755 --- a/tests/functional/cylc-message/02-multi.t +++ b/tests/functional/cylc-message/02-multi.t @@ -57,19 +57,19 @@ sed -i 's/\(^.*\) at .*$/\1/;' 'sed.out' # Note: the continuation bit gets printed twice, because the message gets a # warning as being unhandled. cmp_ok 'sed.out' <<__LOG__ -DEBUG - [1/foo submitted job:01 flows:1] (received)started -WARNING - [1/foo running job:01 flows:1] (received)Warn this -INFO - [1/foo running job:01 flows:1] (received)Greeting -WARNING - [1/foo running job:01 flows:1] (received)Warn that -DEBUG - [1/foo running job:01 flows:1] (received)Remove stuffs such as +DEBUG - [1/foo/01:submitted] (received)started +WARNING - [1/foo/01:running] (received)Warn this +INFO - [1/foo/01:running] (received)Greeting +WARNING - [1/foo/01:running] (received)Warn that +DEBUG - [1/foo/01:running] (received)Remove stuffs such as ${LOG_INDENT}badness ${LOG_INDENT}slowness ${LOG_INDENT}and other incorrectness. ${LOG_INDENT}badness ${LOG_INDENT}slowness ${LOG_INDENT}and other incorrectness. -INFO - [1/foo running job:01 flows:1] (received)whatever -DEBUG - [1/foo running job:01 flows:1] (received)succeeded +INFO - [1/foo/01:running] (received)whatever +DEBUG - [1/foo/01:running] (received)succeeded __LOG__ purge diff --git a/tests/functional/cylc-poll/05-poll-multi-messages/flow.cylc b/tests/functional/cylc-poll/05-poll-multi-messages/flow.cylc index 9adf11d9107..18fd4112e51 100644 --- a/tests/functional/cylc-poll/05-poll-multi-messages/flow.cylc +++ b/tests/functional/cylc-poll/05-poll-multi-messages/flow.cylc @@ -20,8 +20,8 @@ echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|hello1" echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|hello2" } >>"${CYLC_TASK_LOG_ROOT}.status" - cylc__job__poll_grep_workflow_log -E '1/speaker1 running .* \(polled\)hello1' - cylc__job__poll_grep_workflow_log -E '1/speaker1 running .* \(polled\)hello2' + cylc__job__poll_grep_workflow_log -E '1/speaker1/01:running.* \(polled\)hello1' + cylc__job__poll_grep_workflow_log -E '1/speaker1/01:running.* \(polled\)hello2' """ [[[outputs]]] hello1 = "hello1" @@ -34,7 +34,7 @@ # get sent back to the workflow echo "CYLC_MESSAGE=$(date +%FT%H:%M:%SZ)|INFO|greet" \ >>"${CYLC_TASK_LOG_ROOT}.status" - cylc__job__poll_grep_workflow_log -E '1/speaker2 running .* \(polled\)greet' + cylc__job__poll_grep_workflow_log -E '1/speaker2/01:running.* \(polled\)greet' """ [[[outputs]]] greet = "greet" diff --git a/tests/functional/cylc-poll/13-comm-method.t b/tests/functional/cylc-poll/13-comm-method.t index 91da26bd4cd..282f7106671 100755 --- a/tests/functional/cylc-poll/13-comm-method.t +++ b/tests/functional/cylc-poll/13-comm-method.t @@ -39,7 +39,7 @@ PRE_MSG='health:' POST_MSG='.*, polling intervals=10\*PT6S...' for INDEX in 1 2; do for STAGE in 'submission' 'execution'; do - grep_ok "1/t${INDEX} .* ${PRE_MSG} ${STAGE}${POST_MSG}" "${LOG_FILE}" -E + grep_ok "1/t${INDEX}.* ${PRE_MSG} ${STAGE}${POST_MSG}" "${LOG_FILE}" -E done done #------------------------------------------------------------------------------- diff --git a/tests/functional/cylc-poll/14-intervals.t b/tests/functional/cylc-poll/14-intervals.t index 9d9b308f4c3..f7807b27d4d 100755 --- a/tests/functional/cylc-poll/14-intervals.t +++ b/tests/functional/cylc-poll/14-intervals.t @@ -41,7 +41,7 @@ for INDEX in 1 2; do POLL_INT='2\*PT1S,10\*PT6S,' fi POST_MSG=".*, polling intervals=${POLL_INT}..." - grep_ok "1/t${INDEX} .* ${PRE_MSG} ${STAGE}${POST_MSG}" "${LOG_FILE}" -E + grep_ok "1/t${INDEX}.*${PRE_MSG} ${STAGE}${POST_MSG}" "${LOG_FILE}" -E done done #------------------------------------------------------------------------------- diff --git a/tests/functional/cylc-poll/15-job-st-file-no-batch.t b/tests/functional/cylc-poll/15-job-st-file-no-batch.t index 4331cab1b63..017d0c39f1b 100755 --- a/tests/functional/cylc-poll/15-job-st-file-no-batch.t +++ b/tests/functional/cylc-poll/15-job-st-file-no-batch.t @@ -28,7 +28,7 @@ LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" run_ok "${TEST_NAME_BASE}-log-1" \ grep -F '[jobs-poll err] 1/t1/01/job.status: incomplete job runner info' "${LOG}" run_ok "${TEST_NAME_BASE}-log-2" \ - grep -E '1/t1 running .*\(polled\)failed' "${LOG}" + grep -E '1/t1/01:running.*\(polled\)failed' "${LOG}" purge exit diff --git a/tests/functional/cylc-remove/00-simple/flow.cylc b/tests/functional/cylc-remove/00-simple/flow.cylc index 15c0b09dc64..84c740ad421 100644 --- a/tests/functional/cylc-remove/00-simple/flow.cylc +++ b/tests/functional/cylc-remove/00-simple/flow.cylc @@ -15,7 +15,7 @@ script = false [[cleaner]] script = """ -cylc__job__poll_grep_workflow_log -E '1/b\ running .* \(received\)failed' +cylc__job__poll_grep_workflow_log -E '1/b/01:running.* \(received\)failed' # Remove the unhandled failed task cylc remove "$CYLC_WORKFLOW_ID//1/b" # Remove waiting 1/c diff --git a/tests/functional/cylc-remove/02-cycling/flow.cylc b/tests/functional/cylc-remove/02-cycling/flow.cylc index a2a236eb004..3b6c1051493 100644 --- a/tests/functional/cylc-remove/02-cycling/flow.cylc +++ b/tests/functional/cylc-remove/02-cycling/flow.cylc @@ -17,8 +17,8 @@ [runtime] [[remover]] script = """ - cylc__job__poll_grep_workflow_log -E '2020/bar running .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '2021/baz running .* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '2020/bar/01:running.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '2021/baz/01:running.* \(received\)failed' # Remove the two unhandled failed tasks. cylc remove "$CYLC_WORKFLOW_ID//*/ba*:failed" # Remove the two unsatisfied waiting tasks. diff --git a/tests/functional/cylc-set/00-set-succeeded.t b/tests/functional/cylc-set/00-set-succeeded.t new file mode 100644 index 00000000000..df6ed99812d --- /dev/null +++ b/tests/functional/cylc-set/00-set-succeeded.t @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 1 - Carry on as if a failed task had succeeded +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#1-carry-on-as-if-a-failed-task-had-succeeded + +. "$(dirname "$0")/test_header" +set_test_number 6 + +install_and_validate +reftest_run + +for TASK in foo bar +do + sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ + "SELECT status FROM task_states WHERE name is \"$TASK\"" > "${TASK}.1" + + cmp_ok ${TASK}.1 - << "${TASK}.2" + + cmp_json \ + "check-${TASK}-outputs" \ + "${TASK}.2" \ + "${TASK}.2"<<<'["submitted", "started", "succeeded", "hello"]' + +done +purge diff --git a/tests/functional/cylc-set/00-set-succeeded/flow.cylc b/tests/functional/cylc-set/00-set-succeeded/flow.cylc new file mode 100644 index 00000000000..2385af8b427 --- /dev/null +++ b/tests/functional/cylc-set/00-set-succeeded/flow.cylc @@ -0,0 +1,41 @@ +# 1. foo and bar fail incomplete. +# 2. setter sets foo and bar to succeeded. +# 3. foo and bar are completed, post runs, scheduler shuts down. + +[scheduler] + [[events]] + inactivity timeout = PT30S + abort on inactivity timeout = True + expected task failures = 1/foo, 1/bar + +[task parameters] + m = 1..2 + +[scheduling] + [[graph]] + R1 = """ + foo & bar => post + setter + """ +[runtime] + [[post]] + [[foo, bar]] + script = false + [[setter]] + script = """ + # wait for foo and bar to fail. + for TASK in foo bar + do + cylc workflow-state \ + --max-polls=10 \ + --interval=1 \ + --task=$TASK \ + --point=${CYLC_TASK_CYCLE_POINT} \ + --status=failed \ + $CYLC_WORKFLOW_ID + done + # set foo succeeded (via --output) + cylc set -o succeeded $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/foo + # set bar succeeded (via default) + cylc set $CYLC_WORKFLOW_ID//$CYLC_TASK_CYCLE_POINT/bar + """ diff --git a/tests/functional/cylc-set/00-set-succeeded/reference.log b/tests/functional/cylc-set/00-set-succeeded/reference.log new file mode 100644 index 00000000000..26468845a5c --- /dev/null +++ b/tests/functional/cylc-set/00-set-succeeded/reference.log @@ -0,0 +1,5 @@ +1/setter -triggered off [] in flow 1 +1/foo -triggered off [] in flow 1 +1/bar -triggered off [] in flow 1 +1/post_m1 -triggered off ['1/bar', '1/foo'] in flow 1 +1/post_m2 -triggered off ['1/bar', '1/foo'] in flow 1 diff --git a/tests/functional/cylc-set/01-off-flow-pre.t b/tests/functional/cylc-set/01-off-flow-pre.t new file mode 100644 index 00000000000..06a1e864ce3 --- /dev/null +++ b/tests/functional/cylc-set/01-off-flow-pre.t @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# +# "cylc set" proposal examples: 2 - Set off-flow prerequisites to prevent a new flow from stalling. +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#2-set-off-flow-prerequisites-to-prep-for-a-new-flow + +. "$(dirname "$0")/test_header" +set_test_number 8 + +install_and_validate +reftest_run + +grep_workflow_log_ok "${TEST_NAME_BASE}-ab" '1/a does not depend on "1/b_cold:succeeded"' +grep_workflow_log_ok "${TEST_NAME_BASE}-ac" '1/a does not depend on "1/c_cold:succeeded"' + +grep_workflow_log_ok "${TEST_NAME_BASE}-ba" '1/b does not depend on "1/a_cold:succeeded"' +grep_workflow_log_ok "${TEST_NAME_BASE}-bc" '1/b does not depend on "1/c_cold:succeeded"' + +grep_workflow_log_ok "${TEST_NAME_BASE}-ca" '1/c does not depend on "1/a_cold:succeeded"' +grep_workflow_log_ok "${TEST_NAME_BASE}-cb" '1/c does not depend on "1/b_cold:succeeded"' + +purge diff --git a/tests/functional/cylc-set/01-off-flow-pre/flow.cylc b/tests/functional/cylc-set/01-off-flow-pre/flow.cylc new file mode 100644 index 00000000000..5c2ae5b97c4 --- /dev/null +++ b/tests/functional/cylc-set/01-off-flow-pre/flow.cylc @@ -0,0 +1,35 @@ +# start a new flow after setting off-flow prerequites to avoid stall. + +[scheduler] + [[events]] + stall timeout = PT0S + abort on stall timeout = True + inactivity timeout = PT30S + abort on inactivity timeout = True + +[scheduling] + [[graph]] + R1 = """ + # the tasks we want the flow to run + a => b => c => reflow + # the off-flow prerequisites + a_cold => a + b_cold => b + c_cold => c + """ +[runtime] + [[a, b, c]] + [[a_cold, b_cold, c_cold]] + [[reflow]] + script = """ + if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then + # set off-flow prerequisites (and trigger 1/a) + cylc set --flow=new \ + --pre=1/a_cold:succeeded \ + --pre=1/b_cold:succeeded \ + --pre=1/c_cold:succeeded \ + ${CYLC_WORKFLOW_ID}//1/a \ + ${CYLC_WORKFLOW_ID}//1/b \ + ${CYLC_WORKFLOW_ID}//1/c + fi + """ diff --git a/tests/functional/cylc-set/01-off-flow-pre/reference.log b/tests/functional/cylc-set/01-off-flow-pre/reference.log new file mode 100644 index 00000000000..07c980ca981 --- /dev/null +++ b/tests/functional/cylc-set/01-off-flow-pre/reference.log @@ -0,0 +1,11 @@ +1/c_cold -triggered off [] in flow 1 +1/a_cold -triggered off [] in flow 1 +1/b_cold -triggered off [] in flow 1 +1/a -triggered off ['1/a_cold'] in flow 1 +1/b -triggered off ['1/a', '1/b_cold'] in flow 1 +1/c -triggered off ['1/b', '1/c_cold'] in flow 1 +1/reflow -triggered off ['1/c'] in flow 1 +1/a -triggered off ['1/a_cold'] in flow 2 +1/b -triggered off ['1/a', '1/b_cold'] in flow 2 +1/c -triggered off ['1/b', '1/c_cold'] in flow 2 +1/reflow -triggered off ['1/c'] in flow 2 diff --git a/tests/functional/cylc-set/02-off-flow-out.t b/tests/functional/cylc-set/02-off-flow-out.t new file mode 100644 index 00000000000..a18d3e61fbf --- /dev/null +++ b/tests/functional/cylc-set/02-off-flow-out.t @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 2 - Set off-flow outputs to prevent a new flow from stalling. +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#2-set-off-flow-prerequisites-to-prep-for-a-new-flow + +. "$(dirname "$0")/test_header" +set_test_number 11 + +install_and_validate +reftest_run + +# Check that we set: +# - all the required outputs of a_cold +# - the requested and implied outputs of b_cold and c_cold + +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a1" '1/a_cold.* setting implied output: submitted' +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a2" '1/a_cold.* setting implied output: started' +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a3" '1/a_cold.* task completed' + +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a1" '1/b_cold.* setting implied output: submitted' +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a2" '1/b_cold.* setting implied output: started' +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-b3" '1/b_cold.* task completed' + +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a1" '1/c_cold.* setting implied output: submitted' +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-a2" '1/c_cold.* setting implied output: started' +grep_workflow_log_ok "${TEST_NAME_BASE}-grep-c3" '1/c_cold.* task completed' + +purge diff --git a/tests/functional/cylc-set/02-off-flow-out/flow.cylc b/tests/functional/cylc-set/02-off-flow-out/flow.cylc new file mode 100644 index 00000000000..f8f2001f077 --- /dev/null +++ b/tests/functional/cylc-set/02-off-flow-out/flow.cylc @@ -0,0 +1,32 @@ +# start a new flow after setting off-flow outputs to avoid stall. + +[scheduler] + [[events]] + stall timeout = PT0S + abort on stall timeout = True + inactivity timeout = PT30S + abort on inactivity timeout = True + +[scheduling] + [[graph]] + R1 = """ + # the tasks we want the flow to run + a => b => c => reflow + # the off-flow prerequisites + a_cold => a + b_cold => b + c_cold => c + """ +[runtime] + [[a, b, c]] + [[a_cold, b_cold, c_cold]] + [[reflow]] + script = """ + if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then + # set off-flow outputs of x_cold + cylc set --flow=new \ + ${CYLC_WORKFLOW_ID}//1/a_cold \ + ${CYLC_WORKFLOW_ID}//1/b_cold \ + ${CYLC_WORKFLOW_ID}//1/c_cold + fi + """ diff --git a/tests/functional/cylc-set/02-off-flow-out/reference.log b/tests/functional/cylc-set/02-off-flow-out/reference.log new file mode 100644 index 00000000000..07c980ca981 --- /dev/null +++ b/tests/functional/cylc-set/02-off-flow-out/reference.log @@ -0,0 +1,11 @@ +1/c_cold -triggered off [] in flow 1 +1/a_cold -triggered off [] in flow 1 +1/b_cold -triggered off [] in flow 1 +1/a -triggered off ['1/a_cold'] in flow 1 +1/b -triggered off ['1/a', '1/b_cold'] in flow 1 +1/c -triggered off ['1/b', '1/c_cold'] in flow 1 +1/reflow -triggered off ['1/c'] in flow 1 +1/a -triggered off ['1/a_cold'] in flow 2 +1/b -triggered off ['1/a', '1/b_cold'] in flow 2 +1/c -triggered off ['1/b', '1/c_cold'] in flow 2 +1/reflow -triggered off ['1/c'] in flow 2 diff --git a/tests/functional/cylc-set/03-set-failed.t b/tests/functional/cylc-set/03-set-failed.t new file mode 100644 index 00000000000..1910e5b3120 --- /dev/null +++ b/tests/functional/cylc-set/03-set-failed.t @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 4 -check that we can set a dead orphaned job to failed. +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#4-set-jobs-to-failed-when-a-job-platform-is-known-to-be-down + +. "$(dirname "$0")/test_header" +set_test_number 3 + +install_and_validate + +run_ok play-it cylc play --debug "${WORKFLOW_NAME}" + +poll_grep_workflow_log -E "1/foo.* \(internal\)submitted" + +cylc set -o failed "${WORKFLOW_NAME}//1/foo" + + +# Check the log for: +# - set completion message +# - implied outputs reported as already completed + +poll_grep_workflow_log -E "1/foo.* => failed" +poll_grep_workflow_log -E "1/foo.* did not complete required outputs" + +cylc stop --now --now --interval=2 --max-polls=5 "${WORKFLOW_NAME}" + +# Check the DB records all the outputs. +sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ + "SELECT outputs FROM task_outputs WHERE name is \"foo\"" > db-foo.1 + +# Json string list of outputs from the db may not be ordered correctly. +python3 - << __END__ > db-foo.2 +import json +with open("db-foo.1", 'r') as f: + print( + ','.join( + sorted( + json.load(f) + ) + ) + ) +__END__ + +cmp_ok "db-foo.2" - << __OUT__ +failed,started,submitted +__OUT__ + +purge diff --git a/tests/functional/cylc-set/03-set-failed/flow.cylc b/tests/functional/cylc-set/03-set-failed/flow.cylc new file mode 100644 index 00000000000..9d7514ccb83 --- /dev/null +++ b/tests/functional/cylc-set/03-set-failed/flow.cylc @@ -0,0 +1,18 @@ +# A single task that dies silently, requiring set to failed + +[scheduler] + [[events]] + inactivity timeout = PT20S + abort on inactivity timeout = True + +[scheduling] + [[graph]] + R1 = "foo" + +[runtime] + [[foo]] + init-script = cylc__job__disable_fail_signals + script = """ + cylc__job__wait_cylc_message_started + exit 1 + """ diff --git a/tests/functional/cylc-set/04-switch.t b/tests/functional/cylc-set/04-switch.t new file mode 100644 index 00000000000..e5134d32504 --- /dev/null +++ b/tests/functional/cylc-set/04-switch.t @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 5 - Set and complete a future switch task with the "--wait" flag +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#5-set-switch-tasks-at-an-optional-branch-point-to-direct-the-future-flow + +. "$(dirname "$0")/test_header" +set_test_number 5 + +install_and_validate +reftest_run + +# The branch-point task foo should be recorded as succeeded. + +sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ + "SELECT status FROM task_states WHERE name is \"foo\"" > db-foo.2 + +cmp_ok "db-foo.2" - << __OUT__ +succeeded +__OUT__ + +# the outputs of foo should be recorded as: +# a, succeeded +# and the implied outputs (of succeeded) as well: +# submitted, started + +sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ + "SELECT outputs FROM task_outputs WHERE name is \"foo\"" > db-foo.1 + +# Json string list of outputs from the db may not be ordered correctly. +python3 - << __END__ > db-foo.2 +import json +with open("db-foo.1", 'r') as f: + print( + ','.join( + sorted( + json.load(f) + ) + ) + ) +__END__ + +cmp_ok "db-foo.2" - << __OUT__ +a,started,submitted,succeeded +__OUT__ + +# Check the flow-wait worked +grep_workflow_log_ok check-wait "1/foo.* spawning outputs after flow-wait" -E + +purge diff --git a/tests/functional/cylc-set/04-switch/flow.cylc b/tests/functional/cylc-set/04-switch/flow.cylc new file mode 100644 index 00000000000..8a0ded59ce0 --- /dev/null +++ b/tests/functional/cylc-set/04-switch/flow.cylc @@ -0,0 +1,31 @@ +# Set outputs of future task to direct the flow at an optional branch point. + +[scheduler] + [[events]] + inactivity timeout = PT20S + abort on inactivity timeout = True + stall timeout = PT0S + abort on stall timeout = True + +[scheduling] + [[graph]] + R1 = """ + switcher => foo + foo:a? => a + foo:b? => b + """ +[runtime] + [[switcher]] + script = """ + cylc set --output=a,succeeded --wait ${CYLC_WORKFLOW_ID}//1/foo + # wait for command actioned, to avoid race condition + cylc__job__poll_grep_workflow_log "actioned" + """ + [[foo]] + script = "cylc message b" # always go b-way if I run + [[[outputs]]] + a = a + b = b + [[a]] + [[b]] + script = false diff --git a/tests/functional/cylc-set/04-switch/reference.log b/tests/functional/cylc-set/04-switch/reference.log new file mode 100644 index 00000000000..e4fe80d6929 --- /dev/null +++ b/tests/functional/cylc-set/04-switch/reference.log @@ -0,0 +1,2 @@ +1/switcher -triggered off [] in flow 1 +1/a -triggered off ['1/foo'] in flow 1 diff --git a/tests/functional/cylc-set/05-expire.t b/tests/functional/cylc-set/05-expire.t new file mode 100644 index 00000000000..dae60be234e --- /dev/null +++ b/tests/functional/cylc-set/05-expire.t @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 6 - check that forced task expiry works +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#6-expire-a-task + +. "$(dirname "$0")/test_header" +set_test_number 4 + +install_and_validate +reftest_run + +sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ + "SELECT status FROM task_states WHERE name is \"bar\"" > db-bar.1 + +cmp_ok "db-bar.1" - << __OUT__ +expired +__OUT__ + +sqlite3 ~/cylc-run/"${WORKFLOW_NAME}"/log/db \ + "SELECT outputs FROM task_outputs WHERE name is \"bar\"" > db-bar.2 + +cmp_ok "db-bar.2" - << __OUT__ +["expired"] +__OUT__ + +purge diff --git a/tests/functional/cylc-set/05-expire/flow.cylc b/tests/functional/cylc-set/05-expire/flow.cylc new file mode 100644 index 00000000000..57d94dbb99e --- /dev/null +++ b/tests/functional/cylc-set/05-expire/flow.cylc @@ -0,0 +1,24 @@ +# Expire a future task, so it won't run. + +[scheduler] + [[events]] + inactivity timeout = PT20S + abort on inactivity timeout = True + stall timeout = PT0S + abort on stall timeout = True + +[scheduling] + [[graph]] + R1 = """ + # bar and baz should not run if bar expires + expirer => foo => bar? => baz + bar:expired? + """ +[runtime] + [[expirer]] + script = """ + cylc set --output=expired ${CYLC_WORKFLOW_ID}//1/bar + """ + [[foo]] + [[bar, baz]] + script = false diff --git a/tests/functional/cylc-set/05-expire/reference.log b/tests/functional/cylc-set/05-expire/reference.log new file mode 100644 index 00000000000..0966b1b6f90 --- /dev/null +++ b/tests/functional/cylc-set/05-expire/reference.log @@ -0,0 +1,2 @@ +1/expirer -triggered off [] in flow 1 +1/foo -triggered off ['1/expirer'] in flow 1 diff --git a/tests/functional/cylc-set/06-parentless.t b/tests/functional/cylc-set/06-parentless.t new file mode 100644 index 00000000000..9ccf16e30e0 --- /dev/null +++ b/tests/functional/cylc-set/06-parentless.t @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 7 - Check spawning a parentless task without ignoring xtriggers. +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#7-spawning-parentless-tasks + +. "$(dirname "$0")/test_header" +set_test_number 3 + +install_and_validate +REFTEST_OPTS="--start-task=1800/a" reftest_run + +grep_workflow_log_ok "${TEST_NAME_BASE}-clock" "xtrigger satisfied: wall_clock" + +purge diff --git a/tests/functional/cylc-set/06-parentless/flow.cylc b/tests/functional/cylc-set/06-parentless/flow.cylc new file mode 100644 index 00000000000..5078b84e484 --- /dev/null +++ b/tests/functional/cylc-set/06-parentless/flow.cylc @@ -0,0 +1,22 @@ +# Start this with --start-task=1800/a. +# It should stall because x => b is off-flow. +# The stall handler should unstall it by spawning x. +# The log should show a clock-trigger check before x runs. + +[scheduler] + [[events]] + inactivity timeout = PT30S + abort on inactivity timeout = True + stall timeout = PT10S + abort on stall timeout = True + stall handlers = "cylc set --pre=all %(workflow)s//1800/x" + +[scheduling] + initial cycle point = 1800 + [[graph]] + R1 = """ + a => b + @wall_clock => x => b + """ +[runtime] + [[a, b, x]] diff --git a/tests/functional/cylc-set/06-parentless/reference.log b/tests/functional/cylc-set/06-parentless/reference.log new file mode 100644 index 00000000000..f977d1f086b --- /dev/null +++ b/tests/functional/cylc-set/06-parentless/reference.log @@ -0,0 +1,4 @@ +Start task: ['1800/a'] +18000101T0000Z/a -triggered off [] in flow 1 +18000101T0000Z/x -triggered off [] in flow 1 +18000101T0000Z/b -triggered off ['18000101T0000Z/a', '18000101T0000Z/x'] in flow 1 diff --git a/tests/functional/cylc-set/08-switch2.t b/tests/functional/cylc-set/08-switch2.t new file mode 100644 index 00000000000..3f947668501 --- /dev/null +++ b/tests/functional/cylc-set/08-switch2.t @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- + +# "cylc set" proposal examples: 5 - Set and complete a future switch task. +# https://cylc.github.io/cylc-admin/proposal-cylc-set.html#5-set-switch-tasks-at-an-optional-branch-point-to-direct-the-future-flow + +. "$(dirname "$0")/test_header" +set_test_number 2 + +reftest + +purge diff --git a/tests/functional/cylc-set/08-switch2/flow.cylc b/tests/functional/cylc-set/08-switch2/flow.cylc new file mode 100644 index 00000000000..0a221f1227e --- /dev/null +++ b/tests/functional/cylc-set/08-switch2/flow.cylc @@ -0,0 +1,41 @@ + +# Complete a parentless switch task that already exists in the pool but is +# beyond the runahead limit. Cylc should auto-spawn its next instance to +# avoid premature shutdown when it is removed as complete. +# (We only spawn the first runahead-limited instance of parentless tasks). + +[scheduler] + allow implicit tasks = True + +[scheduling] + initial cycle point = 1 + final cycle point = 4 + cycling mode = integer + runahead limit = P0 + [[graph]] + P1 = """ + a:x? => x + a:y? => y + x | y => z + """ +[runtime] + [[a]] + script = """ + cylc__job__wait_cylc_message_started + cylc message -- x # always go x-path + """ + [[[outputs]]] + x = x + y = y + [[z]] + script = """ + if (( CYLC_TASK_CYCLE_POINT == 1 )); then + # mark 2/a as succeeded with output y + # (task will be skipped) + cylc set "${CYLC_WORKFLOW_ID}//2/a" --out=y,succeeded + elif (( CYLC_TASK_CYCLE_POINT == 2 )); then + # mark 2/a as having generated output y + # (task will re-run and generate output x in the prociess) + cylc set "${CYLC_WORKFLOW_ID}//3/a" --out=y + fi + """ diff --git a/tests/functional/cylc-set/08-switch2/reference.log b/tests/functional/cylc-set/08-switch2/reference.log new file mode 100644 index 00000000000..a41fff43a9e --- /dev/null +++ b/tests/functional/cylc-set/08-switch2/reference.log @@ -0,0 +1,17 @@ +# 1/a runs naturally and generates the output "x" +1/a -triggered off [] in flow 1 +1/x -triggered off ['1/a'] in flow 1 +1/z -triggered off ['1/x'] in flow 1 +# 1/a is artificially completed with the output "y" +2/y -triggered off ['2/a'] in flow 1 +2/z -triggered off ['2/y'] in flow 1 +# 1/a has the output "y" is artificially set but is not completed +# (so 1/a will re-run and generate the output "x" naturally) +3/a -triggered off [] in flow 1 +3/x -triggered off ['3/a'] in flow 1 +3/y -triggered off ['3/a'] in flow 1 +3/z -triggered off ['3/y'] in flow 1 +# 1/a runs naturally and generates the output "x" +4/a -triggered off [] in flow 1 +4/x -triggered off ['4/a'] in flow 1 +4/z -triggered off ['4/x'] in flow 1 diff --git a/tests/functional/cylc-set/test_header b/tests/functional/cylc-set/test_header new file mode 120000 index 00000000000..90bd5a36f92 --- /dev/null +++ b/tests/functional/cylc-set/test_header @@ -0,0 +1 @@ +../lib/bash/test_header \ No newline at end of file diff --git a/tests/functional/cylc-trigger/02-filter-failed/flow.cylc b/tests/functional/cylc-trigger/02-filter-failed/flow.cylc index e5a3065782e..7416cf5790d 100644 --- a/tests/functional/cylc-trigger/02-filter-failed/flow.cylc +++ b/tests/functional/cylc-trigger/02-filter-failed/flow.cylc @@ -18,9 +18,9 @@ [[fixer]] script = """ cylc__job__wait_cylc_message_started - cylc__job__poll_grep_workflow_log -E '1/fixable1 running .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '1/fixable2 running .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '1/fixable3 running .* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable1/01:running.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable2/01:running.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable3/01:running.* \(received\)failed' cylc trigger "${CYLC_WORKFLOW_ID}//1/fixable*" """ [[Z]] diff --git a/tests/functional/cylc-trigger/04-filter-names/flow.cylc b/tests/functional/cylc-trigger/04-filter-names/flow.cylc index 5997dcaa201..31839c1b77f 100644 --- a/tests/functional/cylc-trigger/04-filter-names/flow.cylc +++ b/tests/functional/cylc-trigger/04-filter-names/flow.cylc @@ -22,11 +22,11 @@ [[fixer]] script = """ cylc__job__wait_cylc_message_started - cylc__job__poll_grep_workflow_log -E '1/fixable-1a .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '1/fixable-1b .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '1/fixable-2a .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '1/fixable-2b .* \(received\)failed' - cylc__job__poll_grep_workflow_log -E '1/fixable-3 .* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable-1a/01.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable-1b/01.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable-2a/01.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable-2b/01.* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/fixable-3/01.* \(received\)failed' cylc trigger "${CYLC_WORKFLOW_ID}//" \ '//1/FIXABLE-1' '//1/fixable-2*' '//1/fixable-3' """ diff --git a/tests/functional/cylc-trigger/06-already-active/flow.cylc b/tests/functional/cylc-trigger/06-already-active/flow.cylc index b939aa2d290..c7d99f6a6a8 100644 --- a/tests/functional/cylc-trigger/06-already-active/flow.cylc +++ b/tests/functional/cylc-trigger/06-already-active/flow.cylc @@ -9,14 +9,14 @@ [runtime] [[triggerer]] script = """ - cylc__job__poll_grep_workflow_log "1/triggeree .* running" -E + cylc__job__poll_grep_workflow_log "1/triggeree/01:running" cylc trigger "$CYLC_WORKFLOW_ID//1/triggeree" cylc__job__poll_grep_workflow_log \ - "1/triggeree .* ignoring trigger - already active" -E + "1/triggeree.* ignoring trigger - already active" -E """ [[triggeree]] script = """ cylc__job__poll_grep_workflow_log \ - "1/triggeree .* ignoring trigger - already active" -E + "1/triggeree.* ignoring trigger - already active" -E """ diff --git a/tests/functional/events/23-workflow-stalled-handler/flow.cylc b/tests/functional/events/23-workflow-stalled-handler/flow.cylc index 5981611b409..4d3673daafa 100644 --- a/tests/functional/events/23-workflow-stalled-handler/flow.cylc +++ b/tests/functional/events/23-workflow-stalled-handler/flow.cylc @@ -1,6 +1,6 @@ [scheduler] [[events]] - stall handlers = "cylc set-outputs --flow=1 %(workflow)s//1/bar" + stall handlers = "cylc set %(workflow)s//1/bar" stall timeout = PT0S abort on stall timeout = False expected task failures = 1/bar diff --git a/tests/functional/events/38-task-event-handler-custom.t b/tests/functional/events/38-task-event-handler-custom.t index 92d1ca9722e..d1f825e1c16 100755 --- a/tests/functional/events/38-task-event-handler-custom.t +++ b/tests/functional/events/38-task-event-handler-custom.t @@ -28,7 +28,7 @@ WORKFLOW_LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok \ "\[(('event-handler-00', 'custom-1'), 1) out\] !!CUSTOM!! 1/foo fugu Data ready for barring" \ "${FOO_ACTIVITY_LOG}" -grep_ok "1/foo .*Data ready for barring" "${WORKFLOW_LOG}" -E -grep_ok "1/foo .*Data ready for bazzing" "${WORKFLOW_LOG}" -E -grep_ok "1/foo .*Aren't the hydrangeas nice" "${WORKFLOW_LOG}" -E +grep_ok "1/foo.*Data ready for barring" "${WORKFLOW_LOG}" -E +grep_ok "1/foo.*Data ready for bazzing" "${WORKFLOW_LOG}" -E +grep_ok "1/foo.*Aren't the hydrangeas nice" "${WORKFLOW_LOG}" -E purge diff --git a/tests/functional/execution-time-limit/04-polling-intervals.t b/tests/functional/execution-time-limit/04-polling-intervals.t index 4b213c70546..e1df403f155 100644 --- a/tests/functional/execution-time-limit/04-polling-intervals.t +++ b/tests/functional/execution-time-limit/04-polling-intervals.t @@ -47,13 +47,13 @@ poll_grep_workflow_log "INFO - DONE" # NOTE: execution timeout polling is delayed by PT1M to let things settle # PT10M = (3*PT3S + PT9M30S) - PT1M -grep_workflow_log_ok grep-limit10M "\[1/limit10M running job:01 flows:1\] health: execution timeout=None, polling intervals=3\*PT30S,PT9M30S,PT2M,PT7M,..." +grep_workflow_log_ok grep-limit10M "\[1/limit10M/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT9M30S,PT2M,PT7M,..." # PT60M = (3*PT3S + PT10M + PT49M30S) - PT1M -grep_workflow_log_ok grep-limit1H "\[1/limit1H running job:01 flows:1\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT49M30S,PT2M,PT7M,..." +grep_workflow_log_ok grep-limit1H "\[1/limit1H/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT49M30S,PT2M,PT7M,..." # PT70S = (2*PT30S + PT1M10S) - PT1M -grep_workflow_log_ok grep-limit70S "\[1/limit70S running job:01 flows:1\] health: execution timeout=None, polling intervals=2\*PT30S,PT1M10S,PT2M,PT7M,..." +grep_workflow_log_ok grep-limit70S "\[1/limit70S/01:running\] health: execution timeout=None, polling intervals=2\*PT30S,PT1M10S,PT2M,PT7M,..." # PT95M = (3*PT3S + PT10M + PT1H + PT24M30S) - PT1M -grep_workflow_log_ok grep-limit95M "\[1/limit95M running job:01 flows:1\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT1H,PT24M30S,PT2M,PT7M,..." -grep_workflow_log_ok grep-no-limit "\[1/nolimit running job:01 flows:1\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT1H,..." +grep_workflow_log_ok grep-limit95M "\[1/limit95M/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT1H,PT24M30S,PT2M,PT7M,..." +grep_workflow_log_ok grep-no-limit "\[1/nolimit/01:running\] health: execution timeout=None, polling intervals=3\*PT30S,PT10M,PT1H,..." purge diff --git a/tests/functional/ext-trigger/01-no-nudge/flow.cylc b/tests/functional/ext-trigger/01-no-nudge/flow.cylc index f5569601950..1bf744e8ac6 100644 --- a/tests/functional/ext-trigger/01-no-nudge/flow.cylc +++ b/tests/functional/ext-trigger/01-no-nudge/flow.cylc @@ -31,7 +31,7 @@ [[foo]] script = """ cylc kill "$CYLC_WORKFLOW_ID//1/bar" - cylc__job__poll_grep_workflow_log -E '1/bar .* \(internal\)failed' + cylc__job__poll_grep_workflow_log -E '1/bar.* \(internal\)failed' cylc release "$CYLC_WORKFLOW_ID//1/bar" """ [[bar]] diff --git a/tests/functional/flow-triggers/00-new-future/flow.cylc b/tests/functional/flow-triggers/00-new-future/flow.cylc index c67fd0fdd19..94add5a60d0 100644 --- a/tests/functional/flow-triggers/00-new-future/flow.cylc +++ b/tests/functional/flow-triggers/00-new-future/flow.cylc @@ -34,5 +34,5 @@ [[a]] script = """ cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/d - cylc__job__poll_grep_workflow_log -E '1/d.*started' + cylc__job__poll_grep_workflow_log -E '1/d/01.*started' """ diff --git a/tests/functional/flow-triggers/01-all-future.t b/tests/functional/flow-triggers/01-all-future.t index 8ecc1448e74..27a80f9892c 100644 --- a/tests/functional/flow-triggers/01-all-future.t +++ b/tests/functional/flow-triggers/01-all-future.t @@ -41,11 +41,12 @@ install_workflow "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" true reftest_run TEST_NAME="${TEST_NAME_BASE}-order-wait" +# Note flow_wait is updated to False once used. QUERY="SELECT name,flow_nums,flow_wait FROM task_states ORDER BY time_created" run_ok "${TEST_NAME}" sqlite3 "${DB}" "$QUERY" cmp_ok "${TEST_NAME}.stdout" <<\__END__ a|[1]|0 -d|[1]|1 +d|[1]|0 b|[1]|0 c|[1]|0 e|[1]|0 diff --git a/tests/functional/flow-triggers/01-all-future/flow.cylc b/tests/functional/flow-triggers/01-all-future/flow.cylc index a47ce127429..d7a9b0284d9 100644 --- a/tests/functional/flow-triggers/01-all-future/flow.cylc +++ b/tests/functional/flow-triggers/01-all-future/flow.cylc @@ -39,7 +39,5 @@ [[a]] script = """ cylc trigger {{OPT}} ${CYLC_WORKFLOW_ID}//1/d - {% if WAIT is not defined %} cylc__job__poll_grep_workflow_log -E '1/d.*succeeded' - {% endif %} """ diff --git a/tests/functional/flow-triggers/03-new-past/flow.cylc b/tests/functional/flow-triggers/03-new-past/flow.cylc index 9c77123d40c..980599c270b 100644 --- a/tests/functional/flow-triggers/03-new-past/flow.cylc +++ b/tests/functional/flow-triggers/03-new-past/flow.cylc @@ -37,6 +37,6 @@ script = """ if (( $CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/a - cylc__job__poll_grep_workflow_log -E '1/a submitted job:02 .*started' + cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=2\):submitted.*started' fi """ diff --git a/tests/functional/flow-triggers/04-all-past/flow.cylc b/tests/functional/flow-triggers/04-all-past/flow.cylc index 85721f2b5e5..11b6ac7e5d9 100644 --- a/tests/functional/flow-triggers/04-all-past/flow.cylc +++ b/tests/functional/flow-triggers/04-all-past/flow.cylc @@ -41,6 +41,6 @@ script = """ if (( $CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger {{OPT}} ${CYLC_WORKFLOW_ID}//1/a - cylc__job__poll_grep_workflow_log -E '1/a running job:02 .*succeeded' + cylc__job__poll_grep_workflow_log -E '1/a/02:running.*succeeded' fi """ diff --git a/tests/functional/flow-triggers/05-none-past/flow.cylc b/tests/functional/flow-triggers/05-none-past/flow.cylc index 6709ebf0d3c..f59e5e11178 100644 --- a/tests/functional/flow-triggers/05-none-past/flow.cylc +++ b/tests/functional/flow-triggers/05-none-past/flow.cylc @@ -34,6 +34,6 @@ script = """ if (( $CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger --flow=none ${CYLC_WORKFLOW_ID}//1/a - cylc__job__poll_grep_workflow_log -E '1/a submitted job:02 .*started' + cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=none\):submitted.*started' fi """ diff --git a/tests/functional/flow-triggers/06-new-past-switch/flow.cylc b/tests/functional/flow-triggers/06-new-past-switch/flow.cylc index b33855b74ca..e185d763fb6 100644 --- a/tests/functional/flow-triggers/06-new-past-switch/flow.cylc +++ b/tests/functional/flow-triggers/06-new-past-switch/flow.cylc @@ -55,5 +55,5 @@ [[c]] script = """ cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/a - cylc__job__poll_grep_workflow_log -E '1/a submitted job:02 .*started' + cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=2\):submitted.*started' """ diff --git a/tests/functional/flow-triggers/07-all-past-switch/flow.cylc b/tests/functional/flow-triggers/07-all-past-switch/flow.cylc index 87ea0445a2d..4965bc77886 100644 --- a/tests/functional/flow-triggers/07-all-past-switch/flow.cylc +++ b/tests/functional/flow-triggers/07-all-past-switch/flow.cylc @@ -64,6 +64,6 @@ script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger {{OPT}} ${CYLC_WORKFLOW_ID}//1/a - cylc__job__poll_grep_workflow_log -E '1/a running job:02 .*succeeded' + cylc__job__poll_grep_workflow_log -E '1/a/02:running.*succeeded' fi """ diff --git a/tests/functional/flow-triggers/08-none-past-switch/flow.cylc b/tests/functional/flow-triggers/08-none-past-switch/flow.cylc index 419bf72d67a..63a4e532a68 100644 --- a/tests/functional/flow-triggers/08-none-past-switch/flow.cylc +++ b/tests/functional/flow-triggers/08-none-past-switch/flow.cylc @@ -53,5 +53,5 @@ [[c]] script = """ cylc trigger --flow=none ${CYLC_WORKFLOW_ID}//1/a - cylc__job__poll_grep_workflow_log -E '1/a submitted job:02 .*started' + cylc__job__poll_grep_workflow_log -E '1/a/02\(flows=none\):submitted.*started' """ diff --git a/tests/functional/flow-triggers/09-retrigger/flow.cylc b/tests/functional/flow-triggers/09-retrigger/flow.cylc index a8bdb524870..a0e1341c06e 100644 --- a/tests/functional/flow-triggers/09-retrigger/flow.cylc +++ b/tests/functional/flow-triggers/09-retrigger/flow.cylc @@ -1,7 +1,5 @@ -# Check if a task gets triggered multiple times with --wait, only the outputs -# from the last run in the flow are spawned when the flow catches up. - -# baz should run twice in flow 1, then y should trigger off of the second baz. +# If a task gets triggered twice with --wait, the outputs from both runs should be +# spawned when the flow catches up. [scheduling] [[graph]] @@ -15,9 +13,9 @@ [[foo]] script = """ cylc trigger --wait ${CYLC_WORKFLOW_ID}//1/baz - cylc__job__poll_grep_workflow_log "1/baz running job:01 .* succeeded" + cylc__job__poll_grep_workflow_log "1/baz/01:running.*succeeded" cylc trigger --wait ${CYLC_WORKFLOW_ID}//1/baz - cylc__job__poll_grep_workflow_log "1/baz running job:02 .* succeeded" + cylc__job__poll_grep_workflow_log "1/baz/02:running.*succeeded" """ [[baz]] script = """ diff --git a/tests/functional/flow-triggers/09-retrigger/reference.log b/tests/functional/flow-triggers/09-retrigger/reference.log index 2323320048b..00afaf9c6fe 100644 --- a/tests/functional/flow-triggers/09-retrigger/reference.log +++ b/tests/functional/flow-triggers/09-retrigger/reference.log @@ -4,4 +4,5 @@ Final point: 1 1/baz -triggered off [] in flow 1 1/baz -triggered off [] in flow 1 1/bar -triggered off ['1/foo'] in flow 1 +1/x -triggered off ['1/baz'] in flow 1 1/y -triggered off ['1/baz'] in flow 1 diff --git a/tests/functional/flow-triggers/10-specific-flow/flow.cylc b/tests/functional/flow-triggers/10-specific-flow/flow.cylc index 3d648ddcac4..46ba6dab4c1 100644 --- a/tests/functional/flow-triggers/10-specific-flow/flow.cylc +++ b/tests/functional/flow-triggers/10-specific-flow/flow.cylc @@ -1,7 +1,7 @@ # Check targetting a specific flow works, with trigger --wait. # At start-up, trigger f with --wait for flow 2 -# Then when flow 1 reaches d, trigger a new flow (2) at b. +# Then when flow 1 reaches d, trigger flow 2 at b. # Flow 1 should overrun the manually triggered f. # Flow 2 should skip over it to g. @@ -17,6 +17,6 @@ [[trigger-happy]] script = """ cylc trigger --flow=2 --wait ${CYLC_WORKFLOW_ID}//1/f - cylc__job__poll_grep_workflow_log "1/d submitted job:01 .*started" - cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/b + cylc__job__poll_grep_workflow_log "1/d/01:submitted.*running" + cylc trigger --flow=2 ${CYLC_WORKFLOW_ID}//1/b """ diff --git a/tests/functional/flow-triggers/11-wait-merge/flow.cylc b/tests/functional/flow-triggers/11-wait-merge/flow.cylc index 9e398c7bf74..72d77a3ce4f 100644 --- a/tests/functional/flow-triggers/11-wait-merge/flow.cylc +++ b/tests/functional/flow-triggers/11-wait-merge/flow.cylc @@ -16,7 +16,7 @@ [[a]] script = """ if ((CYLC_TASK_SUBMIT_NUMBER == 2)); then - cylc__job__poll_grep_workflow_log "1/d .*(received)started" + cylc__job__poll_grep_workflow_log "\[1/d/01(flows=1,2):submitted] (received)started" fi """ [[b]] @@ -24,11 +24,12 @@ if ((CYLC_TASK_SUBMIT_NUMBER == 1)); then cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//1/a cylc trigger --flow=2 --wait ${CYLC_WORKFLOW_ID}//1/c - cylc__job__poll_grep_workflow_log "1/c .*(received)x" + cylc__job__poll_grep_workflow_log "\[1/c/01(flows=2):running] (received)x" fi """ [[c]] script = """ + cylc__job__wait_cylc_message_started cylc message x if ((CYLC_TASK_SUBMIT_NUMBER == 1)); then cylc__job__poll_grep_workflow_log "merged" diff --git a/tests/functional/flow-triggers/12-all-future-multi/flow.cylc b/tests/functional/flow-triggers/12-all-future-multi/flow.cylc index 782690f24a5..e67aff4f20e 100644 --- a/tests/functional/flow-triggers/12-all-future-multi/flow.cylc +++ b/tests/functional/flow-triggers/12-all-future-multi/flow.cylc @@ -45,9 +45,9 @@ )); then # trigger 3/a in a new flow cylc trigger --flow=new ${CYLC_WORKFLOW_ID}//3/a - cylc__job__poll_grep_workflow_log -E '3/a.*started' + cylc__job__poll_grep_workflow_log -E '3/a.*=> running' # trigger 5/a in all flows cylc trigger ${CYLC_WORKFLOW_ID}//5/a - cylc__job__poll_grep_workflow_log -E '5/a.*started' + cylc__job__poll_grep_workflow_log -E '5/a.*=> running' fi """ diff --git a/tests/functional/flow-triggers/13-noflow-nomerge.t b/tests/functional/flow-triggers/13-noflow-nomerge.t index c8b4528a2f9..8dfa08f994c 100644 --- a/tests/functional/flow-triggers/13-noflow-nomerge.t +++ b/tests/functional/flow-triggers/13-noflow-nomerge.t @@ -27,7 +27,7 @@ run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" poll_grep_workflow_log "Workflow stalled" run_ok "${TEST_NAME_BASE}-trigger" cylc trigger --flow=none "${WORKFLOW_NAME}//1/a" -poll_grep_workflow_log -E "1/a running job:02 flows:none.*=> succeeded" +poll_grep_workflow_log -E "1/a/02\(flows=none\):running.*=> succeeded" cylc stop --now --now --max-polls=5 --interval=2 "$WORKFLOW_NAME" diff --git a/tests/functional/hold-release/00-workflow/flow.cylc b/tests/functional/hold-release/00-workflow/flow.cylc index 4afdbc92980..a0df872ebab 100644 --- a/tests/functional/hold-release/00-workflow/flow.cylc +++ b/tests/functional/hold-release/00-workflow/flow.cylc @@ -23,7 +23,7 @@ script = """ cylc__job__wait_cylc_message_started cylc hold --after=1900 "${CYLC_WORKFLOW_ID}" - cylc__job__poll_grep_workflow_log -F 'INFO - Command actioned: set_hold_point' + cylc__job__poll_grep_workflow_log 'Command "set_hold_point" actioned' cylc release --all "${CYLC_WORKFLOW_ID}" """ [[foo,bar]] diff --git a/tests/functional/hold-release/02-hold-on-spawn.t b/tests/functional/hold-release/02-hold-on-spawn.t index 9efd2e18ee8..dad99d32297 100755 --- a/tests/functional/hold-release/02-hold-on-spawn.t +++ b/tests/functional/hold-release/02-hold-on-spawn.t @@ -34,7 +34,7 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc release "${WORKFLOW_NAME}//1/foo" # 1/foo should run and spawn 1/bar as waiting and held -poll_grep_workflow_log -E '1/bar .* spawned' +poll_grep_workflow_log -E '1/bar.* added to active task pool' sqlite3 "${WORKFLOW_RUN_DIR}/log/db" \ 'SELECT cycle, name, status, is_held FROM task_pool' > task-pool.out diff --git a/tests/functional/hold-release/05-release.t b/tests/functional/hold-release/05-release.t index 805fe2395cc..26f4e22d414 100755 --- a/tests/functional/hold-release/05-release.t +++ b/tests/functional/hold-release/05-release.t @@ -34,7 +34,7 @@ init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' script = """ cylc__job__wait_cylc_message_started cylc hold --after=0 ${CYLC_WORKFLOW_ID} - cylc__job__poll_grep_workflow_log 'Command actioned: set_hold_point' + cylc__job__poll_grep_workflow_log 'Command "set_hold_point" actioned' cylc release "${CYLC_WORKFLOW_ID}//1/*FF" # inexact fam cylc release "${CYLC_WORKFLOW_ID}//1/TOAST" # exact fam cylc release "${CYLC_WORKFLOW_ID}//1/cat*" # inexact tasks @@ -65,7 +65,7 @@ init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' inherit = STOP script = """ cylc__job__poll_grep_workflow_log -E \ - '1/dog1 succeeded .* task proxy removed \(finished\)' + '1/dog1/01:succeeded.* task completed' cylc stop "${CYLC_WORKFLOW_ID}" """ __FLOW_CONFIG__ diff --git a/tests/functional/hold-release/08-hold.t b/tests/functional/hold-release/08-hold.t index 206abc1efdb..27d6020f7aa 100755 --- a/tests/functional/hold-release/08-hold.t +++ b/tests/functional/hold-release/08-hold.t @@ -33,14 +33,14 @@ init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [[holdrelease]] script = """ cylc__job__wait_cylc_message_started - cylc__job__poll_grep_workflow_log -E '1/foo .* spawned' - cylc__job__poll_grep_workflow_log -E '1/bar .* spawned' - cylc__job__poll_grep_workflow_log -E '1/cheese .* spawned' - cylc__job__poll_grep_workflow_log -E '1/jam .* spawned' - cylc__job__poll_grep_workflow_log -E '1/cat1 .* spawned' - cylc__job__poll_grep_workflow_log -E '1/cat2 .* spawned' - cylc__job__poll_grep_workflow_log -E '1/dog1 .* spawned' - cylc__job__poll_grep_workflow_log -E '1/dog2 .* spawned' + cylc__job__poll_grep_workflow_log -E '1/foo.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/bar.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/cheese.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/jam.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/cat1.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/cat2.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/dog1.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '1/dog2.* added to active task pool' cylc hold "${CYLC_WORKFLOW_ID}//1/*FF" # inexact fam cylc hold "${CYLC_WORKFLOW_ID}//1/TOAST" # exact fam cylc hold "${CYLC_WORKFLOW_ID}//1/cat*" # inexact tasks diff --git a/tests/functional/hold-release/11-retrying/flow.cylc b/tests/functional/hold-release/11-retrying/flow.cylc index 03cd0f6b039..0e08699af09 100644 --- a/tests/functional/hold-release/11-retrying/flow.cylc +++ b/tests/functional/hold-release/11-retrying/flow.cylc @@ -18,26 +18,26 @@ t-retry-able => t-analyse [[t-hold-release]] script = """ cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able running job:01.* \(received\)failed' + '1/t-retry-able/01:running.* \(received\)failed' cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able running job:01.* => waiting' + '1/t-retry-able/01:running.* => waiting' cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able waiting job:01.* retrying in PT15S' + '1/t-retry-able:waiting.* retrying in PT15S' cylc hold "${CYLC_WORKFLOW_ID}//1/t-retry-able" cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able waiting job:01.* => waiting\(held\)' + '1/t-retry-able:waiting.* => waiting\(held\)' cylc release "${CYLC_WORKFLOW_ID}//1/t-retry-able" cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able waiting\(held\) job:01.* => waiting' + '1/t-retry-able:waiting\(held\).* => waiting' cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able waiting job:01.* => waiting\(queued\)' + '1/t-retry-able:waiting.* => waiting\(queued\)' """ [[t-analyse]] script = """ diff --git a/tests/functional/hold-release/18-hold-cycle-globs/flow.cylc b/tests/functional/hold-release/18-hold-cycle-globs/flow.cylc index a74c7a20c3f..aa946b9b418 100644 --- a/tests/functional/hold-release/18-hold-cycle-globs/flow.cylc +++ b/tests/functional/hold-release/18-hold-cycle-globs/flow.cylc @@ -23,9 +23,9 @@ [runtime] [[holder]] script = """ - cylc__job__poll_grep_workflow_log -E '19900101T0000Z/t1 .* spawned' - cylc__job__poll_grep_workflow_log -E '20100101T0000Z/t2 .* spawned' - cylc__job__poll_grep_workflow_log -E '20300101T0000Z/t3 .* spawned' + cylc__job__poll_grep_workflow_log -E '19900101T0000Z/t1.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '20100101T0000Z/t2.* added to active task pool' + cylc__job__poll_grep_workflow_log -E '20300101T0000Z/t3.* added to active task pool' cylc hold "${CYLC_WORKFLOW_ID}//*/t*" """ [[releaser]] diff --git a/tests/functional/hold-release/19-no-reset-prereq-on-waiting/flow.cylc b/tests/functional/hold-release/19-no-reset-prereq-on-waiting/flow.cylc index 9ad270b4e84..edbd80b5770 100644 --- a/tests/functional/hold-release/19-no-reset-prereq-on-waiting/flow.cylc +++ b/tests/functional/hold-release/19-no-reset-prereq-on-waiting/flow.cylc @@ -16,7 +16,7 @@ script = true [[holder]] script = """ - cylc__job__poll_grep_workflow_log -E '1/t1 .* spawned' + cylc__job__poll_grep_workflow_log -E '1/t1.* added to active task pool' cylc hold "${CYLC_WORKFLOW_ID}//1/t1" """ [[releaser]] diff --git a/tests/functional/intelligent-host-selection/02-badhosts.t b/tests/functional/intelligent-host-selection/02-badhosts.t index 0689866d22f..2d12c62f4c9 100644 --- a/tests/functional/intelligent-host-selection/02-badhosts.t +++ b/tests/functional/intelligent-host-selection/02-badhosts.t @@ -68,11 +68,11 @@ LOGFILE="${WORKFLOW_RUN_DIR}/log/scheduler/log" # Check that badhosttask has submit failed, but not good or mixed named_grep_ok "badhost task submit failed" \ - "1/badhosttask .* submit-failed" "${LOGFILE}" + "1/badhosttask.* submit-failed" "${LOGFILE}" named_grep_ok "goodhost suceeded" \ - "1/mixedhosttask .* succeeded" "${LOGFILE}" + "1/mixedhosttask.* succeeded" "${LOGFILE}" named_grep_ok "mixedhost task suceeded" \ - "1/goodhosttask .* succeeded" "${LOGFILE}" + "1/goodhosttask.* succeeded" "${LOGFILE}" # Check that when a task fail badhosts associated with that task's platform # are removed from the badhosts set. diff --git a/tests/functional/intelligent-host-selection/05-from-platform-group.t b/tests/functional/intelligent-host-selection/05-from-platform-group.t index 527cfaeba43..801372922fb 100644 --- a/tests/functional/intelligent-host-selection/05-from-platform-group.t +++ b/tests/functional/intelligent-host-selection/05-from-platform-group.t @@ -83,7 +83,7 @@ log_scan \ "platform: ${CYLC_TEST_PLATFORM} - Could not connect to unreachable_host." \ "platform: ${CYLC_TEST_PLATFORM} - remote init (on ${CYLC_TEST_HOST})" \ "platform: ${CYLC_TEST_PLATFORM} - remote file install (on ${CYLC_TEST_HOST})" \ - "\[1/ugly preparing job:01 flows:1\] => submitted" + "\[1/ugly/01:preparing\] => submitted" purge exit 0 diff --git a/tests/functional/job-submission/01-job-nn-localhost/db.sqlite3 b/tests/functional/job-submission/01-job-nn-localhost/db.sqlite3 index d3dcf24f339..0d3598e5cb9 100644 --- a/tests/functional/job-submission/01-job-nn-localhost/db.sqlite3 +++ b/tests/functional/job-submission/01-job-nn-localhost/db.sqlite3 @@ -8,6 +8,7 @@ INSERT INTO inheritance VALUES('foo','["foo", "root"]'); CREATE TABLE workflow_params(key TEXT, value TEXT, PRIMARY KEY(key)); INSERT INTO workflow_params VALUES('cylc_version', '8.0.0'); INSERT INTO workflow_params VALUES('uuid_str', 'Something'); +INSERT INTO workflow_params VALUES('UTC_mode', 1); CREATE TABLE workflow_template_vars(key TEXT, value TEXT, PRIMARY KEY(key)); CREATE TABLE task_action_timers(cycle TEXT, name TEXT, ctx_key TEXT, ctx TEXT, delays TEXT, num INTEGER, delay TEXT, timeout TEXT, PRIMARY KEY(cycle, name, ctx_key)); INSERT INTO task_action_timers VALUES('1','foo','"poll_timer"','["tuple", [[99, "running"]]]','[]',0,NULL,NULL); diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header index c4a07603126..915821c0889 100644 --- a/tests/functional/lib/bash/test_header +++ b/tests/functional/lib/bash/test_header @@ -90,6 +90,7 @@ # tries grepping for each PATTERN in turn. Tests will only pass if the # PATTERNs appear in FILE in the correct order. Runs one test per # pattern, each prefixed by TEST_NAME. +# set LOG_SCAN_GREP_OPTS in the environment, e.g. "-E" for "grep -E" # make_rnd_workflow # Create a randomly-named workflow source directory. # mock_smtpd_init @@ -579,6 +580,7 @@ log_scan () { local FILE="$2" local REPS=$3 local DELAY=$4 + local OPTS=${LOG_SCAN_GREP_OPTS:-} if ${CYLC_TEST_DEBUG:-false}; then local ERR=2 else @@ -595,7 +597,8 @@ log_scan () { echo -n "scanning for '${pattern:0:30}'" >& $ERR for _ in $(seq 1 "${REPS}"); do echo -n '.' >& $ERR - newposition=$(grep -n "$pattern" "$FILE" | \ + # shellcheck disable=SC2086 + newposition=$(grep -n $OPTS "$pattern" "$FILE" | \ tail -n 1 | cut -d ':' -f 1) if (( newposition > position )); then position=$newposition diff --git a/tests/functional/logging/02-duplicates/flow.cylc b/tests/functional/logging/02-duplicates/flow.cylc index 3d1c1dea3ea..30cdbbe6457 100644 --- a/tests/functional/logging/02-duplicates/flow.cylc +++ b/tests/functional/logging/02-duplicates/flow.cylc @@ -22,7 +22,8 @@ script = false [[bar]] script = """ -cylc set-outputs --flow=1 "${CYLC_WORKFLOW_ID}" "foo.${CYLC_TASK_CYCLE_POINT}" + cylc set --output=succeeded \ + "${CYLC_WORKFLOW_ID}//${CYLC_TASK_CYCLE_POINT}/foo" """ [[restart]] script = """ diff --git a/tests/functional/pause-resume/00-workflow/flow.cylc b/tests/functional/pause-resume/00-workflow/flow.cylc index 57a0a24aed5..a668e76ceba 100644 --- a/tests/functional/pause-resume/00-workflow/flow.cylc +++ b/tests/functional/pause-resume/00-workflow/flow.cylc @@ -19,7 +19,7 @@ script = """ wait cylc pause "${CYLC_WORKFLOW_ID}" - cylc__job__poll_grep_workflow_log -F 'INFO - Command actioned: pause()' + cylc__job__poll_grep_workflow_log 'Command "pause" actioned' cylc play "${CYLC_WORKFLOW_ID}" """ [[foo,bar]] diff --git a/tests/functional/pause-resume/12-pause-then-retry/flow.cylc b/tests/functional/pause-resume/12-pause-then-retry/flow.cylc index c732dc3bdc9..7be27343939 100644 --- a/tests/functional/pause-resume/12-pause-then-retry/flow.cylc +++ b/tests/functional/pause-resume/12-pause-then-retry/flow.cylc @@ -19,7 +19,7 @@ [[t-pause]] script = """ cylc pause "${CYLC_WORKFLOW_ID}" - cylc__job__poll_grep_workflow_log -F 'Command actioned: pause' + cylc__job__poll_grep_workflow_log 'Command "pause" actioned' # Poll t-submit-retry-able, should return submit-fail cylc poll "${CYLC_WORKFLOW_ID}//*/t-submit-retry-able" @@ -27,19 +27,19 @@ rm -f "${CYLC_WORKFLOW_RUN_DIR}/file" cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able running .* => waiting' + '1/t-retry-able/01:running.* => waiting' cylc__job__poll_grep_workflow_log -E \ - '1/t-submit-retry-able submitted .* => waiting' + '1/t-submit-retry-able/01:submitted.* => waiting' # Resume the workflow cylc play "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log -E \ - '1/t-retry-able waiting .* => waiting\(queued\)' + '1/t-retry-able:waiting.* => waiting\(queued\)' cylc__job__poll_grep_workflow_log -E \ - '1/t-submit-retry-able waiting .* => waiting\(queued\)' + '1/t-submit-retry-able:waiting.* => waiting\(queued\)' """ [[t-retry-able]] script = """ diff --git a/tests/functional/reload/11-retrying/flow.cylc b/tests/functional/reload/11-retrying/flow.cylc index d5b278b2798..ed4694a5294 100644 --- a/tests/functional/reload/11-retrying/flow.cylc +++ b/tests/functional/reload/11-retrying/flow.cylc @@ -22,7 +22,7 @@ execution retry delays = PT0S [[reloader]] script = """ - cylc__job__poll_grep_workflow_log -E '1/retrier running\(held\) .* => waiting\(held\)' + cylc__job__poll_grep_workflow_log -E '1/retrier/01:running\(held\).* => waiting\(held\)' cylc reload "${CYLC_WORKFLOW_ID}" cylc reload "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log -F 'Reload completed' diff --git a/tests/functional/reload/14-waiting/flow.cylc b/tests/functional/reload/14-waiting/flow.cylc index f81ac3533b0..01f383862e6 100644 --- a/tests/functional/reload/14-waiting/flow.cylc +++ b/tests/functional/reload/14-waiting/flow.cylc @@ -3,7 +3,7 @@ [scheduling] [[graph]] -# SoD: starter:start addeded to create a partially satisfied waiting task +# SoD: starter:start added to create a partially satisfied waiting task # to be present during the reload. R1 = """ starter:start => waiter @@ -23,8 +23,9 @@ done script = true [[reloader]] script = """ +cylc__job__wait_cylc_message_started cylc reload "${CYLC_WORKFLOW_ID}" -cylc__job__poll_grep_workflow_log -E '1/waiter .* reloaded task definition' +cylc__job__poll_grep_workflow_log -E '1/waiter.* reloaded task definition' rm -f "${CYLC_WORKFLOW_WORK_DIR}/1/sleeping-waiter/file" rm -f "${CYLC_WORKFLOW_WORK_DIR}/1/starter/file" """ diff --git a/tests/functional/reload/17-graphing-change.t b/tests/functional/reload/17-graphing-change.t index 26b7b247464..9df561384ff 100755 --- a/tests/functional/reload/17-graphing-change.t +++ b/tests/functional/reload/17-graphing-change.t @@ -66,8 +66,8 @@ cp "${TEST_SOURCE_DIR}/graphing-change/flow-2.cylc" \ "${RUN_DIR}/${WORKFLOW_NAME}/flow.cylc" # Spawn a couple of task proxies, to get "task definition removed" message. -cylc set-outputs --flow=1 "${WORKFLOW_NAME}//1/foo" -cylc set-outputs --flow=1 "${WORKFLOW_NAME}//1/baz" +cylc set "${WORKFLOW_NAME}//1/foo" +cylc set "${WORKFLOW_NAME}//1/baz" # reload workflow run_ok "${TEST_NAME_BASE}-swap-reload" cylc reload "${WORKFLOW_NAME}" poll grep_workflow_log_n_times 'Reload completed' 3 diff --git a/tests/functional/reload/19-remote-kill/flow.cylc b/tests/functional/reload/19-remote-kill/flow.cylc index 2bd548b15f7..4a9e965a73c 100644 --- a/tests/functional/reload/19-remote-kill/flow.cylc +++ b/tests/functional/reload/19-remote-kill/flow.cylc @@ -17,7 +17,7 @@ cylc reload "${CYLC_WORKFLOW_ID}" cylc__job__poll_grep_workflow_log -F 'Reload completed' cylc kill "${CYLC_WORKFLOW_ID}//1/foo" - cylc__job__poll_grep_workflow_log -E '1/foo failed\(held\) job:01.* job killed' + cylc__job__poll_grep_workflow_log -E '1/foo/01:failed\(held\).* job killed' """ [[[job]]] execution time limit = PT1M diff --git a/tests/functional/reload/23-cycle-point-time-zone.t b/tests/functional/reload/23-cycle-point-time-zone.t index 42edfaf1402..d9bf2166560 100644 --- a/tests/functional/reload/23-cycle-point-time-zone.t +++ b/tests/functional/reload/23-cycle-point-time-zone.t @@ -39,13 +39,13 @@ run_ok "${TEST_NAME_BASE}-validate" cylc validate "${WORKFLOW_NAME}" export TZ=BST-1 workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --pause -poll_workflow_running +poll_grep_workflow_log "Paused on start up" # Simulate DST change export TZ=UTC run_ok "${TEST_NAME_BASE}-reload" cylc reload "${WORKFLOW_NAME}" -poll_workflow_running +poll_grep_workflow_log "Reload completed" cylc stop --now --now "${WORKFLOW_NAME}" diff --git a/tests/functional/reload/25-xtriggers.t b/tests/functional/reload/25-xtriggers.t index 8fd1505fe6d..0269a2e3775 100644 --- a/tests/functional/reload/25-xtriggers.t +++ b/tests/functional/reload/25-xtriggers.t @@ -42,8 +42,7 @@ init_workflow "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [[reload]] script = """ # wait for "broken" to fail - cylc__job__poll_grep_workflow_log \ - '1/broken .* (received)failed/ERR' + cylc__job__poll_grep_workflow_log -E '1/broken/01.* \(received\)failed/ERR' # fix "broken" to allow it to pass sed -i 's/false/true/' "${CYLC_WORKFLOW_RUN_DIR}/flow.cylc" # reload the workflow @@ -60,12 +59,20 @@ workflow_run_ok "${TEST_NAME_BASE}-run" cylc play "${WORKFLOW_NAME}" --no-detach # 3. the retry xtrigger for "1/broken" becomes satisfied (after the reload) # (thus proving that the xtrigger survived the reload) # 4. "1/broken" succeeds + +log_scan "${TEST_NAME_BASE}-scan" \ + "$(cylc cat-log -m p "${WORKFLOW_NAME}")" \ + 1 1 \ + '1/broken.* (received)failed/ERR' + +log_scan "${TEST_NAME_BASE}-scan" \ + "$(cylc cat-log -m p "${WORKFLOW_NAME}")" 1 1 \ + 'Command "reload_workflow" actioned' \ + log_scan "${TEST_NAME_BASE}-scan" \ "$(cylc cat-log -m p "${WORKFLOW_NAME}")" \ 1 1 \ - '1/broken .* (received)failed/ERR' \ - 'Command actioned: reload_workflow()' \ 'xtrigger satisfied: _cylc_retry_1/broken' \ - '\[1/broken .* => succeeded' + '1/broken.* => succeeded' purge diff --git a/tests/functional/reload/runahead/flow.cylc b/tests/functional/reload/runahead/flow.cylc index 60d11e6477b..c65b5e11d6d 100644 --- a/tests/functional/reload/runahead/flow.cylc +++ b/tests/functional/reload/runahead/flow.cylc @@ -20,7 +20,7 @@ script = true [[reloader]] script = """ - cylc__job__poll_grep_workflow_log -E "${CYLC_TASK_CYCLE_POINT}/foo running .*\(received\)failed" + cylc__job__poll_grep_workflow_log -E "${CYLC_TASK_CYCLE_POINT}/foo/01:running.*\(received\)failed" perl -pi -e 's/(runahead limit = )P1( # marker)/\1 P3\2/' $CYLC_WORKFLOW_RUN_DIR/flow.cylc cylc reload $CYLC_WORKFLOW_ID """ diff --git a/tests/functional/remote/06-poll.t b/tests/functional/remote/06-poll.t index 4eaaa505251..905516e8499 100644 --- a/tests/functional/remote/06-poll.t +++ b/tests/functional/remote/06-poll.t @@ -52,8 +52,8 @@ log_scan \ "$(cylc cat-log -m p "$WORKFLOW_NAME")" \ 10 \ 1 \ - '\[1/foo submitted .* (polled)foo' \ - '\[1/foo .* (polled)succeeded' + '\[1/foo.* (polled)foo' \ + '\[1/foo.* (polled)succeeded' purge exit diff --git a/tests/functional/remote/09-restart-running-file-install.t b/tests/functional/remote/09-restart-running-file-install.t index deb9cf72b4d..c3249b2f5db 100644 --- a/tests/functional/remote/09-restart-running-file-install.t +++ b/tests/functional/remote/09-restart-running-file-install.t @@ -68,7 +68,7 @@ workflow_run_ok "${TEST_NAME_BASE}-restart" \ cylc play --debug --no-detach "${WORKFLOW_NAME}" LOG="${WORKFLOW_RUN_DIR}/log/scheduler/log" grep_ok "remote file install complete" "${LOG}" -grep_ok "\[1/starter running job:01 flows:1\] (received)succeeded" "${LOG}" +grep_ok "\[1/starter/01:running\] (received)succeeded" "${LOG}" ls "${WORKFLOW_RUN_DIR}/log/remote-install" > 'ls.out' cmp_ok ls.out <<__RLOGS__ 01-start-${CYLC_TEST_INSTALL_TARGET}.log diff --git a/tests/functional/restart/22-hold/flow.cylc b/tests/functional/restart/22-hold/flow.cylc index 4f2b44bdde8..213c8a00acd 100644 --- a/tests/functional/restart/22-hold/flow.cylc +++ b/tests/functional/restart/22-hold/flow.cylc @@ -17,7 +17,7 @@ [[t1]] script = """ if [[ "${CYLC_TASK_CYCLE_POINT}" == '2016' ]]; then - cylc__job__poll_grep_workflow_log -E '2016/t2 .* spawned' + cylc__job__poll_grep_workflow_log -E '2016/t2.* added to active task pool' cylc hold "${CYLC_WORKFLOW_ID}//" //2016/t2 //2017/t2 cylc stop "${CYLC_WORKFLOW_ID}" else diff --git a/tests/functional/restart/50-two-flows/flow.cylc b/tests/functional/restart/50-two-flows/flow.cylc index 8837e8835f4..bd9de46c8b4 100644 --- a/tests/functional/restart/50-two-flows/flow.cylc +++ b/tests/functional/restart/50-two-flows/flow.cylc @@ -14,7 +14,7 @@ [[a]] script = """ if ((CYLC_TASK_FLOW_NUMBERS == 2)); then - cylc__job__poll_grep_workflow_log "\[1/c .* succeeded" + cylc__job__poll_grep_workflow_log -E "/c.* succeeded" fi """ [[b, d]] @@ -23,7 +23,7 @@ if ((CYLC_TASK_FLOW_NUMBERS == 1)); then cylc trigger --flow=new --meta="cheese wizard" \ "$CYLC_WORKFLOW_ID//1/a" - cylc__job__poll_grep_workflow_log "\[1/a submitted job:02 flows:2\] => running" + cylc__job__poll_grep_workflow_log -E "\[1/a/02\(flows=2\):submitted\] => running" cylc stop $CYLC_WORKFLOW_ID fi """ diff --git a/tests/functional/restart/58-removed-task.t b/tests/functional/restart/58-removed-task.t index 17dc19f626e..1c3b79efe05 100755 --- a/tests/functional/restart/58-removed-task.t +++ b/tests/functional/restart/58-removed-task.t @@ -39,10 +39,10 @@ workflow_run_ok "${TEST_NAME}" cylc play --no-detach "${WORKFLOW_NAME}" TEST_NAME="${TEST_NAME_BASE}-restart" workflow_run_ok "${TEST_NAME}" cylc play --set="INCL_B_C=False" --no-detach "${WORKFLOW_NAME}" -grep_workflow_log_ok "grep-3" "\[1/a running job:01 flows:1\] (polled)started" -grep_workflow_log_ok "grep-4" "\[1/b failed job:01 flows:1\] (polled)failed" +grep_workflow_log_ok "grep-3" "\[1/a/01:running\] (polled)started" +grep_workflow_log_ok "grep-4" "\[1/b/01:failed\] (polled)failed" # Failed (but not incomplete) task c should not have been polled. -grep_fail "\[1/c failed job:01 flows:1\] (polled)failed" "${WORKFLOW_RUN_DIR}/log/scheduler/log" +grep_fail "\[1/c/01:failed\] (polled)failed" "${WORKFLOW_RUN_DIR}/log/scheduler/log" purge diff --git a/tests/functional/restart/58-removed-task/flow.cylc b/tests/functional/restart/58-removed-task/flow.cylc index 94c5cf27b24..0584d4b54fc 100644 --- a/tests/functional/restart/58-removed-task/flow.cylc +++ b/tests/functional/restart/58-removed-task/flow.cylc @@ -22,11 +22,11 @@ [runtime] [[a]] script = """ - cylc__job__poll_grep_workflow_log "1/b .*failed" - cylc__job__poll_grep_workflow_log "1/c .*failed" + cylc__job__poll_grep_workflow_log "1/b.*failed" + cylc__job__poll_grep_workflow_log "1/c.*failed" cylc stop --now $CYLC_WORKFLOW_ID - cylc__job__poll_grep_workflow_log "1/a .*(polled)started" - cylc__job__poll_grep_workflow_log "1/b .*(polled)failed" + cylc__job__poll_grep_workflow_log "1/a.*(polled)started" + cylc__job__poll_grep_workflow_log "1/b.*(polled)failed" """ [[b, c]] script = "false" diff --git a/tests/functional/restart/58-waiting-manual-triggered.t b/tests/functional/restart/58-waiting-manual-triggered.t index efba9f42b70..455cb289592 100644 --- a/tests/functional/restart/58-waiting-manual-triggered.t +++ b/tests/functional/restart/58-waiting-manual-triggered.t @@ -41,7 +41,7 @@ __EOF__ # It should restart and shut down normally, not stall with 2/foo waiting on 1/foo. workflow_run_ok "${TEST_NAME_BASE}-restart" cylc play --no-detach "${WORKFLOW_NAME}" # Check that 2/foo job 02 did run before shutdown. -grep_workflow_log_ok "${TEST_NAME_BASE}-grep" "\[2\/foo running job:02 flows:1\] => succeeded" +grep_workflow_log_ok "${TEST_NAME_BASE}-grep" "\[2\/foo\/02:running\] => succeeded" purge exit diff --git a/tests/functional/runahead/06-release-update.t b/tests/functional/runahead/06-release-update.t index c4ab28530e3..45fb680c69f 100644 --- a/tests/functional/runahead/06-release-update.t +++ b/tests/functional/runahead/06-release-update.t @@ -27,14 +27,23 @@ CYLC_RUN_PID="$!" poll_workflow_running YYYY="$(date +%Y)" NEXT1=$(( YYYY + 1 )) -poll_grep_workflow_log -E "${NEXT1}/bar .* spawned" +poll_grep_workflow_log -E "${NEXT1}/bar.* added to active task pool" # sleep a little to allow the datastore to update (`cylc dump` sees the # datastore) TODO can we avoid this flaky sleep somehow? sleep 10 + # (gratuitous use of --flows for test coverage) cylc dump --flows -t "${WORKFLOW_NAME}" | awk '{print $1 $2 $3 $7}' >'log' + +# The scheduler task pool should contain: +# NEXT1/foo - waiting on clock trigger +# NEXT1/bar - waiting, partially satisfied +# The n=1 data store should also contain: +# YYYY/bar - succeeded + cmp_ok 'log' - <<__END__ +bar,$NEXT1,waiting,[1] foo,$NEXT1,waiting,[1] __END__ diff --git a/tests/functional/runahead/default-future/flow.cylc b/tests/functional/runahead/default-future/flow.cylc index ee083a1dc91..c78522d7a8c 100644 --- a/tests/functional/runahead/default-future/flow.cylc +++ b/tests/functional/runahead/default-future/flow.cylc @@ -27,7 +27,7 @@ [[spawner]] script = """ # spawn wibble - cylc set-outputs --flow=1 $CYLC_WORKFLOW_ID 20100101T0800Z/foo + cylc set $CYLC_WORKFLOW_ID 20100101T0800Z/foo """ [[foo]] script = false diff --git a/tests/functional/spawn-on-demand/05-stop-flow/flow.cylc b/tests/functional/spawn-on-demand/05-stop-flow/flow.cylc index cac2ddf8009..2c2d1996009 100644 --- a/tests/functional/spawn-on-demand/05-stop-flow/flow.cylc +++ b/tests/functional/spawn-on-demand/05-stop-flow/flow.cylc @@ -10,5 +10,5 @@ [[bar]] script = """ cylc stop --flow=1 ${CYLC_WORKFLOW_ID} - cylc__job__poll_grep_workflow_log 'Command actioned: stop' + cylc__job__poll_grep_workflow_log 'Command "stop" actioned' """ diff --git a/tests/functional/spawn-on-demand/06-stop-flow-2/flow.cylc b/tests/functional/spawn-on-demand/06-stop-flow-2/flow.cylc index 9c7da97e974..3005fd3a277 100644 --- a/tests/functional/spawn-on-demand/06-stop-flow-2/flow.cylc +++ b/tests/functional/spawn-on-demand/06-stop-flow-2/flow.cylc @@ -14,13 +14,13 @@ script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 2 )); then cylc stop --flow=1 ${CYLC_WORKFLOW_ID} - cylc__job__poll_grep_workflow_log "Command actioned: stop" + cylc__job__poll_grep_workflow_log 'Command "stop" actioned' fi """ [[baz]] script = """ if (( CYLC_TASK_SUBMIT_NUMBER == 1 )); then cylc trigger --flow=new --meta=other "${CYLC_WORKFLOW_ID}//1/foo" - cylc__job__poll_grep_workflow_log -E "1/bar running job:02.* => succeeded" + cylc__job__poll_grep_workflow_log -E "1/bar/02\(flows=2\):running.* => succeeded" fi """ diff --git a/tests/functional/spawn-on-demand/07-abs-triggers/flow.cylc b/tests/functional/spawn-on-demand/07-abs-triggers/flow.cylc index 8084e5c0abe..3f621ccc1e4 100644 --- a/tests/functional/spawn-on-demand/07-abs-triggers/flow.cylc +++ b/tests/functional/spawn-on-demand/07-abs-triggers/flow.cylc @@ -16,7 +16,7 @@ script = """ # Ensure that 1,2/bar are spawned by 1,2/foo and not by 2/start # (so the scheduler must update their prereqs when 2/start finishes). - cylc__job__poll_grep_workflow_log -E "2/bar .* spawned" + cylc__job__poll_grep_workflow_log -E "2/bar.* added to active task pool" """ [[foo]] [[bar]] diff --git a/tests/functional/spawn-on-demand/09-set-outputs.t b/tests/functional/spawn-on-demand/09-set-outputs.t index 41c6b5b6b87..57d34e15024 100644 --- a/tests/functional/spawn-on-demand/09-set-outputs.t +++ b/tests/functional/spawn-on-demand/09-set-outputs.t @@ -16,7 +16,7 @@ # along with this program. If not, see . #------------------------------------------------------------------------------- -# Check that "cylc set-outputs" works like it says on the tin. +# Check that "cylc set" works like it says on the tin. . "$(dirname "$0")/test_header" set_test_number 2 reftest diff --git a/tests/functional/spawn-on-demand/09-set-outputs/flow.cylc b/tests/functional/spawn-on-demand/09-set-outputs/flow.cylc index 1d1d7e7e061..a40e6e9be33 100644 --- a/tests/functional/spawn-on-demand/09-set-outputs/flow.cylc +++ b/tests/functional/spawn-on-demand/09-set-outputs/flow.cylc @@ -1,8 +1,6 @@ -# Test that `cylc set-outputs` has the same effect as natural output +# Test that `cylc set` has the same effect as natural output # completion: i.e. that downstream children are spawned as normal. -# DEBUG mode required: we search for "task proxy removed" in the log. - [scheduler] [[events]] abort on stall timeout = True @@ -35,7 +33,7 @@ [[foo]] # Hang about until setter is finished. script = """ - cylc__job__poll_grep_workflow_log -E "1/setter .* => succeeded" + cylc__job__poll_grep_workflow_log -E "1/setter.* => succeeded" """ [[bar]] script = true @@ -43,11 +41,11 @@ # (To the rescue). script = """ # Set foo outputs while it still exists in the pool. - cylc set-outputs --flow=2 --output=out1 --output=out2 "${CYLC_WORKFLOW_ID}//1/foo" + cylc set --flow=2 --output=out1 --output=out2 "${CYLC_WORKFLOW_ID}//1/foo" # Set bar outputs after it is gone from the pool. - cylc__job__poll_grep_workflow_log -E "1/bar .*task proxy removed" - cylc set-outputs --flow=2 --output=out1 --output=out2 "${CYLC_WORKFLOW_ID}//1/bar" + cylc__job__poll_grep_workflow_log -E "1/bar.* task completed" + cylc set --flow=2 --output=out1 --output=out2 "${CYLC_WORKFLOW_ID}//1/bar" """ [[qux, quw, fux, fuw]] script = true diff --git a/tests/functional/spawn-on-demand/10-retrigger/flow.cylc b/tests/functional/spawn-on-demand/10-retrigger/flow.cylc index 2bdd4365a07..7e9149ce3c9 100644 --- a/tests/functional/spawn-on-demand/10-retrigger/flow.cylc +++ b/tests/functional/spawn-on-demand/10-retrigger/flow.cylc @@ -18,7 +18,7 @@ """ [[triggerer]] script = """ - cylc__job__poll_grep_workflow_log -E '1/oops running .* \(received\)failed' + cylc__job__poll_grep_workflow_log -E '1/oops/01:running.* \(received\)failed' cylc trigger "${CYLC_WORKFLOW_ID}//1/oops" """ [[foo, bar]] diff --git a/tests/functional/spawn-on-demand/11-hold-not-spawned/flow.cylc b/tests/functional/spawn-on-demand/11-hold-not-spawned/flow.cylc index 890d73f78be..c47ca3c93c4 100644 --- a/tests/functional/spawn-on-demand/11-hold-not-spawned/flow.cylc +++ b/tests/functional/spawn-on-demand/11-hold-not-spawned/flow.cylc @@ -15,6 +15,6 @@ script = true [[stopper]] script = """ - cylc__job__poll_grep_workflow_log "\[1/holdee .* holding \(as requested earlier\)" -E + cylc__job__poll_grep_workflow_log "\[1/holdee.* holding \(as requested earlier\)" -E cylc stop $CYLC_WORKFLOW_ID """ diff --git a/tests/functional/spawn-on-demand/12-set-outputs-no-reflow.t b/tests/functional/spawn-on-demand/12-set-outputs-cont-flow.t similarity index 93% rename from tests/functional/spawn-on-demand/12-set-outputs-no-reflow.t rename to tests/functional/spawn-on-demand/12-set-outputs-cont-flow.t index 41c6b5b6b87..36c8bd100b3 100644 --- a/tests/functional/spawn-on-demand/12-set-outputs-no-reflow.t +++ b/tests/functional/spawn-on-demand/12-set-outputs-cont-flow.t @@ -16,7 +16,7 @@ # along with this program. If not, see . #------------------------------------------------------------------------------- -# Check that "cylc set-outputs" works like it says on the tin. +# Check that "cylc set" continues a flow by default. . "$(dirname "$0")/test_header" set_test_number 2 reftest diff --git a/tests/functional/spawn-on-demand/12-set-outputs-no-reflow/flow.cylc b/tests/functional/spawn-on-demand/12-set-outputs-cont-flow/flow.cylc similarity index 70% rename from tests/functional/spawn-on-demand/12-set-outputs-no-reflow/flow.cylc rename to tests/functional/spawn-on-demand/12-set-outputs-cont-flow/flow.cylc index 315007c52fa..353d6d1f41f 100644 --- a/tests/functional/spawn-on-demand/12-set-outputs-no-reflow/flow.cylc +++ b/tests/functional/spawn-on-demand/12-set-outputs-cont-flow/flow.cylc @@ -1,5 +1,5 @@ -# Test that `cylc set-outputs` does not cause reflow by default -# Task setter should cause bar to run, but not subsequently baz. +# Test that `cylc set` continues the active flow by default +# Task "setter" should cause bar to run, then subsequently baz. [scheduler] [[events]] @@ -21,5 +21,5 @@ script = true [[setter]] script = """ - cylc set-outputs "${CYLC_WORKFLOW_ID}//1/foo" + cylc set --output=succeeded "${CYLC_WORKFLOW_ID}//1/foo" """ diff --git a/tests/functional/spawn-on-demand/12-set-outputs-no-reflow/reference.log b/tests/functional/spawn-on-demand/12-set-outputs-cont-flow/reference.log similarity index 79% rename from tests/functional/spawn-on-demand/12-set-outputs-no-reflow/reference.log rename to tests/functional/spawn-on-demand/12-set-outputs-cont-flow/reference.log index 2322cc234da..3c7b498cc8b 100644 --- a/tests/functional/spawn-on-demand/12-set-outputs-no-reflow/reference.log +++ b/tests/functional/spawn-on-demand/12-set-outputs-cont-flow/reference.log @@ -3,3 +3,4 @@ Final point: 1 1/foo -triggered off [] 1/setter -triggered off ['1/foo'] 1/bar -triggered off ['1/foo'] +1/baz -triggered off ['1/bar'] diff --git a/tests/functional/spawn-on-demand/14-trigger-flow-blocker/flow.cylc b/tests/functional/spawn-on-demand/14-trigger-flow-blocker/flow.cylc index 0849b69bb75..9c205e301c5 100644 --- a/tests/functional/spawn-on-demand/14-trigger-flow-blocker/flow.cylc +++ b/tests/functional/spawn-on-demand/14-trigger-flow-blocker/flow.cylc @@ -23,7 +23,7 @@ cylc trigger --flow=none $CYLC_WORKFLOW_ID//3/foo elif ((CYLC_TASK_CYCLE_POINT == 3)); then # Run until I get merged. - cylc__job__poll_grep_workflow_log -E "3/foo .* merged in flow\(s\) 1" + cylc__job__poll_grep_workflow_log -E "3/foo.* merged in flow\(s\) 1" fi """ [[bar]] diff --git a/tests/functional/spawn-on-demand/18-submitted.t b/tests/functional/spawn-on-demand/18-submitted.t index de5041f4ca1..30f022ebafd 100644 --- a/tests/functional/spawn-on-demand/18-submitted.t +++ b/tests/functional/spawn-on-demand/18-submitted.t @@ -40,7 +40,7 @@ reftest_run for number in 1 2 3; do grep_workflow_log_ok \ "${TEST_NAME_BASE}-a${number}" \ - "${number}/a${number} .* did not complete required outputs: \['submitted'\]" + "${number}/a${number}.* did not complete required outputs: \['submitted'\]" done purge diff --git a/tests/functional/spawn-on-demand/19-submitted-compat.t b/tests/functional/spawn-on-demand/19-submitted-compat.t index d529dfb4183..98c603d55a7 100644 --- a/tests/functional/spawn-on-demand/19-submitted-compat.t +++ b/tests/functional/spawn-on-demand/19-submitted-compat.t @@ -51,7 +51,7 @@ grep_workflow_log_ok \ 'Backward compatibility mode ON' grep_workflow_log_ok \ "${TEST_NAME_BASE}-a-complete" \ - '\[1/a running job:01 flows:1\] => succeeded' + '\[1/a/01:running\] => succeeded' grep_workflow_log_ok \ "${TEST_NAME_BASE}-b-incomplete" \ "1/b did not complete required outputs: \['submitted', 'succeeded'\]" diff --git a/tests/functional/special/08-clock-trigger-retry.t b/tests/functional/special/08-clock-trigger-retry.t index d4f591d4870..fa3a65ee150 100644 --- a/tests/functional/special/08-clock-trigger-retry.t +++ b/tests/functional/special/08-clock-trigger-retry.t @@ -42,7 +42,7 @@ workflow_run_ok "${TEST_NAME_BASE}-run" cylc play --no-detach "$WORKFLOW_NAME" log_scan "${TEST_NAME_BASE}-log-scan" \ "${WORKFLOW_RUN_DIR}/log/scheduler/log" 2 1 \ - "\[20150101.*/foo .* job:01 .* retrying in PT5S" \ + "\[20150101.*/foo.* retrying in PT5S" \ "xtrigger satisfied: _cylc_retry_20150101" # (if task resubmits immediately instead of waiting PT5S, xtrigger msg will not appear) diff --git a/tests/functional/startup/00-state-summary.t b/tests/functional/startup/00-state-summary.t index a4a02208899..76d16bbd25d 100644 --- a/tests/functional/startup/00-state-summary.t +++ b/tests/functional/startup/00-state-summary.t @@ -30,7 +30,7 @@ run_ok "${TEST_NAME}" cylc validate "${WORKFLOW_NAME}" cylc play --no-detach "${WORKFLOW_NAME}" > /dev/null 2>&1 # Restart with a failed task and a succeeded task. cylc play "${WORKFLOW_NAME}" -poll_grep_workflow_log -E '1/foo .* \(polled\)failed' +poll_grep_workflow_log -E '1/foo.* \(polled\)failed' cylc dump "${WORKFLOW_NAME}" > dump.out TEST_NAME=${TEST_NAME_BASE}-grep # State summary should not just say "Initializing..." diff --git a/tests/functional/triggering/19-and-suicide/flow.cylc b/tests/functional/triggering/19-and-suicide/flow.cylc index cf7ae49d129..670c361fc96 100644 --- a/tests/functional/triggering/19-and-suicide/flow.cylc +++ b/tests/functional/triggering/19-and-suicide/flow.cylc @@ -16,7 +16,7 @@ [[t0]] # https://github.com/cylc/cylc-flow/issues/2655 # "1/t2" should not suicide on "1/t1:failed" - script = cylc__job__poll_grep_workflow_log -E '1/t1 .* \(received\)failed' + script = cylc__job__poll_grep_workflow_log -E '1/t1.* \(received\)failed' [[t1]] script = false [[t2]] diff --git a/tests/functional/triggering/21-expire.t b/tests/functional/triggering/21-expire.t new file mode 100644 index 00000000000..aaacdf807b0 --- /dev/null +++ b/tests/functional/triggering/21-expire.t @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test expire triggering +. "$(dirname "$0")/test_header" +set_test_number 2 +reftest +exit diff --git a/tests/functional/triggering/21-expire/flow.cylc b/tests/functional/triggering/21-expire/flow.cylc new file mode 100644 index 00000000000..2a5336cb5e8 --- /dev/null +++ b/tests/functional/triggering/21-expire/flow.cylc @@ -0,0 +1,22 @@ +[scheduling] + initial cycle point = 1999 + [[special tasks]] + clock-expire = foo1(PT0S), foo2(PT0S), bar1(PT0S), x(PT0S) + [[graph]] + # Expire: foo1, foo2, bar1, x + # Run: y, bar2, baz, qux + R1 = """ + x? + FOO? + BAR? + x:expire? => y + FOO:expire-all? => baz + BAR:expire-any? => qux + """ +[runtime] + [[FOO, BAR]] + [[foo1, foo2]] + inherit = FOO + [[bar1, bar2]] + inherit = BAR + [[x, y, baz, qux]] diff --git a/tests/functional/triggering/21-expire/reference.log b/tests/functional/triggering/21-expire/reference.log new file mode 100644 index 00000000000..8ba5edca688 --- /dev/null +++ b/tests/functional/triggering/21-expire/reference.log @@ -0,0 +1,4 @@ +19990101T0000Z/bar2 -triggered off [] in flow 1 +19990101T0000Z/baz -triggered off ['19990101T0000Z/foo1', '19990101T0000Z/foo2'] in flow 1 +19990101T0000Z/qux -triggered off ['19990101T0000Z/bar1'] in flow 1 +19990101T0000Z/y -triggered off ['19990101T0000Z/x'] in flow 1 diff --git a/tests/functional/xtriggers/03-sequence.t b/tests/functional/xtriggers/03-sequence.t index 63c360f66c5..a41b970b3f1 100644 --- a/tests/functional/xtriggers/03-sequence.t +++ b/tests/functional/xtriggers/03-sequence.t @@ -49,7 +49,7 @@ run_ok "${TEST_NAME_BASE}-val" cylc validate "${WORKFLOW_NAME}" # Run workflow; it will stall waiting on the never-satisfied xtriggers. cylc play "${WORKFLOW_NAME}" -poll_grep_workflow_log -E '2025/start .* => succeeded' +poll_grep_workflow_log -E '2025/start.* => succeeded' cylc show "${WORKFLOW_NAME}//2026/foo" | grep -E '^ - xtrigger' > 2026.foo.log diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 8d44f0d42b6..d575f2a6af2 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -24,6 +24,7 @@ from typing import List, TYPE_CHECKING, Set, Tuple, Union from cylc.flow.config import WorkflowConfig +from cylc.flow.id import Tokens from cylc.flow.option_parsers import Options from cylc.flow.pathutil import get_cylc_run_dir from cylc.flow.rundb import CylcWorkflowDAO @@ -544,7 +545,9 @@ def complete(): The scheduler to await. tokens_list: If specified, this will wait for the tasks represented by these - tokens to be marked as completed by the task pool. + tokens to be marked as completed by the task pool. Can use + relative task ids as strings (e.g. '1/a') rather than tokens for + convenience. stop_mode: If tokens_list is not provided, this will wait for the scheduler to be shutdown with the specified mode (default = AUTO, i.e. @@ -561,20 +564,26 @@ def complete(): """ async def _complete( schd, - *tokens_list, + *tokens_list: Union[Tokens, str], stop_mode=StopMode.AUTO, - timeout=60, - ): + timeout: int = 60, + ) -> None: start_time = time() - tokens_list = [tokens.task for tokens in tokens_list] + + _tokens_list: List[Tokens] = [] + for tokens in tokens_list: + if isinstance(tokens, str): + tokens = Tokens(tokens, relative=True) + _tokens_list.append(tokens.task) # capture task completion remove_if_complete = schd.pool.remove_if_complete - def _remove_if_complete(itask): + def _remove_if_complete(itask, output=None): + nonlocal _tokens_list ret = remove_if_complete(itask) - if ret and itask.tokens.task in tokens_list: - tokens_list.remove(itask.tokens.task) + if ret and itask.tokens.task in _tokens_list: + _tokens_list.remove(itask.tokens.task) return ret schd.pool.remove_if_complete = _remove_if_complete @@ -595,8 +604,8 @@ def _set_stop(mode=None): schd._set_stop = _set_stop # determine the completion condition - if tokens_list: - condition = lambda: bool(tokens_list) + if _tokens_list: + condition = lambda: bool(_tokens_list) else: condition = lambda: bool(not has_shutdown) @@ -604,9 +613,9 @@ def _set_stop(mode=None): while condition(): # allow the main loop to advance await asyncio.sleep(0) - if time() - start_time > timeout: + if (time() - start_time) > timeout: raise Exception( - f'Timeout waiting for {", ".join(map(str, tokens_list))}' + f'Timeout waiting for {", ".join(map(str, _tokens_list))}' ) # restore regular shutdown logic diff --git a/tests/integration/scripts/test_completion_server.py b/tests/integration/scripts/test_completion_server.py new file mode 100644 index 00000000000..0c792fac3da --- /dev/null +++ b/tests/integration/scripts/test_completion_server.py @@ -0,0 +1,204 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Integration tests for the "cylc completion-server command. + +See also the more extensive unit tests for this module. +""" + +from cylc.flow.scripts.completion_server import complete_cylc + + +def setify(coro): + """Cast returned lists to sets for coroutines. + + Convenience function to use when you want to test output not order. + """ + async def _coro(*args, **kwargs): + nonlocal coro + ret = await coro(*args, **kwargs) + if isinstance(ret, list): + return set(ret) + return ret + return _coro + + +async def test_list_prereqs_and_outputs(flow, scheduler, start): + """Test the success cases for listing task prereqs/outputs. + + The error cases are tested in a unit test (doesn't require a running + scheduler). + """ + _complete_cylc = setify(complete_cylc) # Note: results are un-ordered + + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'initial cycle point': '1', + 'cycling mode': 'integer', + 'graph': { + 'P1': ''' + a => b + c => d + b[-P1] => b + ''' + }, + }, + 'runtime': { + 'a': {}, + 'b': { + 'outputs': { + 'foo': 'abc def ghi', + } + } + } + }) + schd = scheduler(id_) + async with start(schd): + await schd.update_data_structure() + b1 = schd.tokens.duplicate(cycle='1', task='b') + d1 = schd.tokens.duplicate(cycle='1', task='d') + e1 = schd.tokens.duplicate(cycle='1', task='e') # does not exist + + # list prereqs (b1) + assert await _complete_cylc('cylc', 'set', b1.id, '--pre', '') == { + # keywords + 'all', + # intra-cycle dependency + '1/a:succeeded', + # inter-cycle dependency + '0/b:succeeded', + } + + # list outputs (b1) + assert await _complete_cylc('cylc', 'set', b1.id, '--out', '') == { + # regular task outputs + 'expired', + 'failed', + 'started', + 'submit-failed', + 'submitted', + 'succeeded', + # custom task outputs + 'foo', + } + + # list prereqs (d1) + assert await _complete_cylc('cylc', 'set', d1.id, '--pre', '') == { + # keywords + 'all', + # d1 prereqs + '1/c:succeeded', + } + + # list prereqs for multiple (b1, d1) + assert await _complete_cylc( + 'cylc', + 'set', + b1.id, + d1.id, + '--pre', + '', + ) == { + # keywords + 'all', + # b1 prereqs + '1/a:succeeded', + '0/b:succeeded', + # d1 prereqs + '1/c:succeeded', + } + + # list prereqs for multiple (b1, d1) - alternative format + assert await _complete_cylc( + 'cylc', + 'set', + f'{schd.id}//', + f'//{b1.relative_id}', + f'//{d1.relative_id}', + '--pre', + '', + ) == { + # keywords + 'all', + # b1 prereqs + '1/a:succeeded', + '0/b:succeeded', + # d1 prereqs + '1/c:succeeded', + } + + # list outputs for a non-existant task + assert await _complete_cylc('cylc', 'set', e1.id, '--out', '') == set() + + # list outputs for a non-existant workflow + assert await _complete_cylc( + 'cylc', + 'set', + # this invalid workflow shouldn't prevent it from returning values + # for the valid one + 'no-such-workflow//', + f'{schd.id}//', + f'//{b1.relative_id}', + f'//{d1.relative_id}', + '--pre', + '', + ) == { + # keywords + 'all', + # b1 prereqs + '1/a:succeeded', + '0/b:succeeded', + # d1 prereqs + '1/c:succeeded', + } + + # start a second workflow to test multi-workflow functionality + id2 = flow({ + 'scheduling': { + 'graph': { + 'R1': ''' + x => z + ''' + } + }, + 'runtime': {'x': {}, 'z': {}}, + }) + schd2 = scheduler(id2) + async with start(schd2): + await schd2.update_data_structure() + z1 = schd2.tokens.duplicate(cycle='1', task='z') + + # list prereqs for multiple tasks in multiple workflows + # (it should combine the results from both workflows) + assert await _complete_cylc( + 'cylc', + 'set', + b1.id, + z1.id, + '--pre', + '', + ) == { + # keywords + 'all', + # workflow1//1/b prereqs + '0/b:succeeded', + '1/a:succeeded', + # workflow2//1/z prereqs + '1/x:succeeded' + } diff --git a/tests/integration/scripts/test_set.py b/tests/integration/scripts/test_set.py new file mode 100644 index 00000000000..22cd44bbbce --- /dev/null +++ b/tests/integration/scripts/test_set.py @@ -0,0 +1,164 @@ +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +"""Test "cylc set" functionality. + +Note: see also functional tests +""" + +from cylc.flow.cycling.integer import IntegerPoint +from cylc.flow.data_store_mgr import TASK_PROXIES +from cylc.flow.task_state import TASK_STATUS_WAITING, TASK_STATUS_SUCCEEDED + + +async def test_set_parentless_spawning( + flow, + scheduler, + run, + complete, +): + """Ensure that setting outputs does not interfere with parentless spawning. + + Setting outputs manually causes the logic to follow a different code + pathway to "natural" output satisfaction. If we're not careful this could + lead to "premature shutdown" (i.e. the scheduler thinks it's finished when + it isn't), this test makes sure that's not the case. + """ + id_ = flow({ + 'scheduling': { + 'initial cycle point': '1', + 'cycling mode': 'integer', + 'runahead limit': 'P0', + 'graph': {'P1': 'a => z'}, + }, + }) + schd = scheduler(id_, paused_start=False) + async with run(schd): + # mark cycle 1 as succeeded + schd.pool.set_prereqs_and_outputs(['1/a', '1/z'], ['succeeded'], None, ['1']) + + # the parentless task "a" should be spawned out to the runahead limit + assert [ + itask.identity for itask in schd.pool.get_tasks() + ] == ['2/a', '3/a'] + + # the workflow should run on to the next cycle + await complete(schd, '2/a', timeout=5) + + +async def test_rerun_incomplete( + flow, + scheduler, + run, + complete, + reflog, +): + """Incomplete tasks should be re-run.""" + id_ = flow({ + 'scheduling': { + 'graph': {'R1': 'a => z'}, + }, + 'runtime': { + # register a custom output + 'a': {'outputs': {'x': 'xyz'}}, + }, + }) + schd = scheduler(id_, paused_start=False) + async with run(schd): + # generate 1/a:x but do not complete 1/a + schd.pool.set_prereqs_and_outputs(['1/a'], ['x'], None, ['1']) + triggers = reflog(schd) + await complete(schd) + + assert triggers == { + # the task 1/a should have been run despite the earlier + # setting of the "x" output + ('1/a', None), + ('1/z', ('1/a',)), + } + + +async def test_data_store( + flow, + scheduler, + start, +): + """Test that manually set prereqs/outputs are applied to the data store.""" + id_ = flow({ + 'scheduling': { + 'graph': {'R1': 'a => z'}, + }, + 'runtime': { + # register a custom output + 'a': {'outputs': {'x': 'xyz'}}, + }, + }) + schd = scheduler(id_) + async with start(schd): + await schd.update_data_structure() + data = schd.data_store_mgr.data[schd.tokens.id] + task_a = data[TASK_PROXIES][ + schd.pool.get_task(IntegerPoint('1'), 'a').tokens.id + ] + + # set the 1/a:succeeded prereq of 1/z + schd.pool.set_prereqs_and_outputs( + ['1/z'], None, ['1/a:succeeded'], ['1']) + task_z = data[TASK_PROXIES][ + schd.pool.get_task(IntegerPoint('1'), 'z').tokens.id + ] + await schd.update_data_structure() + assert task_z.prerequisites[0].satisfied is True + + # set 1/a:x the task should be waiting with output x satisfied + schd.pool.set_prereqs_and_outputs(['1/a'], ['x'], None, ['1']) + await schd.update_data_structure() + assert task_a.state == TASK_STATUS_WAITING + assert task_a.outputs['x'].satisfied is True + assert task_a.outputs['succeeded'].satisfied is False + + # set 1/a:succeeded the task should be succeeded with output x sat + schd.pool.set_prereqs_and_outputs(['1/a'], ['succeeded'], None, ['1']) + await schd.update_data_structure() + assert task_a.state == TASK_STATUS_SUCCEEDED + assert task_a.outputs['x'].satisfied is True + assert task_a.outputs['succeeded'].satisfied is True + + +async def test_incomplete_detection( + one_conf, + flow, + scheduler, + start, + log_filter, +): + """It should detect and log finished tasks left with incomplete outputs.""" + schd = scheduler(flow(one_conf)) + async with start(schd) as log: + schd.pool.set_prereqs_and_outputs(['1/one'], ['failed'], None, ['1']) + assert log_filter(log, contains='1/one did not complete') + + +async def test_pre_all(flow, scheduler, run): + """Ensure that --pre=all is interpreted as a special case + and _not_ tokenized. + """ + id_ = flow({'scheduling': {'graph': {'R1': 'a => z'}}}) + schd = scheduler(id_, paused_start=False) + async with run(schd) as log: + schd.pool.set_prereqs_and_outputs(['1/z'], [], ['all'], ['all']) + warn_or_higher = [i for i in log.records if i.levelno > 30] + assert warn_or_higher == [] diff --git a/tests/integration/test_data_store_mgr.py b/tests/integration/test_data_store_mgr.py index 21b95280443..78b24200634 100644 --- a/tests/integration/test_data_store_mgr.py +++ b/tests/integration/test_data_store_mgr.py @@ -171,14 +171,14 @@ async def test_delta_task_state(harness): w_id = schd.data_store_mgr.workflow_id schd.data_store_mgr.data[w_id] = data assert TASK_STATUS_FAILED not in set(collect_states(data, TASK_PROXIES)) - for itask in schd.pool.get_all_tasks(): + for itask in schd.pool.get_tasks(): itask.state.reset(TASK_STATUS_FAILED) schd.data_store_mgr.delta_task_state(itask) assert TASK_STATUS_FAILED in set(collect_states( schd.data_store_mgr.updated, TASK_PROXIES)) # put things back the way we found them - for itask in schd.pool.get_all_tasks(): + for itask in schd.pool.get_tasks(): itask.state.reset(TASK_STATUS_WAITING) schd.data_store_mgr.delta_task_state(itask) await schd.update_data_structure() @@ -191,7 +191,7 @@ async def test_delta_task_held(harness): schd.pool.hold_tasks('*') await schd.update_data_structure() assert True in {t.is_held for t in data[TASK_PROXIES].values()} - for itask in schd.pool.get_all_tasks(): + for itask in schd.pool.get_tasks(): itask.state.reset(is_held=False) schd.data_store_mgr.delta_task_held(itask) assert True not in { @@ -269,7 +269,7 @@ async def test_update_data_structure(harness): assert TASK_STATUS_FAILED not in set(collect_states(data, FAMILY_PROXIES)) assert TASK_STATUS_FAILED not in data[WORKFLOW].state_totals assert len({t.is_held for t in data[TASK_PROXIES].values()}) == 2 - for itask in schd.pool.get_all_tasks(): + for itask in schd.pool.get_tasks(): itask.state.reset(TASK_STATUS_FAILED) schd.data_store_mgr.delta_task_state(itask) schd.data_store_mgr.update_data_structure() @@ -288,15 +288,17 @@ async def test_update_data_structure(harness): def test_delta_task_prerequisite(harness): """Test delta_task_prerequisites.""" schd, data = harness - schd.pool.force_spawn_children([ - t.identity - for t in schd.pool.get_all_tasks() - ], (TASK_STATUS_SUCCEEDED,), "flow1") + schd.pool.set_prereqs_and_outputs( + [t.identity for t in schd.pool.get_tasks()], + [(TASK_STATUS_SUCCEEDED,)], + [], + "all" + ) assert all({ p.satisfied for t in schd.data_store_mgr.updated[TASK_PROXIES].values() for p in t.prerequisites}) - for itask in schd.pool.get_all_tasks(): + for itask in schd.pool.get_tasks(): # set prereqs as not-satisfied for prereq in itask.state.prerequisites: prereq._all_satisfied = False diff --git a/tests/integration/test_examples.py b/tests/integration/test_examples.py index 882d4c10163..5cdb1f21552 100644 --- a/tests/integration/test_examples.py +++ b/tests/integration/test_examples.py @@ -138,7 +138,7 @@ async def test_task_pool(one, start): async with start(one): # pump the scheduler's heart manually one.pool.release_runahead_tasks() - assert len(one.pool.main_pool) == 1 + assert len(one.pool.active_tasks) == 1 async def test_exception(one, run, log_filter): diff --git a/tests/integration/test_reload.py b/tests/integration/test_reload.py index d9de0826dfb..5bd07c17af4 100644 --- a/tests/integration/test_reload.py +++ b/tests/integration/test_reload.py @@ -99,11 +99,11 @@ def change_state(_=0): [ # the task should have entered the preparing state before the # reload was requested - '[1/foo waiting(queued) job:00 flows:1] => preparing(queued)', + '[1/foo:waiting(queued)] => preparing(queued)', # the reload should have put the workflow into the paused state - 'PAUSING the workflow now: Reloading workflow', + 'Pausing the workflow: Reloading workflow', # reload should have waited for the task to submit - '[1/foo preparing(queued) job:00 flows:1]' + '[1/foo/00:preparing(queued)]' ' => submitted(queued)', # before then reloading the workflow config 'Reloading the workflow definition.', diff --git a/tests/integration/test_resolvers.py b/tests/integration/test_resolvers.py index 1190cd70978..17e09983dc3 100644 --- a/tests/integration/test_resolvers.py +++ b/tests/integration/test_resolvers.py @@ -20,12 +20,11 @@ import pytest -from cylc.flow import CYLC_LOG from cylc.flow.data_store_mgr import EDGES, TASK_PROXIES from cylc.flow.id import Tokens +from cylc.flow import CYLC_LOG from cylc.flow.network.resolvers import Resolvers from cylc.flow.scheduler import Scheduler -from cylc.flow.workflow_status import StopMode @pytest.fixture @@ -217,44 +216,32 @@ async def test_mutation_mapper(mock_flow): """Test the mapping of mutations to internal command methods.""" meta = {} response = await mock_flow.resolvers._mutation_mapper('pause', {}, meta) - assert response is None + assert response[0] is True # (True, command-uuid-str) with pytest.raises(ValueError): await mock_flow.resolvers._mutation_mapper('non_exist', {}, meta) -@pytest.mark.asyncio -async def test_stop( - one: Scheduler, run: Callable, log_filter: Callable, -): - """Test the stop resolver.""" - async with run(one) as log: - resolvers = Resolvers( - one.data_store_mgr, - schd=one - ) - resolvers.stop(StopMode.REQUEST_CLEAN) - await one.process_command_queue() - assert log_filter( - log, level=logging.INFO, contains="Command actioned: stop" - ) - assert one.stop_mode == StopMode.REQUEST_CLEAN - - -async def test_command_logging(mock_flow, caplog): - """It should log the command, with user name if not owner.""" - caplog.set_level(logging.INFO, logger=CYLC_LOG) - owner = mock_flow.owner - other = f"not-{mock_flow.owner}" - - command = "stop" - mock_flow.resolvers._log_command(command, owner) - assert caplog.records[-1].msg == f"[command] {command}" - mock_flow.resolvers._log_command(command, other) - msg1 = f"[command] {command} (issued by {other})" - assert caplog.records[-1].msg == msg1 - - command = "put_messages" - mock_flow.resolvers._log_command(command, owner) - assert caplog.records[-1].msg == msg1 # (prev message, i.e. not logged). - mock_flow.resolvers._log_command(command, other) - assert caplog.records[-1].msg == f"[command] {command} (issued by {other})" +async def test_command_logging(mock_flow, caplog, log_filter): + """The command log message should include non-owner name.""" + + meta = {} + + caplog.set_level(logging.INFO, CYLC_LOG) + + await mock_flow.resolvers._mutation_mapper("stop", {}, meta) + assert log_filter(caplog, contains='Command "stop" received') + + # put_messages: only log for owner + kwargs = { + "task_job": "1/foo/01", + "event_time": "bedtime", + "messages": [[logging.CRITICAL, "it's late"]] + } + meta["auth_user"] = mock_flow.owner + await mock_flow.resolvers._mutation_mapper("put_messages", kwargs, meta) + assert not log_filter(caplog, contains='Command "put_messages" received:') + + meta["auth_user"] = "Dr Spock" + await mock_flow.resolvers._mutation_mapper("put_messages", kwargs, meta) + assert log_filter( + caplog, contains='Command "put_messages" received from Dr Spock') diff --git a/tests/integration/test_scheduler.py b/tests/integration/test_scheduler.py index 6befe2b34d1..49c5e07a9fc 100644 --- a/tests/integration/test_scheduler.py +++ b/tests/integration/test_scheduler.py @@ -24,7 +24,6 @@ from cylc.flow.exceptions import CylcError from cylc.flow.parsec.exceptions import ParsecError from cylc.flow.scheduler import Scheduler, SchedulerStop -from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED from cylc.flow.task_state import ( TASK_STATUS_WAITING, TASK_STATUS_SUBMIT_FAILED, @@ -212,7 +211,7 @@ async def test_no_poll_waiting_tasks( log: pytest.LogCaptureFixture async with start(one) as log: # Test assumes start up with a waiting task. - task = (one.pool.get_all_tasks())[0] + task = (one.pool.get_tasks())[0] assert task.state.status == TASK_STATUS_WAITING polled_tasks = capture_polling(one) @@ -325,7 +324,7 @@ async def test_uuid_unchanged_on_restart( cf_uuid = uuid_re.findall(contact_file.read_text()) assert cf_uuid == [schd.uuid_str] - + async def test_restart_timeout( flow, one_conf, @@ -348,11 +347,15 @@ async def test_restart_timeout( id_ = flow(one_conf) # run the workflow to completion - schd = scheduler(id_) - async with start(schd): - for itask in schd.pool.get_all_tasks(): - itask.state_reset(TASK_OUTPUT_SUCCEEDED) - schd.pool.spawn_on_output(itask, TASK_OUTPUT_SUCCEEDED) + # (by setting the only task to completed) + schd = scheduler(id_, paused_start=False) + async with start(schd) as log: + for itask in schd.pool.get_tasks(): + # (needed for job config in sim mode:) + schd.task_job_mgr.submit_task_jobs( + schd.workflow, [itask], None, None) + schd.pool.set_prereqs_and_outputs( + [itask.identity], None, None, ['all']) # restart the completed workflow schd = scheduler(id_) diff --git a/tests/integration/test_task_pool.py b/tests/integration/test_task_pool.py index aec35516514..3d75074cf15 100644 --- a/tests/integration/test_task_pool.py +++ b/tests/integration/test_task_pool.py @@ -14,21 +14,31 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from copy import deepcopy import logging -from typing import AsyncGenerator, Callable, Iterable, List, Tuple, Union +from typing import ( + TYPE_CHECKING, + AsyncGenerator, + Callable, + Iterable, + List, + Tuple, + Union +) import pytest from pytest import param +from json import loads from cylc.flow import CYLC_LOG -from cylc.flow.cycling import PointBase from cylc.flow.cycling.integer import IntegerPoint from cylc.flow.cycling.iso8601 import ISO8601Point from cylc.flow.data_store_mgr import TASK_PROXIES -from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED -from cylc.flow.scheduler import Scheduler -from cylc.flow.flow_mgr import FLOW_ALL +from cylc.flow.task_events_mgr import TaskEventsManager +from cylc.flow.task_outputs import ( + TASK_OUTPUT_SUCCEEDED +) + +from cylc.flow.flow_mgr import FLOW_ALL, FLOW_NONE from cylc.flow.task_state import ( TASK_STATUS_WAITING, TASK_STATUS_PREPARING, @@ -38,9 +48,13 @@ TASK_STATUS_FAILED, TASK_STATUS_EXPIRED, TASK_STATUS_SUBMIT_FAILED, - TASK_STATUSES_ALL, ) +if TYPE_CHECKING: + from cylc.flow.cycling import PointBase + from cylc.flow.scheduler import Scheduler + from cylc.flow.task_proxy import TaskProxy + # NOTE: foo and bar have no parents so at start-up (even with the workflow # paused) they get spawned out to the runahead limit. 2/pub spawns # immediately too, because we spawn autospawn absolute-triggered tasks as @@ -83,8 +97,17 @@ } +def pool_get_task_ids( + pool: List['TaskProxy'] +) -> List[str]: + """Return sorted list of IDs of tasks in a task pool.""" + return sorted( + [itask.identity for itask in pool.get_tasks()] + ) + + def get_task_ids( - name_point_list: Iterable[Tuple[str, Union[PointBase, str, int]]] + name_point_list: Iterable[Tuple[str, Union['PointBase', str, int]]] ) -> List[str]: """Helper function to return sorted task identities from a list of (name, point) tuples.""" @@ -116,17 +139,16 @@ def assert_expected_log( @pytest.fixture(scope='module') async def mod_example_flow( mod_flow: Callable, mod_scheduler: Callable, mod_run: Callable -) -> Scheduler: +) -> 'Scheduler': """Return a scheduler for interrogating its task pool. This is module-scoped so faster than example_flow, but should only be used where the test does not mutate the state of the scheduler or task pool. """ id_ = mod_flow(EXAMPLE_FLOW_CFG) - schd: Scheduler = mod_scheduler(id_, paused_start=True) + schd: 'Scheduler' = mod_scheduler(id_, paused_start=True) async with mod_run(schd): - pass - return schd + yield schd @pytest.fixture @@ -135,7 +157,7 @@ async def example_flow( scheduler: Callable, start, caplog: pytest.LogCaptureFixture, -) -> AsyncGenerator[Scheduler, None]: +) -> AsyncGenerator['Scheduler', None]: """Return a scheduler for interrogating its task pool. This is function-scoped so slower than mod_example_flow; only use this @@ -145,7 +167,7 @@ async def example_flow( # set up caplog and do schd.install()/.initialise()/.configure() instead caplog.set_level(logging.INFO, CYLC_LOG) id_ = flow(EXAMPLE_FLOW_CFG) - schd: Scheduler = scheduler(id_) + schd: 'Scheduler' = scheduler(id_) async with start(schd): yield schd @@ -153,17 +175,16 @@ async def example_flow( @pytest.fixture(scope='module') async def mod_example_flow_2( mod_flow: Callable, mod_scheduler: Callable, mod_run: Callable -) -> Scheduler: +) -> 'Scheduler': """Return a scheduler for interrogating its task pool. This is module-scoped so faster than example_flow, but should only be used where the test does not mutate the state of the scheduler or task pool. """ id_ = mod_flow(EXAMPLE_FLOW_2_CFG) - schd: Scheduler = mod_scheduler(id_, paused_start=True) + schd: 'Scheduler' = mod_scheduler(id_, paused_start=True) async with mod_run(schd): - pass - return schd + yield schd @pytest.mark.parametrize( @@ -216,9 +237,8 @@ async def test_filter_task_proxies( expected_task_ids: List[str], expected_bad_items: List[str], expected_warnings: List[str], - mod_example_flow: Scheduler, - caplog: pytest.LogCaptureFixture, - monkeypatch, + mod_example_flow: 'Scheduler', + caplog: pytest.LogCaptureFixture ) -> None: """Test TaskPool.filter_task_proxies(). @@ -241,68 +261,6 @@ async def test_filter_task_proxies( assert_expected_log(caplog, expected_warnings) -@pytest.mark.parametrize( - 'items, expected_task_ids, expected_bad_items, expected_warnings', - [ - param( - ['*:waiting'], - ['1/waz', '1/foo', '1/bar', '2/foo', '2/bar', '2/pub', '3/foo', - '3/bar', '4/foo', '4/bar', '5/foo', '5/bar'], [], [], - id="Task state" - ), - ] -) -async def test_filter_task_proxies_hidden( - items: List[str], - expected_task_ids: List[str], - expected_bad_items: List[str], - expected_warnings: List[str], - mod_example_flow: Scheduler, - caplog: pytest.LogCaptureFixture, - monkeypatch, -) -> None: - """Test TaskPool.filter_task_proxies(). - - This is the same as test_filter_task_proxies except we artificially add a - new proxy "1/waz" to the hidden pool. Filtering should find a single copy - each of the hidden and main pool tasks. - - See GitHub #4909: a bug in filtering was doubling up tasks in cycle points - that appeared in both pools. - - The NOTE before EXAMPLE_FLOW_CFG above explains which tasks should be - expected for the tests here. - - Params: - items: Arg passed to filter_task_proxies(). - expected_task_ids: IDs of the TaskProxys that are expected to be - returned, of the form "{point}/{name}"/ - expected_bad_items: Expected to be returned. - expected_warnings: Expected to be logged. - """ - monkeypatch.setattr( - # make Tokens objects mutable to allow deepcopy to work on TaskProxy - # objects - 'cylc.flow.id.Tokens.__setitem__', - lambda self, key, value: dict.__setitem__(self, key, value), - ) - - caplog.set_level(logging.WARNING, CYLC_LOG) - task_pool = mod_example_flow.pool - - # Duplicate a task proxy, rename it, and add it to the hidden pool. - a_task = deepcopy(task_pool.get_tasks()[0]) - a_task.identity = "1/waz" - task_pool.hidden_pool.setdefault(a_task.point, {}) - task_pool.hidden_pool[a_task.point][a_task.identity] = a_task - - itasks, _, bad_items = task_pool.filter_task_proxies(items) - task_ids = [itask.identity for itask in itasks] - assert sorted(task_ids) == sorted(expected_task_ids) - assert sorted(bad_items) == sorted(expected_bad_items) - assert_expected_log(caplog, expected_warnings) - - @pytest.mark.parametrize( 'items, expected_task_ids, expected_warnings', [ @@ -353,7 +311,7 @@ async def test_match_taskdefs( items: List[str], expected_task_ids: List[str], expected_warnings: List[str], - mod_example_flow: Scheduler, + mod_example_flow: 'Scheduler', caplog: pytest.LogCaptureFixture ) -> None: """Test TaskPool.match_taskdefs(). @@ -415,7 +373,7 @@ async def test_hold_tasks( items: List[str], expected_tasks_to_hold_ids: List[str], expected_warnings: List[str], - example_flow: Scheduler, caplog: pytest.LogCaptureFixture, + example_flow: 'Scheduler', caplog: pytest.LogCaptureFixture, db_select: Callable ) -> None: """Test TaskPool.hold_tasks(). @@ -434,7 +392,7 @@ async def test_hold_tasks( task_pool = example_flow.pool n_warnings = task_pool.hold_tasks(items) - for itask in task_pool.get_all_tasks(): + for itask in task_pool.get_tasks(): hold_expected = itask.identity in expected_tasks_to_hold_ids assert itask.state.is_held is hold_expected @@ -448,7 +406,7 @@ async def test_hold_tasks( async def test_release_held_tasks( - example_flow: Scheduler, db_select: Callable + example_flow: 'Scheduler', db_select: Callable ) -> None: """Test TaskPool.release_held_tasks(). @@ -463,7 +421,7 @@ async def test_release_held_tasks( task_pool = example_flow.pool expected_tasks_to_hold_ids = sorted(['1/foo', '1/bar', '3/asd']) task_pool.hold_tasks(expected_tasks_to_hold_ids) - for itask in task_pool.get_all_tasks(): + for itask in task_pool.get_tasks(): hold_expected = itask.identity in expected_tasks_to_hold_ids assert itask.state.is_held is hold_expected assert get_task_ids(task_pool.tasks_to_hold) == expected_tasks_to_hold_ids @@ -472,7 +430,7 @@ async def test_release_held_tasks( # Test task_pool.release_held_tasks(['1/foo', '3/asd']) - for itask in task_pool.get_all_tasks(): + for itask in task_pool.get_tasks(): assert itask.state.is_held is (itask.identity == '1/bar') expected_tasks_to_hold_ids = sorted(['1/bar']) @@ -494,7 +452,7 @@ async def test_release_held_tasks( async def test_hold_point( hold_after_point: int, expected_held_task_ids: List[str], - example_flow: Scheduler, db_select: Callable + example_flow: 'Scheduler', db_select: Callable ) -> None: """Test TaskPool.set_hold_point() and .release_hold_point()""" expected_held_task_ids = sorted(expected_held_task_ids) @@ -506,7 +464,7 @@ async def test_hold_point( assert ('holdcp', str(hold_after_point)) in db_select( example_flow, True, 'workflow_params') - for itask in task_pool.get_all_tasks(): + for itask in task_pool.get_tasks(): hold_expected = itask.identity in expected_held_task_ids assert itask.state.is_held is hold_expected @@ -521,7 +479,7 @@ async def test_hold_point( ('holdcp', None) ] - for itask in task_pool.get_all_tasks(): + for itask in task_pool.get_tasks(): assert itask.state.is_held is False assert task_pool.tasks_to_hold == set() @@ -604,7 +562,7 @@ async def test_reload_stopcp( } } } - schd: Scheduler = scheduler(flow(cfg)) + schd: 'Scheduler' = scheduler(flow(cfg)) async with start(schd): assert str(schd.pool.stop_point) == '2020' await schd.command_reload_workflow() @@ -612,7 +570,7 @@ async def test_reload_stopcp( async def test_runahead_after_remove( - example_flow: Scheduler + example_flow: 'Scheduler' ) -> None: """The runahead limit should be recomputed after tasks are removed. @@ -634,7 +592,7 @@ async def test_load_db_bad_platform( ): """Test that loading an unavailable platform from the database doesn't cause calamitous failure.""" - schd: Scheduler = scheduler(flow(one_conf)) + schd: 'Scheduler' = scheduler(flow(one_conf)) async with start(schd): result = schd.pool.load_db_task_pool_for_restart(0, ( @@ -645,7 +603,7 @@ async def test_load_db_bad_platform( def list_tasks(schd): - """Return a list of task pool tasks (incl hidden pool tasks). + """Return a sorted list of task pool tasks. Returns a list in the format: [ @@ -655,7 +613,7 @@ def list_tasks(schd): """ return sorted( (itask.tokens['cycle'], itask.tokens['task'], itask.state.status) - for itask in schd.pool.get_all_tasks() + for itask in schd.pool.get_tasks() ) @@ -746,8 +704,8 @@ async def test_restart_prereqs( assert list_tasks(schd) == expected_1 # Mark 1/a as succeeded and spawn 1/z - schd.pool.get_all_tasks()[0].state_reset('succeeded') - schd.pool.spawn_on_output(schd.pool.get_all_tasks()[0], 'succeeded') + task_a = schd.pool.get_tasks()[0] + schd.pool.task_events_mgr.process_message(task_a, 1, 'succeeded') assert list_tasks(schd) == expected_2 # Save our progress @@ -770,7 +728,9 @@ async def test_restart_prereqs( schd.data_store_mgr.update_data_structure() # Check resulting dependencies of task z - task_z = schd.pool.get_all_tasks()[0] + task_z = [ + t for t in schd.pool.get_tasks() if t.tdef.name == "z" + ][0] assert sorted( ( p.satisfied @@ -867,8 +827,8 @@ async def test_reload_prereqs( assert list_tasks(schd) == expected_1 # Mark 1/a as succeeded and spawn 1/z - schd.pool.get_all_tasks()[0].state_reset('succeeded') - schd.pool.spawn_on_output(schd.pool.get_all_tasks()[0], 'succeeded') + task_a = schd.pool.get_tasks()[0] + schd.pool.task_events_mgr.process_message(task_a, 1, 'succeeded') assert list_tasks(schd) == expected_2 # Modify flow.cylc to add a new dependency on "z" @@ -880,7 +840,9 @@ async def test_reload_prereqs( assert list_tasks(schd) == expected_3 # Check resulting dependencies of task z - task_z = schd.pool.get_all_tasks()[0] + task_z = [ + t for t in schd.pool.get_tasks() if t.tdef.name == "z" + ][0] assert sorted( ( p.satisfied @@ -904,10 +866,9 @@ async def _test_restart_prereqs_sat(): ] # Mark both as succeeded and spawn 1/c - for itask in schd.pool.get_all_tasks(): - itask.state_reset('succeeded') - schd.pool.spawn_on_output(itask, 'succeeded') - schd.workflow_db_mgr.put_insert_task_outputs(itask) + for itask in schd.pool.get_tasks(): + schd.pool.task_events_mgr.process_message(itask, 1, 'succeeded') + schd.workflow_db_mgr.put_update_task_outputs(itask) schd.pool.remove_if_complete(itask) schd.workflow_db_mgr.process_queued_ops() assert list_tasks(schd) == [ @@ -922,7 +883,7 @@ async def _test_restart_prereqs_sat(): ] # Check resulting dependencies of task z - task_c = schd.pool.get_all_tasks()[0] + task_c = schd.pool.get_tasks()[0] assert sorted( (*key, satisfied) for prereq in task_c.state.prerequisites @@ -1018,8 +979,8 @@ async def test_runahead_limit_for_sequence_before_start_cycle( ): """It should obey the runahead limit. - Ensure the runahead limit is computed correctly for sequences that begin - before the start cycle. + Ensure the runahead limit is computed correctly for sequences before the + start cycle See https://github.com/cylc/cylc-flow/issues/5603 """ @@ -1080,7 +1041,7 @@ async def test_db_update_on_removal( task_a = schd.pool.get_tasks()[0] # set the task to running - task_a.state_reset('running') + schd.pool.task_events_mgr.process_message(task_a, 1, 'started') # update the db await schd.update_data_structure() @@ -1092,7 +1053,7 @@ async def test_db_update_on_removal( ] # mark the task as succeeded and allow it to be removed from the pool - task_a.state_reset('succeeded') + schd.pool.task_events_mgr.process_message(task_a, 1, 'succeeded') schd.pool.remove_if_complete(task_a) # update the DB, note no new tasks have been added to the pool @@ -1110,8 +1071,7 @@ async def test_no_flow_tasks_dont_spawn( ): """Ensure no-flow tasks don't spawn downstreams. - No-flow tasks (i.e `--flow=none`) are one-offs which are not attached to - any "flow". + No-flow tasks (i.e `--flow=none`) are not attached to any "flow". See https://github.com/cylc/cylc-flow/issues/5613 """ @@ -1128,9 +1088,14 @@ async def test_no_flow_tasks_dont_spawn( schd = scheduler(id_) async with start(schd): - # mark task 1/a as succeeded task_a = schd.pool.get_tasks()[0] - task_a.state_reset(TASK_OUTPUT_SUCCEEDED) + + # set as no-flow: + task_a.flow_nums = set() + + # Set as completed: should not spawn children. + schd.pool.set_prereqs_and_outputs( + [task_a.identity], None, None, [FLOW_NONE]) for flow_nums, force, pool in ( # outputs yielded from a no-flow task should not spawn downstreams @@ -1153,16 +1118,13 @@ async def test_no_flow_tasks_dont_spawn( TASK_OUTPUT_SUCCEEDED, forced=force, ) + schd.pool.spawn_on_all_outputs(task_a) # ensure the pool is as expected assert [ (itask.identity, itask.flow_nums) - for pool in [ - schd.pool.get_tasks(), - schd.pool.get_hidden_tasks(), - ] - for itask in pool + for itask in schd.pool.get_tasks() ] == pool @@ -1176,23 +1138,16 @@ async def test_task_proxy_remove_from_queues( # Set up a scheduler with a non-default queue: one_conf['scheduling'] = { 'queues': {'queue_two': {'members': 'one, control'}}, - 'graph': {'R1': 'two & one & hidden & control & hidden_control'}, + 'graph': {'R1': 'two & one & control'}, } schd = scheduler(flow(one_conf)) async with start(schd): # Get a list of itasks: itasks = schd.pool.get_tasks() - point = itasks[0].point for itask in itasks: id_ = itask.identity - # Move some tasks to the hidden_pool to ensure that these are - # removed too: - if 'hidden' in itask.identity: - schd.pool.hidden_pool.setdefault(point, {id_: itask}) - del schd.pool.main_pool[point][id_] - # The meat of the test - remove itask from pool if it # doesn't have "control" in the name: if 'control' not in id_: @@ -1203,12 +1158,11 @@ async def test_task_proxy_remove_from_queues( name: [itask.identity for itask in queue.deque] for name, queue in schd.pool.task_queue_mgr.queues.items()} - assert queues_after['default'] == ['1/hidden_control'] assert queues_after['queue_two'] == ['1/control'] async def test_runahead_offset_start( - mod_example_flow_2: Scheduler + mod_example_flow_2: 'Scheduler' ) -> None: """Late-start recurrences should not break the runahead limit at start-up. @@ -1224,25 +1178,21 @@ async def test_detect_incomplete_tasks( start, log_filter, ): - """Finished tasks should be marked as incomplete. + """Finished but incomplete tasks should be retains as incomplete.""" - If a task finishes without completing all required outputs, then it should - be marked as incomplete. - """ - incomplete_final_task_states = [ - TASK_STATUS_FAILED, - TASK_STATUS_EXPIRED, - TASK_STATUS_SUBMIT_FAILED, - ] + final_task_states = { + TASK_STATUS_FAILED: TaskEventsManager.EVENT_FAILED, + TASK_STATUS_EXPIRED: TaskEventsManager.EVENT_EXPIRED, + TASK_STATUS_SUBMIT_FAILED: TaskEventsManager.EVENT_SUBMIT_FAILED + } id_ = flow({ 'scheduler': { 'allow implicit tasks': 'True', }, 'scheduling': { 'graph': { - # a workflow with one task for each of the incomplete final - # task states - 'R1': '\n'.join(incomplete_final_task_states) + # a workflow with one task for each of the final task states + 'R1': '\n'.join(final_task_states.keys()) } } }) @@ -1250,15 +1200,451 @@ async def test_detect_incomplete_tasks( async with start(schd) as log: itasks = schd.pool.get_tasks() for itask in itasks: + itask.state_reset(is_queued=False) # spawn the output corresponding to the task - schd.pool.spawn_on_output(itask, itask.tdef.name) + schd.pool.task_events_mgr.process_message( + itask, 1, + final_task_states[itask.tdef.name] + ) # ensure that it is correctly identified as incomplete assert itask.state.outputs.get_incomplete() assert itask.state.outputs.is_incomplete() - assert log_filter( - log, contains=f"[{itask}] did not complete required outputs:") - # the task should not have been removed - assert itask in schd.pool.get_tasks() + if itask.tdef.name == TASK_STATUS_EXPIRED: + assert log_filter( + log, + contains=f"[{itask}] removed (expired)" + ) + # the task should have been removed + assert itask not in schd.pool.get_tasks() + else: + assert log_filter( + log, + contains=( + f"[{itask}] did not complete " + "required outputs:" + ) + ) + # the task should not have been removed + assert itask in schd.pool.get_tasks() + + +async def test_future_trigger_final_point( + flow, + scheduler, + start, + log_filter, +): + """Check spawning of future-triggered tasks: foo[+P1] => bar. + + Don't spawn if a prerequisite reaches beyond the final cycle point. + + """ + id_ = flow( + { + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'cycling mode': 'integer', + 'initial cycle point': 1, + 'final cycle point': 1, + 'graph': { + 'P1': "foo\n foo[+P1] & bar => baz" + } + } + } + ) + schd = scheduler(id_) + async with start(schd) as log: + for itask in schd.pool.get_tasks(): + schd.pool.spawn_on_output(itask, "succeeded") + assert log_filter( + log, + regex=( + ".*1/baz.*not spawned: a prerequisite is beyond" + r" the workflow stop point \(1\)" + ) + ) + + +async def test_set_failed_complete( + flow, + scheduler, + start, + one_conf, + log_filter, + db_select: Callable +): + """Test manual completion of an incomplete failed task.""" + id_ = flow(one_conf) + schd = scheduler(id_) + async with start(schd) as log: + one = schd.pool.get_tasks()[0] + one.state_reset(is_queued=False) + + schd.pool.task_events_mgr.process_message(one, 1, "failed") + assert log_filter( + log, regex="1/one.* setting implied output: submitted") + assert log_filter( + log, regex="1/one.* setting implied output: started") + assert log_filter( + log, regex="failed.* did not complete required outputs") + + # Set failed task complete via default "set" args. + schd.pool.set_prereqs_and_outputs([one.identity], None, None, ['all']) + + assert log_filter( + log, contains=f'[{one}] task completed') + + db_outputs = db_select( + schd, True, 'task_outputs', 'outputs', + **{'name': 'one'} + ) + assert ( + sorted(loads((db_outputs[0])[0])) == [ + "failed", "started", "submitted", "succeeded" + ] + ) + + +async def test_set_prereqs( + flow, + scheduler, + start, + log_filter, +): + """Check manual setting of prerequisites. + + """ + id_ = flow( + { + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'initial cycle point': '2040', + 'graph': { + 'R1': "foo & bar & baz => qux" + } + }, + 'runtime': { + 'foo': { + 'outputs': { + 'a': 'drugs and money', + } + } + } + } + ) + schd = scheduler(id_) + + async with start(schd) as log: + + # it should start up with foo, bar, baz + assert ( + pool_get_task_ids(schd.pool) == [ + "20400101T0000Z/bar", + "20400101T0000Z/baz", + "20400101T0000Z/foo"] + ) + + # try to set an invalid prereq of qux + schd.pool.set_prereqs_and_outputs( + ["20400101T0000Z/qux"], None, ["20400101T0000Z/foo:a"], ['all']) + assert log_filter( + log, contains='20400101T0000Z/qux does not depend on "20400101T0000Z/foo:a"') + + # it should not add 20400101T0000Z/qux to the pool + assert ( + pool_get_task_ids(schd.pool) == [ + "20400101T0000Z/bar", + "20400101T0000Z/baz", + "20400101T0000Z/foo"] + ) + + # set one prereq of future task 20400101T0000Z/qux + schd.pool.set_prereqs_and_outputs( + ["20400101T0000Z/qux"], + None, + ["20400101T0000Z/foo:succeeded"], + ['all']) + + # it should add 20400101T0000Z/qux to the pool + assert ( + pool_get_task_ids(schd.pool) == [ + "20400101T0000Z/bar", + "20400101T0000Z/baz", + "20400101T0000Z/foo", + "20400101T0000Z/qux" + ] + ) + + # get the 20400101T0000Z/qux task proxy + qux = schd.pool.get_task(ISO8601Point("20400101T0000Z"), "qux") + assert not qux.state.prerequisites_all_satisfied() + + # set its other prereqs (test implicit "succeeded" and "succeed") + # and truncated cycle point + schd.pool.set_prereqs_and_outputs( + ["2040/qux"], None, ["2040/bar", "2040/baz:succeed"], ['all']) + + # it should now be fully satisfied + assert qux.state.prerequisites_all_satisfied() + + +async def test_set_bad_prereqs( + flow, + scheduler, + start, + log_filter, +): + """Check manual setting of prerequisites. + + """ + id_ = flow({ + 'scheduler': { + 'allow implicit tasks': 'True', + 'cycle point format': '%Y'}, + 'scheduling': { + 'initial cycle point': '2040', + 'graph': {'R1': "foo => bar"}}, + }) + schd = scheduler(id_) + + def set_prereqs(prereqs): + """Shorthand so only varible under test given as arg""" + schd.pool.set_prereqs_and_outputs( + ["2040/bar"], None, prereqs, ['all']) + + async with start(schd) as log: + # Invalid: task name wildcard: + set_prereqs(["2040/*"]) + assert log_filter(log, contains='Invalid prerequisite task name' ) + + # Invalid: cycle point wildcard. + set_prereqs(["*/foo"]) + assert log_filter(log, contains='Invalid prerequisite cycle point') + + +async def test_set_outputs_live( + flow, + scheduler, + start, + log_filter, +): + """Check manual set outputs in an active (spawned) task. + + """ + id_ = flow( + { + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'graph': { + 'R1': """ + foo:x => bar + foo => baz + foo:y + """ + } + }, + 'runtime': { + 'foo': { + 'outputs': { + 'x': 'xylophone', + 'y': 'yacht' + } + } + } + } + ) + schd = scheduler(id_) + + async with start(schd) as log: + + # it should start up with just 1/foo + assert pool_get_task_ids(schd.pool) == ["1/foo"] + + # fake failed + foo = schd.pool.get_task(IntegerPoint("1"), "foo") + foo.state_reset(is_queued=False) + schd.pool.task_events_mgr.process_message(foo, 1, 'failed') + + # set foo:x: it should spawn bar but not baz + schd.pool.set_prereqs_and_outputs(["1/foo"], ["x"], None, ['all']) + assert ( + pool_get_task_ids(schd.pool) == ["1/bar", "1/foo"] + ) + # Foo should have been removed from the queue: + assert '1/foo' not in [ + i.identity for i + in schd.pool.task_queue_mgr.queues['default'].deque + ] + # set foo:succeed: it should spawn baz but foo remains incomplete. + schd.pool.set_prereqs_and_outputs( + ["1/foo"], ["succeeded"], None, ['all']) + assert ( + pool_get_task_ids(schd.pool) == ["1/bar", "1/baz", "1/foo"] + ) + + # it should complete implied outputs (submitted, started) too + assert log_filter( + log, contains="setting implied output: submitted") + assert log_filter( + log, contains="setting implied output: started") + + # set foo (default: all required outputs) to complete y. + schd.pool.set_prereqs_and_outputs(["1/foo"], None, None, ['all']) + assert log_filter( + log, contains="output 1/foo:succeeded completed") + assert ( + pool_get_task_ids(schd.pool) == ["1/bar", "1/baz"] + ) + + +async def test_set_outputs_live2( + flow, + scheduler, + start, + log_filter, +): + """Assert that optional outputs are satisfied before completion + outputs to prevent incomplete task warnings. + """ + id_ = flow( + { + 'scheduler': {'allow implicit tasks': 'True'}, + 'scheduling': {'graph': { + 'R1': """ + foo:a => apple + foo:b => boat + """}}, + 'runtime': {'foo': {'outputs': { + 'a': 'xylophone', + 'b': 'yacht'}}} + } + ) + schd = scheduler(id_) + + async with start(schd) as log: + schd.pool.set_prereqs_and_outputs(["1/foo"], None, None, ['all']) + assert not log_filter( + log, + contains="did not complete required outputs: ['a', 'b']" + ) + + +async def test_set_outputs_future( + flow, + scheduler, + start, + log_filter, +): + """Check manual setting of future task outputs. + + """ + id_ = flow( + { + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'graph': { + 'R1': "a:x & a:y => b => c" + } + }, + 'runtime': { + 'a': { + 'outputs': { + 'x': 'xylophone', + 'y': 'yacht' + } + } + } + } + ) + schd = scheduler(id_) + + async with start(schd) as log: + + # it should start up with just 1/a + assert pool_get_task_ids(schd.pool) == ["1/a"] + + # setting future task b succeeded should spawn c but not b + schd.pool.set_prereqs_and_outputs( + ["1/b"], ["succeeded"], None, ['all']) + assert ( + pool_get_task_ids(schd.pool) == ["1/a", "1/c"] + ) + + schd.pool.set_prereqs_and_outputs( + items=["1/a"], + outputs=["x", "y", "cheese"], + prereqs=None, + flow=['all'] + ) + assert log_filter(log, contains="output 1/a:cheese not found") + assert log_filter(log, contains="completed output x") + assert log_filter(log, contains="completed output y") + + +async def test_prereq_satisfaction( + flow, + scheduler, + start, + log_filter, +): + """Check manual setting of task prerequisites. + + """ + id_ = flow( + { + 'scheduler': { + 'allow implicit tasks': 'True', + }, + 'scheduling': { + 'graph': { + 'R1': "a:x & a:y => b" + } + }, + 'runtime': { + 'a': { + 'outputs': { + 'x': 'xylophone', + 'y': 'yacht' + } + } + } + } + ) + schd = scheduler(id_) + async with start(schd) as log: + # it should start up with just 1/a + assert pool_get_task_ids(schd.pool) == ["1/a"] + # spawn b + schd.pool.set_prereqs_and_outputs(["1/a"], ["x"], None, ['all']) + assert ( + pool_get_task_ids(schd.pool) == ["1/a", "1/b"] + ) + + b = schd.pool.get_task(IntegerPoint("1"), "b") + + assert not b.is_waiting_prereqs_done() + + # set valid and invalid prerequisites, by label and message. + schd.pool.set_prereqs_and_outputs( + prereqs=["1/a:xylophone", "1/a:y", "1/a:w", "1/a:z"], + items=["1/b"], outputs=None, flow=['all'] + ) + assert log_filter(log, contains="1/a:z not found") + assert log_filter(log, contains="1/a:w not found") + assert not log_filter(log, contains='1/b does not depend on "1/a:x"') + assert not log_filter( + log, contains='1/b does not depend on "1/a:xylophone"') + assert not log_filter(log, contains='1/b does not depend on "1/a:y"') + + assert b.is_waiting_prereqs_done() @pytest.mark.parametrize('compat_mode', ['compat-mode', 'normal-mode']) @@ -1284,6 +1670,7 @@ async def test_compute_runahead( """ if cycling_mode == 'integer': + config = { 'scheduler': { 'allow implicit tasks': 'True', @@ -1408,32 +1795,22 @@ async def test_runahead_future_trigger( async with start(schd, level=logging.DEBUG): assert str(schd.pool.runahead_limit_point) == '20010103' schd.pool.release_runahead_tasks() - for itask in schd.pool.get_all_tasks(): + for itask in schd.pool.get_tasks(): schd.pool.spawn_on_output(itask, 'succeeded') # future trigger raises the limit by one cycle point assert str(schd.pool.runahead_limit_point) == '20010104' -async def test_compute_runahead_against_task_state( - flow, - scheduler, - start, - monkeypatch, -): - """For each task status check whether changing the oldest task - to that status will cause compute_runahead to make a change. +@pytest.fixture(scope='module') +async def mod_blah( + mod_flow: Callable, mod_scheduler: Callable, mod_run: Callable +) -> 'Scheduler': + """Return a scheduler for interrogating its task pool. + + This is module-scoped so faster than example_flow, but should only be used + where the test does not mutate the state of the scheduler or task pool. """ - states = [ - # (Status, Are we expecting an update?) - (TASK_STATUS_WAITING, False), - (TASK_STATUS_EXPIRED, True), - (TASK_STATUS_PREPARING, False), - (TASK_STATUS_SUBMIT_FAILED, True), - (TASK_STATUS_SUBMITTED, False), - (TASK_STATUS_RUNNING, False), - (TASK_STATUS_FAILED, True), - (TASK_STATUS_SUCCEEDED, True) - ] + config = { 'scheduler': { 'allow implicit tasks': 'True', @@ -1447,26 +1824,53 @@ async def test_compute_runahead_against_task_state( }, } } + id_ = mod_flow(config) + schd: 'Scheduler' = mod_scheduler(id_, paused_start=True) + async with mod_run(schd): + yield schd + + +@pytest.mark.parametrize( + 'status, expected', + [ + # (Status, Are we expecting an update?) + (TASK_STATUS_WAITING, False), + (TASK_STATUS_EXPIRED, False), + (TASK_STATUS_PREPARING, False), + (TASK_STATUS_SUBMIT_FAILED, False), + (TASK_STATUS_SUBMITTED, False), + (TASK_STATUS_RUNNING, False), + (TASK_STATUS_FAILED, True), + (TASK_STATUS_SUCCEEDED, True) + ] +) +async def test_runahead_c7_compat_task_state( + status, + expected, + mod_blah, + monkeypatch, +): + """For each task status check whether changing the oldest task + to that status will cause compute_runahead to make a change. + + Compat mode: Cylc 7 ignored failed tasks but not submit-failed! + + """ def max_cycle(tasks): return max([int(t.tokens.get("cycle")) for t in tasks]) monkeypatch.setattr( - 'cylc.flow.flags.cylc7_back_compat', - True) + 'cylc.flow.flags.cylc7_back_compat', True) monkeypatch.setattr( 'cylc.flow.task_events_mgr.TaskEventsManager._insert_task_job', lambda *_: True) - schd = scheduler(flow(config)) - async with start(schd): - for task_status, new_runahead in states: - before = max_cycle(schd.pool.get_tasks()) - itask = schd.pool.get_task(ISO8601Point(f'{before - 2:04}'), 'a') - schd.task_events_mgr.process_message( - itask, - logging.INFO, - task_status, - ) - after = max_cycle(schd.pool.get_tasks()) - assert bool(before != after) == new_runahead + mod_blah.pool.compute_runahead() + before_pt = max_cycle(mod_blah.pool.get_tasks()) + before = mod_blah.pool.runahead_limit_point + itask = mod_blah.pool.get_task(ISO8601Point(f'{before_pt - 2:04}'), 'a') + itask.state_reset(status, is_queued=False) + mod_blah.pool.compute_runahead() + after = mod_blah.pool.runahead_limit_point + assert bool(before != after) == expected diff --git a/tests/integration/test_trigger.py b/tests/integration/test_trigger.py index e008caa4a7c..30ae3404ed8 100644 --- a/tests/integration/test_trigger.py +++ b/tests/integration/test_trigger.py @@ -37,7 +37,7 @@ async def test_trigger_invalid(mod_one, start, log_filter, flow_strs): """Ensure invalid flow values are rejected.""" async with start(mod_one) as log: log.clear() - assert mod_one.pool.force_trigger_tasks(['*'], flow_strs) == 0 + assert mod_one.pool.force_trigger_tasks(['*'], flow_strs) is None assert len(log_filter(log, level=logging.WARN)) == 1 diff --git a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html index ca1a8a73f0c..6e6490ebc05 100644 --- a/tests/integration/tui/screenshots/test_auto_expansion.later-time.html +++ b/tests/integration/tui/screenshots/test_auto_expansion.later-time.html @@ -1,9 +1,9 @@ 
Cylc Tui   workflows filtered (W - edit, E - reset)                             
                                                                                 
 - ~cylc                                                                         
-   - one - paused                                                               
-      - ̿○ 1                                                                     
-           ̿○ b                                                                  
+   - one - paused 1■                                                            
+      - ● 1                                                                     
+           ● b                                                                  
       - ̿○ 2                                                                     
          - ̿○ A                                                                  
               ̿○ a                                                               
diff --git a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
index b8a802948f3..8bfe41ea904 100644
--- a/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
+++ b/tests/integration/tui/screenshots/test_online_mutation.hold-mutation-selected.html
@@ -9,7 +9,7 @@
                │  < log                                      >  │               
                │  < poll                                     >  │               
                │  < release                                  >  │               
-               │  < show                                     >  │               
+               │  < set                                      >  │               
                │                                                │               
 quit: q  help: │ q to close                                     │↥ ↧ Home End   
 filter tasks: T└────────────────────────────────────────────────┘               
diff --git a/tests/integration/tui/screenshots/test_set_mutation.set-command-selected.html b/tests/integration/tui/screenshots/test_set_mutation.set-command-selected.html
new file mode 100644
index 00000000000..8f27deac20e
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_set_mutation.set-command-selected.html
@@ -0,0 +1,16 @@
+
Cylc Tui   work┌────────────────────────────────────────────────┐               
+               │  id: 1/a                                       │               
+- ~cylc        │                                                │               
+   - one - paus│  Action                                        │               
+      - ̿○ 1    │  < (cancel)                                 >  │               
+           ̿○ a │                                                │               
+           ○ z │  < hold                                     >  │               
+               │  < kill                                     >  │               
+               │  < log                                      >  │               
+               │  < poll                                     >  │               
+               │  < release                                  >  │               
+               │  < set                                      >  │               
+               │                                                │               
+quit: q  help: │ q to close                                     │↥ ↧ Home End   
+filter tasks: T└────────────────────────────────────────────────┘               
+

\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_set_mutation.task-state-updated.html b/tests/integration/tui/screenshots/test_set_mutation.task-state-updated.html
new file mode 100644
index 00000000000..b99e16cf6ea
--- /dev/null
+++ b/tests/integration/tui/screenshots/test_set_mutation.task-state-updated.html
@@ -0,0 +1,16 @@
+
Cylc Tui   workflows filtered (W - edit, E - reset)                             
+                                                                                
+- ~cylc                                                                         
+   - one - paused 1■                                                            
+      - ̿○ 1                                                                     
+           ● a                                                                  
+           ̿○ z                                                                  
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+                                                                                
+quit: q  help: h  context: enter  tree: - ← + →  navigation: ↑ ↓ ↥ ↧ Home End   
+filter tasks: T f s r R  filter workflows: W E p                                
+

\ No newline at end of file
diff --git a/tests/integration/tui/screenshots/test_show.fail.html b/tests/integration/tui/screenshots/test_show.fail.html
index f788e5b3a55..66a4836bc01 100644
--- a/tests/integration/tui/screenshots/test_show.fail.html
+++ b/tests/integration/tui/screenshots/test_show.fail.html
@@ -21,7 +21,7 @@
                │  < │                                                │          
                │  < │                                                │          
                │  < │                                                │          
-               │    │                                                │          
+               │  < │                                                │          
                │    │                                                │          
                │    │                                                │          
                │    │                                                │          
diff --git a/tests/integration/tui/test_app.py b/tests/integration/tui/test_app.py
index 908866fc948..b9f50d7d027 100644
--- a/tests/integration/tui/test_app.py
+++ b/tests/integration/tui/test_app.py
@@ -18,13 +18,7 @@
 import pytest
 import urwid
 
-from cylc.flow.cycling.integer import IntegerPoint
-from cylc.flow.task_state import (
-#     TASK_STATUS_RUNNING,
-    TASK_STATUS_SUCCEEDED,
-#     TASK_STATUS_FAILED,
-#     TASK_STATUS_WAITING,
-)
+from cylc.flow.task_outputs import TASK_OUTPUT_SUCCEEDED
 from cylc.flow.workflow_status import StopMode
 
 
@@ -329,12 +323,16 @@ async def test_auto_expansion(flow, scheduler, start, rakiura):
                 'on-load',
                 'cycle "1" and top-level family "1/A" should be expanded',
             )
-
             for task in ('a', 'b'):
-                itask = schd.pool.get_task(IntegerPoint('1'), task)
-                itask.state_reset(TASK_STATUS_SUCCEEDED)
-                schd.pool.spawn_on_output(itask, TASK_STATUS_SUCCEEDED)
+                schd.pool.set_prereqs_and_outputs(
+                    items=[f"1/{task}"],
+                    outputs=[TASK_OUTPUT_SUCCEEDED],
+                    prereqs=None,
+                    flow=['all']
+                )
+
             await schd.update_data_structure()
+            schd.update_data_store()
 
             rk.compare_screenshot(
                 'later-time',
diff --git a/tests/integration/tui/test_mutations.py b/tests/integration/tui/test_mutations.py
index 18da88d227d..844a0654ab1 100644
--- a/tests/integration/tui/test_mutations.py
+++ b/tests/integration/tui/test_mutations.py
@@ -27,7 +27,32 @@ async def gen_commands(schd):
     while True:
         await asyncio.sleep(0.1)
         if not schd.command_queue.empty():
-            yield schd.command_queue.get()
+            # (ignore first item: command UUID)
+            yield schd.command_queue.get()[1:]
+
+
+async def process_command(schd, tries=10, interval=0.1):
+    """Wait for command(s) to be queued and run.
+
+    Waits for at least one command to be queued and for all queued commands to
+    be run.
+    """
+    # wait for the command to be queued
+    for _try in range(tries):
+        await asyncio.sleep(interval)
+        if not schd.command_queue.empty():
+            break
+    else:
+        raise Exception(f'No command was queued after {tries * interval}s')
+
+    # run the command
+    await schd.process_command_queue()
+
+    # push out updates
+    await schd.update_data_structure()
+
+    # make sure it ran
+    assert schd.command_queue.empty(), 'command queue has not emptied'
 
 
 async def test_online_mutation(
@@ -76,7 +101,7 @@ async def test_online_mutation(
             command = None
             async for command in gen_commands(schd):
                 break
-            assert command == ('hold', (['1/one'],), {})
+            assert command == ('hold', [], {'tasks': ['1/one']})
 
         # close the dialogue and re-run the hold mutation
         rk.user_input('q', 'q', 'enter')
@@ -214,3 +239,55 @@ async def test_offline_mutation(
             'there should be a box displaying the error containing the stderr'
             ' returned by the command',
         )
+
+
+async def test_set_mutation(
+    flow,
+    scheduler,
+    start,
+    rakiura,
+):
+    id_ = flow({
+        'scheduling': {
+            'graph': {
+                'R1': 'a => z'
+            },
+        },
+    }, name='one')
+    schd = scheduler(id_)
+    async with start(schd):
+        await schd.update_data_structure()
+        with rakiura(schd.tokens.id, size='80,15') as rk:
+            # open the context menu on 1/a
+            rk.user_input('down', 'down', 'down', 'enter')
+            rk.force_update()
+
+            # select the "set" mutation
+            rk.user_input(*(('down',) * 6))  # 6th command down
+
+            rk.compare_screenshot(
+                # take a screenshot to ensure we have focused on the mutation
+                # successfully
+                'set-command-selected',
+                'The command menu should be open for the task 1/a with the'
+                ' set command selected.'
+            )
+
+            # issue the "set" mutation
+            rk.user_input('enter')
+
+            # wait for the command to be received and run it
+            await process_command(schd)
+
+            # close the error dialogue
+            # NOTE: This hides an asyncio error that does not occur outside of
+            #       the tests
+            rk.user_input('q', 'q', 'q')
+
+            rk.compare_screenshot(
+                # take a screenshot to ensure we have focused on the mutation
+                # successfully
+                'task-state-updated',
+                '1/a should now show as succeeded,'
+                ' there should be no associated job.'
+            )
diff --git a/tests/integration/tui/test_show.py b/tests/integration/tui/test_show.py
index c664cdd1393..063fd4b8781 100644
--- a/tests/integration/tui/test_show.py
+++ b/tests/integration/tui/test_show.py
@@ -48,7 +48,7 @@ async def test_show(flow, scheduler, start, rakiura, monkeypatch):
             rk.user_input('down', 'down', 'enter')
 
             # select the "show" context option
-            rk.user_input(*(['down'] * 6), 'enter')
+            rk.user_input(*(['down'] * 7), 'enter')
             rk.compare_screenshot(
                 'success',
                 'the show output should be displayed',
@@ -63,7 +63,7 @@ def cli_cmd_fail(*args, **kwargs):
             )
 
             # select the "show" context option
-            rk.user_input('q', 'enter', *(['down'] * 6), 'enter')
+            rk.user_input('q', 'enter', *(['down'] * 7), 'enter')
             rk.compare_screenshot(
                 'fail',
                 'the error should be displayed',
diff --git a/tests/integration/validate/test_outputs.py b/tests/integration/validate/test_outputs.py
index a91393366b5..5675372a09f 100644
--- a/tests/integration/validate/test_outputs.py
+++ b/tests/integration/validate/test_outputs.py
@@ -50,6 +50,7 @@
                 'foo+bar',
                 # keywords
                 'required',
+                'optional',
                 'all',
                 # built-in qualifiers
                 'succeeded',
diff --git a/tests/unit/scripts/test_completion_server.py b/tests/unit/scripts/test_completion_server.py
index 186e13b7272..975ac946da6 100644
--- a/tests/unit/scripts/test_completion_server.py
+++ b/tests/unit/scripts/test_completion_server.py
@@ -20,6 +20,7 @@
 from cylc.flow.id import Tokens
 from cylc.flow.network.scan import scan
 from cylc.flow.scripts.completion_server import (
+    _list_prereqs_and_outputs,
     server,
     complete_cylc,
     complete_command,
@@ -540,7 +541,7 @@ async def test_list_dir(tmp_path, monkeypatch):
     # => list $PWD
     assert {
         str(path)
-        for path in await _list_dir(None, None)
+        for path in await _list_dir(None)
     } == {'x/'}
 
     # no trailing `/` at the end of the path
@@ -548,7 +549,7 @@ async def test_list_dir(tmp_path, monkeypatch):
     # => list the parent
     assert {
         str(path)
-        for path in await _list_dir(None, 'x')
+        for path in await _list_dir('x')
     } == {'x/'}
 
     # # trailing `/` at the end of the path
@@ -556,14 +557,14 @@ async def test_list_dir(tmp_path, monkeypatch):
     # # => list dir path
     assert {
         str(path)
-        for path in await _list_dir(None, 'x/')
+        for path in await _list_dir('x/')
     } == {'x/y/', 'x/z'}  # "y" is a dir, "z" is a file
 
     # listing a file
     # => noting to list, just return the file
     assert {
         str(path)
-        for path in await _list_dir(None, 'x/z/')
+        for path in await _list_dir('x/z/')
     } == {'x/z'}
 
     # --- absolute paths ---
@@ -574,7 +575,7 @@ async def test_list_dir(tmp_path, monkeypatch):
     assert {
         # '/'.join(path.rsplit('/', 2)[-2:])
         path.replace(str(tmp_path), '')
-        for path in await _list_dir(None, str(tmp_path / 'x'))
+        for path in await _list_dir(str(tmp_path / 'x'))
     } == {'/x/'}
 
     # trailing `/` at the end of the path
@@ -582,14 +583,14 @@ async def test_list_dir(tmp_path, monkeypatch):
     # => list dir path
     assert {
         path.replace(str(tmp_path), '')
-        for path in await _list_dir(None, str(tmp_path / 'x') + '/')
+        for path in await _list_dir(str(tmp_path / 'x') + '/')
     } == {'/x/y/', '/x/z'}  # "y" is a dir, "z" is a file
 
     # listing a file
     # => noting to list, just return the file
     assert {
         path.replace(str(tmp_path), '')
-        for path in await _list_dir(None, str(tmp_path / 'x' / 'z') + '/')
+        for path in await _list_dir(str(tmp_path / 'x' / 'z') + '/')
     } == {'/x/z'}
 
 
@@ -599,12 +600,12 @@ async def test_list_flows():
     Currently this only provides the textural options i.e. it doesn't list
     "flows" running in a workflow, yet...
     """
-    assert 'all' in await list_flows(None, None)
+    assert 'all' in await list_flows(None)
 
 
 async def test_list_colours():
     """Test listing values for the --color option."""
-    assert 'always' in await list_colours(None, None)
+    assert 'always' in await list_colours(None)
 
 
 async def test_cli_detokenise():
@@ -715,3 +716,18 @@ def _get_current_completion_script_version(_script, lang):
     out, err = capsys.readouterr()
     assert not out  # never write to stdout
     assert not err
+
+
+async def test_prereqs_and_outputs():
+    """Test the error cases for listing task prereqs/outputs.
+
+    The succeess cases are tested in an integration test (requires a running
+    scheduler).
+    """
+    # if no tokens are provided, no prereqs or outputs are returned
+    assert await _list_prereqs_and_outputs([]) == ([], [])
+
+    # if an invalid workflow is provided, we can't list anything
+    assert await _list_prereqs_and_outputs(
+        [Tokens(workflow='no-such-workflow')]
+    ) == ([], [])
diff --git a/tests/unit/scripts/test_trigger.py b/tests/unit/scripts/test_trigger.py
index d464bda0a34..87f392d73f8 100644
--- a/tests/unit/scripts/test_trigger.py
+++ b/tests/unit/scripts/test_trigger.py
@@ -18,12 +18,17 @@
 
 from optparse import Values
 import pytest
-from typing import Iterable, Optional, Tuple, Type
+from typing import Optional, Tuple, Type
 
 from cylc.flow.exceptions import InputError
 from cylc.flow.option_parsers import Options
-from cylc.flow.flow_mgr import FLOW_ALL, FLOW_NEW, FLOW_NONE
-from cylc.flow.scripts.trigger import get_option_parser, _validate
+from cylc.flow.flow_mgr import (
+    FLOW_ALL,
+    FLOW_NEW,
+    FLOW_NONE,
+    validate_flow_opts
+)
+from cylc.flow.scripts.trigger import get_option_parser
 
 
 Opts = Options(get_option_parser())
@@ -39,6 +44,13 @@
             ),
             None
         ),
+        (
+            Opts(
+                flow=None,
+                flow_wait=False
+            ),
+            None
+        ),
         (
             Opts(
                 flow=[FLOW_NEW],
@@ -57,17 +69,6 @@
                 "Multiple flow options must all be integer valued"
             )
         ),
-        (
-            Opts(
-                flow=[FLOW_ALL],
-                flow_wait=False,
-                flow_descr="the quick brown fox"
-            ),
-            (
-                InputError,
-                "Metadata is only for new flows"
-            )
-        ),
         (
             Opts(
                 flow=["cheese"],
@@ -75,7 +76,7 @@
             ),
             (
                 InputError,
-                "Flow values must be integer, 'all', 'new', or 'none'"
+                "Flow values must be an integer, or 'all', 'new', or 'none'"
             )
         ),
         (
@@ -117,7 +118,7 @@ def test_validate(
     if expected_err:
         err, msg = expected_err
         with pytest.raises(err) as exc:
-            _validate(opts)
+            validate_flow_opts(opts)
         assert msg in str(exc.value)
     else:
-        _validate(opts)
+        validate_flow_opts(opts)
diff --git a/tests/unit/test_flow_mgr.py b/tests/unit/test_flow_mgr.py
index 5fea08c1b97..c9171b02073 100644
--- a/tests/unit/test_flow_mgr.py
+++ b/tests/unit/test_flow_mgr.py
@@ -26,6 +26,7 @@
 
 
 FAKE_NOW = datetime.datetime(2020, 12, 25, 17, 5, 55)
+FAKE_NOW_ISO = FAKE_NOW.isoformat()
 
 
 @pytest.fixture
@@ -33,7 +34,7 @@ def patch_datetime_now(monkeypatch):
 
     class mydatetime:
         @classmethod
-        def now(cls):
+        def now(cls, tz=None):
             return FAKE_NOW
 
     monkeypatch.setattr(datetime, 'datetime', mydatetime)
@@ -43,25 +44,53 @@ def test_all(
     patch_datetime_now,
     caplog: pytest.LogCaptureFixture,
 ):
+    """Test flow number management."""
+
     db_mgr = WorkflowDatabaseManager()
     flow_mgr = FlowMgr(db_mgr)
     caplog.set_level(logging.INFO, CYLC_LOG)
 
-    count = 1
     meta = "the quick brown fox"
-    msg1 = f"flow: {count} ({meta}) {FAKE_NOW}"
-    assert flow_mgr.get_new_flow(meta) == count
+    assert flow_mgr.get_flow_num(None, meta) == 1
+    msg1 = f"flow: 1 ({meta}) {FAKE_NOW_ISO}"
     assert f"New {msg1}" in caplog.messages
 
-    count = 2
-    meta = "jumped over the lazy"
-    msg2 = f"flow: {count} ({meta}) {FAKE_NOW}"
-    assert flow_mgr.get_new_flow(meta) == count
+    # automatic: expect 2
+    meta = "jumped over the lazy dog"
+    assert flow_mgr.get_flow_num(None, meta) == 2
+    msg2 = f"flow: 2 ({meta}) {FAKE_NOW_ISO}"
     assert f"New {msg2}" in caplog.messages
 
+    # give flow 2: not a new flow
+    meta = "jumped over the moon"
+    assert flow_mgr.get_flow_num(2, meta) == 2
+    msg3 = f"flow: 2 ({meta}) {FAKE_NOW_ISO}"
+    assert f"New {msg3}" not in caplog.messages
+    assert f"Ignoring flow metadata \"{meta}\": 2 is not a new flow" in caplog.messages
+
+    # give flow 4: new flow
+    meta = "jumped over the cheese"
+    assert flow_mgr.get_flow_num(4, meta) == 4
+    msg4 = f"flow: 4 ({meta}) {FAKE_NOW_ISO}"
+    assert f"New {msg4}" in caplog.messages
+
+    # automatic: expect 3
+    meta = "jumped over the log"
+    assert flow_mgr.get_flow_num(None, meta) == 3
+    msg5 = f"flow: 3 ({meta}) {FAKE_NOW_ISO}"
+    assert f"New {msg5}" in caplog.messages
+
+    # automatic: expect 5 (skip over 4)
+    meta = "crawled under the log"
+    assert flow_mgr.get_flow_num(None, meta) == 5
+    msg6 = f"flow: 5 ({meta}) {FAKE_NOW_ISO}"
+    assert f"New {msg6}" in caplog.messages
     flow_mgr._log()
     assert (
         "Flows:\n"
         f"{msg1}\n"
-        f"{msg2}"
+        f"{msg2}\n"
+        f"{msg4}\n"
+        f"{msg5}\n"
+        f"{msg6}"
     ) in caplog.messages
diff --git a/tests/unit/test_graph_parser.py b/tests/unit/test_graph_parser.py
index ddd443a3597..ca19b2060b4 100644
--- a/tests/unit/test_graph_parser.py
+++ b/tests/unit/test_graph_parser.py
@@ -778,6 +778,10 @@ def test_family_optional_outputs(qual, task_output):
             "FAM => foo",  # bare family on LHS
             "Illegal family trigger"
         ],
+        [
+            "FAM:expire-all => foo",
+            "must be optional"
+        ],
     ]
 )
 def test_family_trigger_errors(graph, error):
@@ -819,6 +823,10 @@ def test_family_trigger_errors(graph, error):
             "a:finish? => b",
             "Pseudo-output a:finished can't be optional",
         ],
+        [
+            "a:expire => b",
+            "must be optional",
+        ],
     ]
 )
 def test_task_optional_output_errors_order(
diff --git a/tests/unit/test_id_match.py b/tests/unit/test_id_match.py
index d26e85b092d..8727023a864 100644
--- a/tests/unit/test_id_match.py
+++ b/tests/unit/test_id_match.py
@@ -43,6 +43,7 @@ def _task_proxy(id_, hier):
         hier.append('root')
         tdef = create_autospec(TaskDef, namespace_hierarchy=hier)
         tdef.name = tokens['task']
+        tdef.expiration_offset = None
         return TaskProxy(
             Tokens('~user/workflow'),
             tdef,
@@ -127,7 +128,7 @@ def test_filter_ids_task_mode(task_pool, ids, matched, not_matched):
         {}
     )
 
-    _matched, _not_matched = filter_ids([pool], ids)
+    _matched, _not_matched = filter_ids(pool, ids)
     assert [get_task_id(itask) for itask in _matched] == matched
     assert _not_matched == not_matched
 
@@ -188,21 +189,21 @@ def test_filter_ids_cycle_mode(task_pool, ids, matched, not_matched):
         {}
     )
 
-    _matched, _not_matched = filter_ids([pool], ids, out=IDTokens.Cycle)
+    _matched, _not_matched = filter_ids(pool, ids, out=IDTokens.Cycle)
     assert _matched == [IntegerPoint(i) for i in matched]
     assert _not_matched == not_matched
 
 
 def test_filter_ids_invalid(caplog):
     """Ensure invalid IDs are handled elegantly."""
-    matched, not_matched = filter_ids([{}], ['#'])
+    matched, not_matched = filter_ids({}, ['#'])
     assert matched == []
     assert not_matched == ['#']
     assert caplog.record_tuples == [
         ('cylc', 30, 'No active tasks matching: #'),
     ]
     caplog.clear()
-    matched, not_matched = filter_ids([{}], ['#'], warn=False)
+    matched, not_matched = filter_ids({}, ['#'], warn=False)
     assert caplog.record_tuples == []
 
 
@@ -216,7 +217,7 @@ def test_filter_ids_pattern_match_off(task_pool):
     )
 
     _matched, _not_matched = filter_ids(
-        [pool],
+        pool,
         ['1/a'],
         out=IDTokens.Task,
         pattern_match=False,
@@ -238,7 +239,7 @@ def test_filter_ids_toggle_pattern_matching(task_pool, caplog):
 
     # ensure pattern matching works
     _matched, _not_matched = filter_ids(
-        [pool],
+        pool,
         ids,
         out=IDTokens.Task,
         pattern_match=True,
@@ -249,7 +250,7 @@ def test_filter_ids_toggle_pattern_matching(task_pool, caplog):
     # ensure pattern matching can be disabled
     caplog.clear()
     _matched, _not_matched = filter_ids(
-        [pool],
+        pool,
         ids,
         out=IDTokens.Task,
         pattern_match=False,
@@ -285,7 +286,7 @@ def test_filter_ids_namespace_hierarchy(task_pool, ids, matched, not_matched):
     )
 
     _matched, _not_matched = filter_ids(
-        [pool],
+        pool,
         ids,
         pattern_match=False,
     )
diff --git a/tests/unit/test_prerequisite.py b/tests/unit/test_prerequisite.py
index d8981c2febf..410176e508c 100644
--- a/tests/unit/test_prerequisite.py
+++ b/tests/unit/test_prerequisite.py
@@ -18,6 +18,7 @@
 
 from cylc.flow.cycling.loader import ISO8601_CYCLING_TYPE, get_point
 from cylc.flow.prerequisite import Prerequisite
+from cylc.flow.id import Tokens
 
 
 @pytest.fixture
@@ -61,10 +62,10 @@ def test_satisfied(prereq):
         ('2001', 'd', 'custom'): False,
     }
     # mark two prerequisites as satisfied
-    prereq.satisfy_me({
-        ('2000', 'b', 'succeeded'),
-        ('2000', 'c', 'succeeded'),
-    })
+    prereq.satisfy_me([
+        Tokens('2000/b:succeeded', relative=True),
+        Tokens('2000/c:succeeded', relative=True),
+    ])
     assert prereq.satisfied == {
         # the pre-initial dependency should be marked as satisfied
         ('1999', 'a', 'succeeded'): 'satisfied naturally',
diff --git a/tests/unit/test_subprocpool.py b/tests/unit/test_subprocpool.py
index feffdab19d0..c72ffc4d094 100644
--- a/tests/unit/test_subprocpool.py
+++ b/tests/unit/test_subprocpool.py
@@ -328,6 +328,7 @@ def test__run_command_exit_add_to_badhosts_log(caplog, mock_ctx):
             SimpleNamespace(
                 name='t', dependencies={}, sequential='',
                 external_triggers=[], xtrig_labels={},
+                expiration_offset=None,
                 outputs={
                     'submitted': [None, None], 'submit-failed': [None, None]
                 },
diff --git a/tests/unit/test_task_state.py b/tests/unit/test_task_state.py
index 82e19f9e215..e655c74b7bb 100644
--- a/tests/unit/test_task_state.py
+++ b/tests/unit/test_task_state.py
@@ -21,9 +21,12 @@
 from cylc.flow.task_trigger import Dependency, TaskTrigger
 from cylc.flow.task_state import (
     TaskState,
+    TASK_STATUS_PREPARING,
+    TASK_STATUS_SUBMIT_FAILED,
+    TASK_STATUS_SUBMITTED,
     TASK_STATUS_SUCCEEDED,
-    TASK_STATUS_FAILED,
     TASK_STATUS_WAITING,
+    TASK_STATUS_RUNNING,
 )
 
 
@@ -79,58 +82,6 @@ def test_reset(state, is_held, should_reset):
         assert tstate.status == state
 
 
-@pytest.mark.parametrize(
-    'before,after,outputs',
-    [
-        (
-            (TASK_STATUS_WAITING, False),
-            (TASK_STATUS_SUCCEEDED, False),
-            ['submitted', 'started', 'succeeded']
-        ),
-        (
-            (TASK_STATUS_WAITING, False),
-            (TASK_STATUS_FAILED, False),
-            ['submitted', 'started', 'failed']
-        ),
-        (
-            (TASK_STATUS_WAITING, False),
-            (TASK_STATUS_FAILED, None),  # no change to is_held
-            ['submitted', 'started', 'failed']
-        ),
-        (
-            (TASK_STATUS_WAITING, False),
-            (None, False),  # no change to status
-            []
-        ),
-        # only reset task outputs if not setting task to held
-        # https://github.com/cylc/cylc-flow/pull/2116
-        (
-            (TASK_STATUS_WAITING, False),
-            (TASK_STATUS_FAILED, True),
-            []
-        ),
-        # only reset task outputs if not setting task to held
-        # https://github.com/cylc/cylc-flow/pull/2116
-        (
-            (TASK_STATUS_WAITING, False),
-            (TASK_STATUS_SUCCEEDED, True),
-            []
-        )
-    ]
-)
-def test_reset_outputs(before, after, outputs):
-    """Test that outputs are reset correctly on state changes."""
-    tdef = TaskDef('foo', {}, 'live', '123', '123')
-
-    orig_status, orig_is_held = before
-    new_status, new_is_held = after
-
-    tstate = TaskState(tdef, '123', orig_status, orig_is_held)
-    assert tstate.outputs.get_completed() == []
-    tstate.reset(status=new_status, is_held=new_is_held)
-    assert tstate.outputs.get_completed() == outputs
-
-
 def test_task_prereq_duplicates(set_cycling_type):
     """Test prerequisite duplicates from multiple recurrences are discarded."""
 
@@ -152,3 +103,19 @@ def test_task_prereq_duplicates(set_cycling_type):
     prereqs = [p.satisfied for p in tstate.prerequisites]
 
     assert prereqs == [{("1", "a", "succeeded"): False}]
+
+
+def test_task_state_order():
+    """Test is_gt and is_gte methods."""
+
+    tdef = TaskDef('foo', {}, 'live', IntegerPoint("1"), IntegerPoint("1"))
+    tstate = TaskState(tdef, IntegerPoint("1"), TASK_STATUS_SUBMITTED, False)
+
+    assert tstate.is_gt(TASK_STATUS_WAITING)
+    assert tstate.is_gt(TASK_STATUS_PREPARING)
+    assert tstate.is_gt(TASK_STATUS_SUBMIT_FAILED)
+    assert not tstate.is_gt(TASK_STATUS_SUBMITTED)
+    assert tstate.is_gte(TASK_STATUS_SUBMITTED)
+    assert not tstate.is_gt(TASK_STATUS_RUNNING)
+    assert not tstate.is_gte(TASK_STATUS_RUNNING)
+