Merge branch 'master' into cylc-set-task

.github/workflows/branch_sync.yml

@@ -14,109 +14,7 @@ on:
 
 jobs:
   sync:
-    runs-on: ubuntu-latest
-    timeout-minutes: 5
-    env:
-      BASE_BRANCH: master
-      HEAD_BRANCH: ${{ inputs.head_branch || github.ref_name }}
-      STATUS_JSON: https://raw.githubusercontent.com/cylc/cylc-admin/master/docs/status/branches.json
-      FORCE_COLOR: 2
-    steps:
-      - name: Check branch name
-        shell: python
-        run: |
-          import os
-          import json
-          import sys
-          from urllib.request import urlopen
-
-          if os.environ['GITHUB_EVENT_NAME'] == 'schedule':
-            # Get branch from status page
-            meta = json.loads(
-              urlopen(os.environ['STATUS_JSON']).read()
-            )['meta_releases']
-            version = min(meta)
-            branch = meta[version][os.environ['GITHUB_REPOSITORY']]
-          else:
-            branch = os.environ['HEAD_BRANCH'].strip()
-
-          if branch.endswith('-sync'):
-            sys.exit("::error::Do not run this workflow for already-created sync branches")
-
-          with open(os.environ['GITHUB_ENV'], 'a') as F:
-            print(f'HEAD_BRANCH={branch}', file=F)
-            print(f'SYNC_BRANCH={branch}-sync', file=F)
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          ref: master
-
-      - name: Configure git
-        uses: cylc/release-actions/configure-git@v1
-
-      - name: Checkout sync branch if it exists
-        continue-on-error: true
-        run: |
-          git switch -c "$SYNC_BRANCH" "origin/${SYNC_BRANCH}"
-          echo "BASE_BRANCH=$SYNC_BRANCH" >> "$GITHUB_ENV"
-
-      - name: Attempt fast-forward
-        id: ff
-        run: |
-          if git merge "origin/${HEAD_BRANCH}" --ff-only; then
-            if [[ "$(git rev-parse HEAD)" == "$(git rev-parse "origin/${BASE_BRANCH}")" ]]; then
-              echo "::notice::$BASE_BRANCH is up to date with $HEAD_BRANCH"
-              exit 0
-            fi
-            git push origin "$BASE_BRANCH"
-          elif [[ "$BASE_BRANCH" == "$SYNC_BRANCH" ]]; then
-            echo "::notice::Cannot fast-forward $BASE_BRANCH to $HEAD_BRANCH; merge existing PR first"
-          else
-            echo "continue=true" >> "$GITHUB_OUTPUT"
-          fi
-
-      - name: Attempt merge into master
-        id: merge
-        if: steps.ff.outputs.continue
-        run: |
-          git switch master
-          if git merge "origin/${HEAD_BRANCH}"; then
-            if git diff HEAD^ --exit-code --stat; then
-              echo "::notice::No diff between master and $HEAD_BRANCH"
-              exit 0
-            fi
-          else
-            git merge --abort
-          fi
-          echo "continue=true" >> $GITHUB_OUTPUT
-
-      - name: Push sync branch
-        id: push
-        if: steps.merge.outputs.continue
-        run: |
-          git switch -c "$SYNC_BRANCH" "origin/${HEAD_BRANCH}"
-          git push origin "$SYNC_BRANCH"
-          echo "continue=true" >> $GITHUB_OUTPUT
-
-      - name: Open PR
-        if: steps.push.outputs.continue
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          BODY: |
-            Please do a **normal merge**, not squash merge.
-            Please fix conflicts if necessary.
-
-            ---
-
-            Triggered by `${{ github.event_name }}`
-        run: |
-          url="$(
-            gh pr create --head "$SYNC_BRANCH" \
-            --title "🤖 Merge ${SYNC_BRANCH} into master" \
-            --body "$BODY"
-          )"
-          echo "::notice::PR created at ${url}"
-
-          gh pr edit "$SYNC_BRANCH" --add-label "sync" || true
+    uses: cylc/release-actions/.github/workflows/branch-sync.yml@v1
+    with:
+      head_branch: ${{ inputs.head_branch }}
+    secrets: inherit

CONTRIBUTING.md

@@ -64,7 +64,7 @@ requests_).
  - Prasanna Challuri
  - David Matthews
  - Tim Whitcomb
- - (Scott Wales)
+ - Scott Wales
  - Tomek Trzeciak
  - Thomas Coleman
  - Bruno Kinoshita

changes.d/5709.feat.md

@@ -0,0 +1 @@
+Forward arbitrary environment variables over SSH connections

changes.d/5727.break.md

@@ -0,0 +1 @@
+Cylc now ignores `PYTHONPATH` to make it more robust to task environments which set this value. If you want to add to the Cylc environment itself, e.g. to install a Cylc extension, use `CYLC_PYTHONPATH`.

changes.d/5776.fix.md

@@ -0,0 +1 @@
+Ensure that submit-failed tasks are marked as incomplete (so remain visible) when running in back-compat mode.

conda-environment.yml

@@ -9,7 +9,7 @@ dependencies:
   - graphviz  # for static graphing
   # Note: can't pin jinja2 any higher than this until we give up on Cylc 7 back-compat
   - jinja2 >=3.0,<3.1
-  - metomi-isodatetime >=1!3.0.0, <1!3.1.0
+  - metomi-isodatetime >=1!3.0.0, <1!3.2.0
   # Constrain protobuf version for compatible Scheduler-UIS comms across hosts
   - protobuf >=4.21.2,<4.22.0
   - psutil >=5.6.0

cylc/flow/cfgspec/globalcfg.py

@@ -1652,6 +1652,14 @@ def default_for(
 
                 .. versionadded:: 8.0.0
             ''')
+            Conf('ssh forward environment variables', VDR.V_STRING_LIST, '',
+                 desc='''
+                A list containing the names of the environment variables to
+                forward with SSH connections to the workflow host from
+                the host running 'cylc play'
+
+                .. versionadded:: 8.3.0
+            ''')
             with Conf('selection', desc='''
                 How to select a host from the list of platform hosts.
 

cylc/flow/cfgspec/workflow.py

@@ -1260,10 +1260,17 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                     - ``all`` - all instance of the task will fail
                     - ``2017-08-12T06, 2017-08-12T18`` - these instances of
                       the task will fail
+
+                    If you set :cylc:conf:`[..][..]execution retry delays`
+                    the second attempt will succeed unless you set
+                    :cylc:conf:`[..]fail try 1 only = False`.
                 ''')
                 Conf('fail try 1 only', VDR.V_BOOLEAN, True, desc='''
                     If ``True`` only the first run of the task
                     instance will fail, otherwise retries will fail too.
+
+                    Task instances must be set to fail by
+                    :cylc:conf:`[..]fail cycle points`.
                 ''')
                 Conf('disable task event handlers', VDR.V_BOOLEAN, True,
                      desc='''

cylc/flow/config.py

@@ -79,8 +79,8 @@
     get_cylc_run_dir,
     is_relative_to,
 )
-from cylc.flow.platforms import FORBIDDEN_WITH_PLATFORM
 from cylc.flow.print_tree import print_tree
+from cylc.flow.simulation import configure_sim_modes
 from cylc.flow.subprocctx import SubFuncContext
 from cylc.flow.task_events_mgr import (
     EventData,
@@ -521,7 +521,8 @@ def __init__(
         self.process_runahead_limit()
 
         if self.run_mode('simulation', 'dummy'):
-            self.configure_sim_modes()
+            configure_sim_modes(
+                self.taskdefs.values(), self.run_mode())
 
         self.configure_workflow_state_polling_tasks()
 
@@ -1340,68 +1341,6 @@ def configure_workflow_state_polling_tasks(self):
             script = "echo " + comstr + "\n" + comstr
             rtc['script'] = script
 
-    def configure_sim_modes(self):
-        """Adjust task defs for simulation and dummy mode."""
-        for tdef in self.taskdefs.values():
-            # Compute simulated run time by scaling the execution limit.
-            rtc = tdef.rtconfig
-            limit = rtc['execution time limit']
-            speedup = rtc['simulation']['speedup factor']
-            if limit and speedup:
-                sleep_sec = (DurationParser().parse(
-                    str(limit)).get_seconds() / speedup)
-            else:
-                sleep_sec = DurationParser().parse(
-                    str(rtc['simulation']['default run length'])
-                ).get_seconds()
-            rtc['execution time limit'] = (
-                sleep_sec + DurationParser().parse(str(
-                    rtc['simulation']['time limit buffer'])).get_seconds()
-            )
-            rtc['job']['simulated run length'] = sleep_sec
-
-            # Generate dummy scripting.
-            rtc['init-script'] = ""
-            rtc['env-script'] = ""
-            rtc['pre-script'] = ""
-            rtc['post-script'] = ""
-            scr = "sleep %d" % sleep_sec
-            # Dummy message outputs.
-            for msg in rtc['outputs'].values():
-                scr += "\ncylc message '%s'" % msg
-            if rtc['simulation']['fail try 1 only']:
-                arg1 = "true"
-            else:
-                arg1 = "false"
-            arg2 = " ".join(rtc['simulation']['fail cycle points'])
-            scr += "\ncylc__job__dummy_result %s %s || exit 1" % (arg1, arg2)
-            rtc['script'] = scr
-
-            # Dummy mode jobs should run on platform localhost
-            # All Cylc 7 config items which conflict with platform are removed.
-            for section, keys in FORBIDDEN_WITH_PLATFORM.items():
-                if section in rtc:
-                    for key in keys:
-                        if key in rtc[section]:
-                            rtc[section][key] = None
-
-            rtc['platform'] = 'localhost'
-
-            # Disable environment, in case it depends on env-script.
-            rtc['environment'] = {}
-
-            # Simulation mode tasks should fail in which cycle points?
-            f_pts = []
-            f_pts_orig = rtc['simulation']['fail cycle points']
-            if 'all' in f_pts_orig:
-                # None for "fail all points".
-                f_pts = None
-            else:
-                # (And [] for "fail no points".)
-                for point_str in f_pts_orig:
-                    f_pts.append(get_point(point_str).standardise())
-            rtc['simulation']['fail cycle points'] = f_pts
-
     def get_parent_lists(self):
         return self.runtime['parents']