Support Python 3.12 (#5794)

* add generic Python 3 (i.e. A bleeding edge version) to tests * remove report timings from Python 3.12 * fix flake8 issues * Commented out broken test lines (see #5812) * replace smtpd with aiosmtpd in the test_header file * fix tests --------- Co-authored-by: Ronnie Dutta <[email protected]>
cylc · Nov 13, 2023 · 99fa0f5 · 99fa0f5
1 parent 869edf8
commit 99fa0f5
Show file tree

Hide file tree

Showing 14 changed files with 80 additions and 55 deletions.
diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml
@@ -20,7 +20,7 @@ jobs:
       fail-fast: false # don't stop on first failure
       matrix:
         os: ['ubuntu-latest']
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3']
         include:
           - os: 'macos-latest'
             python-version: '3.7'

diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml
@@ -46,9 +46,9 @@ jobs:
         # NOTE: includes must define ALL of the matrix values
         include:
           # latest python
-          - name: 'py-3.11'
+          - name: 'py-3-latest'
             os: 'ubuntu-latest'
-            python-version: '3.11'
+            python-version: '3'
             test-base: 'tests/f'
             chunk: '1/4'
             platform: '_local_background*'

diff --git a/changes.d/5794.break.md b/changes.d/5794.break.md
@@ -0,0 +1 @@
+Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12.
diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py
@@ -1222,6 +1222,9 @@ def default_for(
 
                    {PLATFORM_REPLACES.format("[job]batch system")}
             ''')
+            replaces = PLATFORM_REPLACES.format(
+                "[job]batch submit command template"
+            )
             Conf('job runner command template', VDR.V_STRING, desc=f'''
                 Set the command used by the chosen job runner.
 
@@ -1230,9 +1233,7 @@ def default_for(
 
                 .. versionadded:: 8.0.0
 
-                   {PLATFORM_REPLACES.format(
-                       "[job]batch submit command template"
-                    )}
+                   {replaces}
             ''')
             Conf('shell', VDR.V_STRING, '/bin/bash', desc='''
 
@@ -1465,16 +1466,19 @@ def default_for(
                    {REPLACES}``global.rc[hosts][<host>]retrieve job logs
                    command``.
             ''')
+            replaces = PLATFORM_REPLACES.format(
+                "[remote]retrieve job logs max size")
             Conf('retrieve job logs max size', VDR.V_STRING, desc=f'''
                 {LOG_RETR_SETTINGS['retrieve job logs max size']}
 
                 .. versionchanged:: 8.0.0
 
                    {REPLACES}``global.rc[hosts][<host>]retrieve job logs
                    max size``.
-                   {PLATFORM_REPLACES.format(
-                       "[remote]retrieve job logs max size")}
+                   {replaces}
             ''')
+            replaces = PLATFORM_REPLACES.format(
+                "[remote]retrieve job logs retry delays")
             Conf('retrieve job logs retry delays', VDR.V_INTERVAL_LIST,
                  desc=f'''
                 {LOG_RETR_SETTINGS['retrieve job logs retry delays']}
@@ -1483,8 +1487,7 @@ def default_for(
 
                    {REPLACES}``global.rc[hosts][<host>]retrieve job logs
                    retry delays``.
-                   {PLATFORM_REPLACES.format(
-                       "[remote]retrieve job logs retry delays")}
+                   {replaces}
             ''')
             Conf('tail command template',
                  VDR.V_STRING, 'tail -n +1 --follow=name %(filename)s',

diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py
@@ -401,7 +401,7 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                     # differentiate between not set vs set to empty
                     default = None
                 elif item.endswith("handlers"):
-                    desc = desc + '\n\n' + dedent(rf'''
+                    desc = desc + '\n\n' + dedent(f'''
                         Examples:
 
                         .. code-block:: cylc
@@ -413,9 +413,9 @@ def get_script_common_text(this: str, example: Optional[str] = None):
                            {item} = echo %(workflow)s
 
                            # configure multiple event handlers
-                           {item} = \
-                               'echo %(workflow)s, %(event)s', \
-                               'my_exe %(event)s %(message)s' \
+                           {item} = \\
+                               'echo %(workflow)s, %(event)s', \\
+                               'my_exe %(event)s %(message)s' \\
                                'curl -X PUT -d event=%(event)s host:port'
                     ''')
                 elif item.startswith("abort on"):
@@ -1856,7 +1856,7 @@ def upg(cfg, descr):
         ['scheduling', 'max active cycle points'],
         ['scheduling', 'runahead limit'],
         cvtr=converter(
-            lambda x: f'P{int(x)-1}' if x != '' else '',
+            lambda x: f'P{int(x) - 1}' if x != '' else '',
             '"{old}" -> "{new}"'
         ),
         silent=cylc.flow.flags.cylc7_back_compat,

diff --git a/cylc/flow/id.py b/cylc/flow/id.py
@@ -497,7 +497,7 @@ def duplicate(
             )?
             (?:
                 # cycle/task/job
-                { RELATIVE_PATTERN }
+                {RELATIVE_PATTERN}
             )?
           )?
         )?

diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py
@@ -54,6 +54,8 @@
 import sys
 from typing import TYPE_CHECKING
 
+
+from cylc.flow import LOG
 from cylc.flow.exceptions import CylcError
 from cylc.flow.id_cli import parse_id
 from cylc.flow.option_parsers import (
@@ -123,6 +125,12 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None:
         constraint='workflows',
     )
 
+    LOG.warning(
+        "cylc report-timings is deprecated."
+        " The analysis view in the GUI provides"
+        " similar functionality."
+    )
+
     output_options = [
         options.show_raw, options.show_summary, options.html_summary
     ]
@@ -246,7 +254,10 @@ def _check_imports(self):
         try:
             import pandas
         except ImportError:
-            raise CylcError('Cannot import pandas - summary unavailable.')
+            raise CylcError(
+                'Cannot import pandas - summary unavailable.'
+                ' try: pip install cylc-flow[report-timings]'
+            )
         else:
             del pandas
 

diff --git a/setup.cfg b/setup.cfg
@@ -52,6 +52,7 @@ classifiers =
     Programming Language :: Python :: 3.9
     Programming Language :: Python :: 3.10
     Programming Language :: Python :: 3.11
+    Programming Language :: Python :: 3.12
     Programming Language :: Python :: 3 :: Only
     Programming Language :: Python :: Implementation :: CPython
     Topic :: Scientific/Engineering :: Atmospheric Science
@@ -102,6 +103,7 @@ report-timings =
     pandas==1.*
     matplotlib
 tests =
+    aiosmtpd
     async_generator
     bandit>=1.7.0
     coverage>=5.0.0,<7.3.1
@@ -139,7 +141,6 @@ all =
     %(main_loop-log_db)s
     %(main_loop-log_main_loop)s
     %(main_loop-log_memory)s
-    %(report-timings)s
     %(tests)s
     %(tutorials)s
 

diff --git a/tests/functional/events/09-task-event-mail.t b/tests/functional/events/09-task-event-mail.t
@@ -50,14 +50,15 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'retry: 1/t1/01'
-b'succeeded: 1/t1/02'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+retry: 1/t1/01
+succeeded: 1/t1/02
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[1/t1/02 succeeded\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/02 succeeded\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill

diff --git a/tests/functional/events/18-workflow-event-mail.t b/tests/functional/events/18-workflow-event-mail.t
@@ -49,11 +49,11 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'event: startup'
-b'message: workflow starting'
-b'event: shutdown'
-b'message: AUTOMATIC'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+event: startup
+message: workflow starting
+event: shutdown
+message: AUTOMATIC
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
 
 purge

diff --git a/tests/functional/events/29-task-event-mail-1.t b/tests/functional/events/29-task-event-mail-1.t
@@ -37,12 +37,14 @@ run_ok "${TEST_NAME_BASE}-validate" \
 workflow_run_ok "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME"
 
+
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'retry: 1/t1/01'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+retry: 1/t1/01
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 
 purge
 mock_smtpd_kill

diff --git a/tests/functional/events/30-task-event-mail-2.t b/tests/functional/events/30-task-event-mail-2.t
@@ -50,27 +50,29 @@ workflow_run_fail "${TEST_NAME_BASE}-run" \
     cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}"
 
 contains_ok "${TEST_SMTPD_LOG}" <<__LOG__
-b'retry: 1/t1/01'
-b'retry: 1/t2/01'
-b'retry: 1/t3/01'
-b'retry: 1/t4/01'
-b'retry: 1/t5/01'
-b'retry: 1/t1/02'
-b'retry: 1/t2/02'
-b'retry: 1/t3/02'
-b'retry: 1/t4/02'
-b'retry: 1/t5/02'
-b'failed: 1/t1/03'
-b'failed: 1/t2/03'
-b'failed: 1/t3/03'
-b'failed: 1/t4/03'
-b'failed: 1/t5/03'
-b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/'
+retry: 1/t1/01
+retry: 1/t2/01
+retry: 1/t3/01
+retry: 1/t4/01
+retry: 1/t5/01
+retry: 1/t1/02
+retry: 1/t2/02
+retry: 1/t3/02
+retry: 1/t4/02
+retry: 1/t5/02
+failed: 1/t1/03
+failed: 1/t2/03
+failed: 1/t3/03
+failed: 1/t4/03
+failed: 1/t5/03
+see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/
 __LOG__
+
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[. tasks retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+    grep -qPizo "Subject: \[. tasks retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
 run_ok "${TEST_NAME_BASE}-grep-log" \
-    grep -q "Subject: \\[. tasks failed\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
-purge
+    grep -qPizo "Subject: \[. tasks failed\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}"
+
+    purge
 mock_smtpd_kill
 exit
diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header
@@ -807,10 +807,13 @@ mock_smtpd_init() {  # Logic borrowed from Rose
         local SMTPD_LOG="${TEST_DIR}/smtpd.log"
         local SMTPD_HOST="localhost:${SMTPD_PORT}"
         # Set up fake SMTP server to catch outgoing mail & redirect to log:
-        python3 -u -m 'smtpd' -c 'DebuggingServer' -d -n "${SMTPD_HOST}" \
+        python3 -u -m 'aiosmtpd' \
+            --class aiosmtpd.handlers.Debugging stdout \
+            --debug --nosetuid \
+            --listen "${SMTPD_HOST}" \
             1>"${SMTPD_LOG}" 2>&1 &  # Runs in background
         local SMTPD_PID="$!"
-        while ! grep -q 'DebuggingServer started' "${SMTPD_LOG}" 2>'/dev/null'
+        while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null'
         do
             if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then
                 sleep 1  # Still waiting for fake server to start

diff --git a/tests/unit/test_indep_task_queues.py b/tests/unit/test_indep_task_queues.py
@@ -92,9 +92,10 @@ def test_queue_and_release(
     assert sorted([r.tdef.name for r in released]) == sorted(expected_released)
 
     # check released tasks change state to "preparing", and not is_queued
-    for r in released:
-        assert r.state.reset.called_with(TASK_STATUS_PREPARING)
-        assert r.state.reset.called_with(is_queued=False)
+    # Commented out pending https://github.com/cylc/cylc-flow/issues/5812
+    # for r in released:
+    #     assert r.state.reset.called_with(TASK_STATUS_PREPARING)
+    #     assert r.state.reset.called_with(is_queued=False)
 
     # check that adopted orphans end up in the default queue
     orphans = ["orphan1", "orphan2"]