diff --git a/.github/workflows/test_fast.yml b/.github/workflows/test_fast.yml index d0bff50326d..49f2f44b51f 100644 --- a/.github/workflows/test_fast.yml +++ b/.github/workflows/test_fast.yml @@ -20,7 +20,7 @@ jobs: fail-fast: false # Don't let a failed MacOS run stop the Ubuntu runs matrix: os: ['ubuntu-latest'] - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3'] include: - os: 'macos-latest' python-version: '3.7' diff --git a/.github/workflows/test_functional.yml b/.github/workflows/test_functional.yml index 75dac06a964..baaad6b61f3 100644 --- a/.github/workflows/test_functional.yml +++ b/.github/workflows/test_functional.yml @@ -46,9 +46,9 @@ jobs: # NOTE: includes must define ALL of the matrix values include: # latest python - - name: 'py-3.11' + - name: 'py-3-latest' os: 'ubuntu-latest' - python-version: '3.11' + python-version: '3' test-base: 'tests/f' chunk: '1/4' platform: '_local_background*' diff --git a/changes.d/5794.break.md b/changes.d/5794.break.md new file mode 100644 index 00000000000..53c5315b013 --- /dev/null +++ b/changes.d/5794.break.md @@ -0,0 +1 @@ +Remove `cylc report-timings` from automatic installation with `pip install cylc-flow[all]`. If you now wish to install it use `pip install cylc-flow[report-timings]`. `cylc report-timings` is incompatible with Python 3.12. \ No newline at end of file diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index 86c639af113..23dfffba244 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -1222,6 +1222,9 @@ def default_for( {PLATFORM_REPLACES.format("[job]batch system")} ''') + replaces = PLATFORM_REPLACES.format( + "[job]batch submit command template" + ) Conf('job runner command template', VDR.V_STRING, desc=f''' Set the command used by the chosen job runner. @@ -1230,9 +1233,7 @@ def default_for( .. versionadded:: 8.0.0 - {PLATFORM_REPLACES.format( - "[job]batch submit command template" - )} + {replaces} ''') Conf('shell', VDR.V_STRING, '/bin/bash', desc=''' @@ -1465,6 +1466,8 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs command``. ''') + replaces = PLATFORM_REPLACES.format( + "[remote]retrieve job logs max size") Conf('retrieve job logs max size', VDR.V_STRING, desc=f''' {LOG_RETR_SETTINGS['retrieve job logs max size']} @@ -1472,9 +1475,10 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs max size``. - {PLATFORM_REPLACES.format( - "[remote]retrieve job logs max size")} + {replaces} ''') + replaces = PLATFORM_REPLACES.format( + "[remote]retrieve job logs retry delays") Conf('retrieve job logs retry delays', VDR.V_INTERVAL_LIST, desc=f''' {LOG_RETR_SETTINGS['retrieve job logs retry delays']} @@ -1483,8 +1487,7 @@ def default_for( {REPLACES}``global.rc[hosts][]retrieve job logs retry delays``. - {PLATFORM_REPLACES.format( - "[remote]retrieve job logs retry delays")} + {replaces} ''') Conf('tail command template', VDR.V_STRING, 'tail -n +1 --follow=name %(filename)s', diff --git a/cylc/flow/cfgspec/workflow.py b/cylc/flow/cfgspec/workflow.py index f521614c73a..d75013fd621 100644 --- a/cylc/flow/cfgspec/workflow.py +++ b/cylc/flow/cfgspec/workflow.py @@ -401,7 +401,7 @@ def get_script_common_text(this: str, example: Optional[str] = None): # differentiate between not set vs set to empty default = None elif item.endswith("handlers"): - desc = desc + '\n\n' + dedent(rf''' + desc = desc + '\n\n' + dedent(f''' Examples: .. code-block:: cylc @@ -413,9 +413,9 @@ def get_script_common_text(this: str, example: Optional[str] = None): {item} = echo %(workflow)s # configure multiple event handlers - {item} = \ - 'echo %(workflow)s, %(event)s', \ - 'my_exe %(event)s %(message)s' \ + {item} = \\ + 'echo %(workflow)s, %(event)s', \\ + 'my_exe %(event)s %(message)s' \\ 'curl -X PUT -d event=%(event)s host:port' ''') elif item.startswith("abort on"): @@ -1856,7 +1856,7 @@ def upg(cfg, descr): ['scheduling', 'max active cycle points'], ['scheduling', 'runahead limit'], cvtr=converter( - lambda x: f'P{int(x)-1}' if x != '' else '', + lambda x: f'P{int(x) - 1}' if x != '' else '', '"{old}" -> "{new}"' ), silent=cylc.flow.flags.cylc7_back_compat, diff --git a/cylc/flow/id.py b/cylc/flow/id.py index b8f34fb217f..222d16f82b6 100644 --- a/cylc/flow/id.py +++ b/cylc/flow/id.py @@ -497,7 +497,7 @@ def duplicate( )? (?: # cycle/task/job - { RELATIVE_PATTERN } + {RELATIVE_PATTERN} )? )? )? diff --git a/cylc/flow/scripts/report_timings.py b/cylc/flow/scripts/report_timings.py index 8d05ed2748d..b6d796ad732 100755 --- a/cylc/flow/scripts/report_timings.py +++ b/cylc/flow/scripts/report_timings.py @@ -54,6 +54,8 @@ import sys from typing import TYPE_CHECKING + +from cylc.flow import LOG from cylc.flow.exceptions import CylcError from cylc.flow.id_cli import parse_id from cylc.flow.option_parsers import ( @@ -123,6 +125,12 @@ def main(parser: COP, options: 'Values', workflow_id: str) -> None: constraint='workflows', ) + LOG.warning( + "cylc report-timings is deprecated." + " The analysis view in the GUI provides" + " similar functionality." + ) + output_options = [ options.show_raw, options.show_summary, options.html_summary ] @@ -246,7 +254,10 @@ def _check_imports(self): try: import pandas except ImportError: - raise CylcError('Cannot import pandas - summary unavailable.') + raise CylcError( + 'Cannot import pandas - summary unavailable.' + ' try: pip install cylc-flow[report-timings]' + ) else: del pandas diff --git a/setup.cfg b/setup.cfg index 6581f341571..48d940a6a29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -52,6 +52,7 @@ classifiers = Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Programming Language :: Python :: 3 :: Only Programming Language :: Python :: Implementation :: CPython Topic :: Scientific/Engineering :: Atmospheric Science @@ -103,6 +104,7 @@ report-timings = pandas==1.* matplotlib tests = + aiosmtpd async_generator bandit>=1.7.0 coverage>=5.0.0,<7.3.1 @@ -141,7 +143,6 @@ all = %(main_loop-log_db)s %(main_loop-log_main_loop)s %(main_loop-log_memory)s - %(report-timings)s %(tests)s %(tutorials)s diff --git a/tests/functional/events/09-task-event-mail.t b/tests/functional/events/09-task-event-mail.t index fb95c228a68..a689268ea1d 100755 --- a/tests/functional/events/09-task-event-mail.t +++ b/tests/functional/events/09-task-event-mail.t @@ -50,14 +50,15 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'retry: 1/t1/01' -b'succeeded: 1/t1/02' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +retry: 1/t1/01 +succeeded: 1/t1/02 +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ + run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[1/t1/02 succeeded\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[1/t1/02 succeeded\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill diff --git a/tests/functional/events/18-workflow-event-mail.t b/tests/functional/events/18-workflow-event-mail.t index fcde6da0bd7..eae45962db0 100755 --- a/tests/functional/events/18-workflow-event-mail.t +++ b/tests/functional/events/18-workflow-event-mail.t @@ -49,11 +49,11 @@ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'event: startup' -b'message: workflow starting' -b'event: shutdown' -b'message: AUTOMATIC' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +event: startup +message: workflow starting +event: shutdown +message: AUTOMATIC +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ purge diff --git a/tests/functional/events/29-task-event-mail-1.t b/tests/functional/events/29-task-event-mail-1.t index b6669ea8741..bef890c9c71 100755 --- a/tests/functional/events/29-task-event-mail-1.t +++ b/tests/functional/events/29-task-event-mail-1.t @@ -37,12 +37,14 @@ run_ok "${TEST_NAME_BASE}-validate" \ workflow_run_ok "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach "$WORKFLOW_NAME" + contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'retry: 1/t1/01' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +retry: 1/t1/01 +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ + run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[1/t1/01 retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[1/t1/01 retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" purge mock_smtpd_kill diff --git a/tests/functional/events/30-task-event-mail-2.t b/tests/functional/events/30-task-event-mail-2.t index 6e8ff8a1e86..10ef0119892 100755 --- a/tests/functional/events/30-task-event-mail-2.t +++ b/tests/functional/events/30-task-event-mail-2.t @@ -50,27 +50,29 @@ workflow_run_fail "${TEST_NAME_BASE}-run" \ cylc play --reference-test --debug --no-detach ${OPT_SET} "${WORKFLOW_NAME}" contains_ok "${TEST_SMTPD_LOG}" <<__LOG__ -b'retry: 1/t1/01' -b'retry: 1/t2/01' -b'retry: 1/t3/01' -b'retry: 1/t4/01' -b'retry: 1/t5/01' -b'retry: 1/t1/02' -b'retry: 1/t2/02' -b'retry: 1/t3/02' -b'retry: 1/t4/02' -b'retry: 1/t5/02' -b'failed: 1/t1/03' -b'failed: 1/t2/03' -b'failed: 1/t3/03' -b'failed: 1/t4/03' -b'failed: 1/t5/03' -b'see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/' +retry: 1/t1/01 +retry: 1/t2/01 +retry: 1/t3/01 +retry: 1/t4/01 +retry: 1/t5/01 +retry: 1/t1/02 +retry: 1/t2/02 +retry: 1/t3/02 +retry: 1/t4/02 +retry: 1/t5/02 +failed: 1/t1/03 +failed: 1/t2/03 +failed: 1/t3/03 +failed: 1/t4/03 +failed: 1/t5/03 +see: http://localhost/stuff/${USER}/${WORKFLOW_NAME}/ __LOG__ + run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[. tasks retry\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + grep -qPizo "Subject: \[. tasks retry\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" run_ok "${TEST_NAME_BASE}-grep-log" \ - grep -q "Subject: \\[. tasks failed\\].* ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" -purge + grep -qPizo "Subject: \[. tasks failed\]\n ${WORKFLOW_NAME}" "${TEST_SMTPD_LOG}" + + purge mock_smtpd_kill exit diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header index 458a7dc908c..c4a07603126 100644 --- a/tests/functional/lib/bash/test_header +++ b/tests/functional/lib/bash/test_header @@ -807,10 +807,13 @@ mock_smtpd_init() { # Logic borrowed from Rose local SMTPD_LOG="${TEST_DIR}/smtpd.log" local SMTPD_HOST="localhost:${SMTPD_PORT}" # Set up fake SMTP server to catch outgoing mail & redirect to log: - python3 -u -m 'smtpd' -c 'DebuggingServer' -d -n "${SMTPD_HOST}" \ + python3 -u -m 'aiosmtpd' \ + --class aiosmtpd.handlers.Debugging stdout \ + --debug --nosetuid \ + --listen "${SMTPD_HOST}" \ 1>"${SMTPD_LOG}" 2>&1 & # Runs in background local SMTPD_PID="$!" - while ! grep -q 'DebuggingServer started' "${SMTPD_LOG}" 2>'/dev/null' + while ! grep -q 'is listening' "${SMTPD_LOG}" 2>'/dev/null' do if ps "${SMTPD_PID}" 1>/dev/null 2>&1; then sleep 1 # Still waiting for fake server to start diff --git a/tests/unit/test_indep_task_queues.py b/tests/unit/test_indep_task_queues.py index a0a1894cece..bb11fbce463 100644 --- a/tests/unit/test_indep_task_queues.py +++ b/tests/unit/test_indep_task_queues.py @@ -92,9 +92,10 @@ def test_queue_and_release( assert sorted([r.tdef.name for r in released]) == sorted(expected_released) # check released tasks change state to "preparing", and not is_queued - for r in released: - assert r.state.reset.called_with(TASK_STATUS_PREPARING) - assert r.state.reset.called_with(is_queued=False) + # Commented out pending https://github.com/cylc/cylc-flow/issues/5812 + # for r in released: + # assert r.state.reset.called_with(TASK_STATUS_PREPARING) + # assert r.state.reset.called_with(is_queued=False) # check that adopted orphans end up in the default queue orphans = ["orphan1", "orphan2"]