diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 59f6ef5de3..5dd910f070 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,7 +79,7 @@ jobs: - name: build & test run: | - make html spelling linkcheck doctest \ + make html spelling linkcheck \ SPHINXOPTS='-Wn --keep-going' FORCE_COLOR=true - name: debug sphinx failure diff --git a/Makefile b/Makefile index 4cb1451f0d..65961853c2 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ clean: # remove auto-generated content rm -rf src/plugins/main-loop/built-in rm -rf src/plugins/install/built-in + rm -rf src/plugins/xtriggers/built-in rm -rf src/user-guide/task-implementation/job-runner-handlers watch: clean diff --git a/README.md b/README.md index 76a0827aa2..f1e5838dee 100644 --- a/README.md +++ b/README.md @@ -183,7 +183,7 @@ allows it to pick up changes to autodocumented items in the source code. ```console # note: -W tells Sphinx to fail on warnings -$ make html linkcheck doctest SPHINXOPTS='-W' +$ make html linkcheck SPHINXOPTS='-W' ``` ### Deploying diff --git a/bin/version b/bin/version index 265ad16ed2..7b871de217 100755 --- a/bin/version +++ b/bin/version @@ -27,7 +27,6 @@ from pkg_resources import parse_version DUMMY_FORMATS = [ 'doctrees', 'linkcheck', - 'doctest', 'coverage' ] diff --git a/src/conf.py b/src/conf.py index 8976e559f9..a2022c039a 100644 --- a/src/conf.py +++ b/src/conf.py @@ -34,7 +34,6 @@ # sphinx built-in extensions 'sphinx.ext.autodoc', 'sphinx.ext.autosummary', - 'sphinx.ext.doctest', 'sphinx.ext.graphviz', 'sphinx.ext.intersphinx', 'sphinx.ext.napoleon', @@ -111,6 +110,10 @@ autosummary_generate_overwrite = True autosummary_imported_members = False +# Autodoc type hints can look very messy if they are included in the signature, +# so we only include them in the description instead: +autodoc_typehints = 'description' + # Mapping to other Sphinx projects we want to import references from. # NOTE: To search available references, use: # $ python -m sphinx.ext.intersphinx /objects.inv | less diff --git a/src/dictionaries/words b/src/dictionaries/words index 79b3a1f8bc..ce50325d09 100644 --- a/src/dictionaries/words +++ b/src/dictionaries/words @@ -46,6 +46,7 @@ filesystem filesystems fillchar foo +fpath freeform fullmatch globals @@ -128,6 +129,7 @@ py pythonically queueing qux +randint rc refactor regex @@ -147,6 +149,7 @@ ret retrigger retriggered retriggering +rmtree roadmap ruleset rulesets @@ -155,10 +158,12 @@ runnable runtime schd screenshot +secs setup setups shebang shutdown +shutil situ speedup ssh @@ -185,6 +190,7 @@ symlink symlinked symlinking symlinks +sys taskdef templated templating @@ -194,6 +200,7 @@ timeouts timestep timezone timezones +tuple uiserver unpaused unpausing @@ -208,6 +215,7 @@ vs wallclock websocket websockets +whitelist whitelisting whitespace workflow diff --git a/src/plugins/xtriggers/index.rst b/src/plugins/xtriggers/index.rst index 01e976923b..5a920f79cd 100644 --- a/src/plugins/xtriggers/index.rst +++ b/src/plugins/xtriggers/index.rst @@ -1,52 +1,70 @@ Xtrigger Plugins ====================================== -Xtrigger plugins allow you to install and use xtriggers without them being -in your ``CYLC_PYTHONPATH``. +.. versionadded:: 8.3 + Xtrigger plugins allow you to install and use + :ref:`xtriggers
` without them being + in ``/lib/python/`` or ``$CYLC_PYTHONPATH``. -Built In Plugins ----------------- +.. seealso:: -Cylc Flow provides the following xtriggers. + * :ref:`Built-in Clock Triggers` + * :ref:`Built-in Workflow State Triggers` + * :ref:`Built-in Toy Xtriggers` -.. autosummary:: - :toctree: built-in - :template: docstring_only.rst - cylc.flow.xtriggers.echo - cylc.flow.xtriggers.workflow_state - cylc.flow.xtriggers.xrandom - -.. Note: Autosummary generates files in this directory, these are cleaned - up by `make clean`. +.. _developing.xtrigger.plugins: Developing ``xtrigger`` plugins ------------------------------- -Cylc uses entry points registered by setuptools to search for xtrigger -plugins. +Cylc uses the ``cylc.xtriggers`` entry point registered by setuptools to search +for xtrigger plugins. Each xtrigger is registered individually. Example ^^^^^^^ -Plugins are registered by registering them with the ``cylc.xtriggers`` -entry points. Each xtrigger is registered individually. +Consider a package called ``my_package`` with the following structure: + +.. code-block:: python + :caption: ``my_package/foo.py`` + + def foo(): + ... + + def bar(): + ... + +.. code-block:: python + :caption: ``my_package/baz.py`` + + def baz(): + ... + +These xtriggers can be registered in the package's ``setup.cfg`` or +``pyproject.toml`` file. .. code-block:: ini :caption: ``setup.cfg`` [options.entry_points] - cylc.xtriggers = - foo = my_package.foo:foo - bar = my_package.foo:bar - baz = my_package.baz:baz + cylc.xtriggers = + foo = my_package.foo + bar = my_package.foo + baz = my_package.baz .. code-block:: toml :caption: ``pyproject.toml`` [project.entry-points."cylc.xtriggers"] - foo = "my_package.foo:foo" - bar = "my_package.foo:bar" - baz = "my_package.baz:baz" + foo = "my_package.foo" + bar = "my_package.foo" + baz = "my_package.baz" + +.. tip:: + + It is recommended to implement only one xtrigger per module. This allows + you to write a ``validate`` function for each xtrigger - see + :ref:`user-guide.xtrigger-validation-functions`. diff --git a/src/tutorial/furthertopics/clock-triggered-tasks.rst b/src/tutorial/furthertopics/clock-triggered-tasks.rst index 0612200c21..f3bb85ddfe 100644 --- a/src/tutorial/furthertopics/clock-triggered-tasks.rst +++ b/src/tutorial/furthertopics/clock-triggered-tasks.rst @@ -14,7 +14,8 @@ Clock-triggering effectively enables us to tether the "cycle time" to the .. note:: Clock triggers are :ref:`Section External Triggers`. They differ from - custom external triggers only in that they are provided with Cylc. + custom external triggers only in that they are provided with Cylc, and + that the ``:interval`` suffix has no effect. Clock Triggering @@ -135,9 +136,9 @@ Edit the ``[[scheduling]]`` section to read: initial cycle point = now final cycle point = +P1D # Run for one day [[xtriggers]] - quarter_past_trigger = wall_clock(offset=PT15M):PT30S - half_past_trigger = wall_clock(offset=PT30M):PT30S - quarter_to_trigger = wall_clock(offset=PT45M):PT30S + quarter_past_trigger = wall_clock(offset=PT15M) + half_past_trigger = wall_clock(offset=PT30M) + quarter_to_trigger = wall_clock(offset=PT45M) [[graph]] PT1H = """ @wall_clock => bell @@ -165,15 +166,6 @@ Leave your workflow running for a while to confirm it is working as expected before stopping it. -.. note:: - - You may have noticed the ``:PT30S`` at the end of each clock trigger - definition. This how often the :ref:`Section External Triggers` is checked. - By default external triggers are checked every 10 seconds, but if there - are a lot of external triggers this can be hard work for the computer - running the workflow and it may not be necessary to check this often. - - Summary ------- diff --git a/src/tutorial/scheduling/further-scheduling.rst b/src/tutorial/scheduling/further-scheduling.rst index b97f251f43..b2f9f87b96 100644 --- a/src/tutorial/scheduling/further-scheduling.rst +++ b/src/tutorial/scheduling/further-scheduling.rst @@ -75,7 +75,7 @@ Clock Triggers [scheduling] initial cycle point = 2000-01-01T00Z [[xtriggers]] - PT1H_trigger = wall_clock(offset=-PT1H):PT30S + PT1H_trigger = wall_clock(offset=-PT1H) [[graph]] # "daily" will run, at the earliest, one hour before midday. T12 = @PT1H_trigger => daily diff --git a/src/user-guide/writing-workflows/external-triggers.rst b/src/user-guide/writing-workflows/external-triggers.rst index 9dbbc0a73e..279060d295 100644 --- a/src/user-guide/writing-workflows/external-triggers.rst +++ b/src/user-guide/writing-workflows/external-triggers.rst @@ -34,13 +34,11 @@ broker. Cylc has several built-in external trigger functions: -- clock triggers - see :ref:`Built-in Clock Triggers` -- inter-workflow triggers - see :ref:`Built-in Workflow State Triggers` +- :ref:`Built-in Clock Triggers` +- :ref:`Built-in Workflow State Triggers` Trigger functions are normal Python functions, with certain constraints as -described below in: - -- custom trigger functions - see :ref:`Custom Trigger Functions` +described below in :ref:`Custom Trigger Functions`. External triggers are configured in the :cylc:conf:`flow.cylc[scheduling][xtriggers]` section. @@ -62,9 +60,7 @@ in :ref:`ClockTriggerTasks`. Clock triggers, unlike other trigger functions, are executed synchronously in the main process. The clock trigger function signature looks like this: -.. code-block:: python - - wall_clock(offset=None) +.. autofunction:: cylc.flow.xtriggers.wall_clock.wall_clock The ``offset`` argument is a datetime duration (``PT1H`` is 1 hour) relative to the dependent task's cycle point (automatically passed to the @@ -79,7 +75,7 @@ cycle point value by one hour: [scheduling] initial cycle point = 2018-01-01 [[xtriggers]] - clock_1 = wall_clock(offset=PT1H):PT10S + clock_1 = wall_clock(offset=PT1H) [[graph]] P1D = "@clock_1 => foo" [runtime] @@ -87,9 +83,7 @@ cycle point value by one hour: script = run-foo.sh Notice that the short label ``clock_1`` is used to represent the -trigger function in the graph. The function call interval, which determines how -often the :term:`scheduler` checks the clock, is optional. Here it is -``PT10S`` (i.e. 10 seconds, which is also the default value). +trigger function in the graph. Argument keywords can be omitted if called in the right order, so the ``clock_1`` trigger can also be declared like this: @@ -130,10 +124,7 @@ tasks off of remote task statuses or messages in other workflows. The workflow state trigger function signature looks like this: -.. code-block:: python - - workflow_state(workflow, task, point, offset=None, status='succeeded', - message=None, cylc_run_dir=None, debug=False) +.. autofunction:: cylc.flow.xtriggers.workflow_state.workflow_state The first three arguments are compulsory; they single out the target workflow name (``workflow``) task name (``task``) and cycle point @@ -269,29 +260,31 @@ Custom Trigger Functions Trigger functions are just normal Python functions, with a few special properties: -- they must: +- They must: - - be defined in a module with the same name as the function, unless + - Be defined in a module with the same name as the function, unless provided using the ``cylc.xtriggers`` entry point; - be compatible with the same Python version that runs the scheduler (see :ref:`Requirements` for the latest version specification). -- they can be located either: +- They can be located either: - - in ``/lib/python/``; - - anywhere in your ``$CYLC_PYTHONPATH``; - - or defined using the ``cylc.xtriggers`` entry point for an installed - package. + - In ``/lib/python/``; + - Anywhere in your ``$CYLC_PYTHONPATH``; + - Defined using the ``cylc.xtriggers`` entry point for an installed + package - see :ref:`developing.xtrigger.plugins` -- they can take arbitrary positional and keyword arguments +- They can take arbitrary positional and keyword arguments (except ``sequential``, which is reserved - see :ref:`Sequential Xtriggers`) -- workflow and task identity, and cycle point, can be passed to trigger +- Workflow and task identity, and cycle point, can be passed to trigger functions by using string templates in function arguments (see below) -- integer, float, boolean, and string arguments will be recognized and +- Integer, float, boolean, and string arguments will be recognized and passed to the function as such -- if a trigger function depends on files or directories (for example) +- If a trigger function depends on files or directories (for example) that might not exist when the function is first called, just return - unsatisfied until everything required does exist. +- The module can also provide a ``validate`` function for checking configured + arguments / keyword arguments, see + :ref:`user-guide.xtrigger-validation-functions` for details. .. note:: @@ -299,6 +292,7 @@ properties: because each call is executed in an independent process in the process pool. If necessary the filesystem can be used for this purpose. + .. spelling:word-list:: vv @@ -344,6 +338,27 @@ The :term:`scheduler` manages trigger functions as follows: from the sub-process in which the function executes) +.. _user-guide.xtrigger-validation-functions: + +Xtrigger Validation Functions +----------------------------- + +The arguments you call the xtrigger function with are automatically validated +against the function signature, however, you might wish to add extra validation +logic to your custom xtrigger, e.g. to check argument types or values. + +If a function named ``validate`` is present alongside the xtrigger in its +module, it will be automatically called with a dictionary of all the arguments +passed to the xtrigger function in the workflow config. It should raise a +:py:obj:`cylc.flow.exceptions.WorkflowConfigError` exception if an error is +detected. + +The :py:mod:`cylc.flow.xtriggers.xrandom` xtrigger module contains an example +of an xtrigger validation function. + + +.. _Built-in Toy Xtriggers: + Toy Examples ^^^^^^^^^^^^ @@ -353,14 +368,9 @@ echo The trivial built-in ``echo`` function takes any number of positional and keyword arguments (from the workflow configuration) and simply prints them to stdout, and then returns False (i.e. trigger condition not -satisfied). Here it is in its entirety. +satisfied). -.. code-block:: python - - def echo(*args, **kwargs): - print("echo: ARGS:", args) - print("echo: KWARGS:", kwargs) - return False, {} +.. autofunction:: cylc.flow.xtriggers.echo.echo Here's an example echo trigger workflow: @@ -389,16 +399,9 @@ time (useful for testing the effect of a long-running trigger function - which should be avoided) and has a configurable random chance of success. The function signature is: -.. code-block:: python - - xrandom(percent, secs=0, _=None, debug=False) - -The ``percent`` argument sets the odds of success in any given call; -``secs`` is the number of seconds to sleep before returning; and the -``_`` argument (underscore is a conventional name for a variable -that is not used, in Python) is provided to allow specialization of the -trigger to (for example) task name, task ID, or cycle point (just use -the appropriate string templates in the workflow configuration for this). +.. automodule:: cylc.flow.xtriggers.xrandom + :members: xrandom, validate + :member-order: bysource An example xrandom trigger workflow: diff --git a/src/user-guide/writing-workflows/jinja2.rst b/src/user-guide/writing-workflows/jinja2.rst index 841aba1272..2aaa15c57c 100644 --- a/src/user-guide/writing-workflows/jinja2.rst +++ b/src/user-guide/writing-workflows/jinja2.rst @@ -227,14 +227,6 @@ In the argument list of a filter or test function, the first argument is the variable value to be filtered or tested, and subsequent arguments can be whatever is needed. Currently three custom filters are supplied: -.. import the filters to allow their doctests to pass (make doctest) - -.. testsetup:: - - from cylc.flow.jinja.filters.pad import pad - from cylc.flow.jinja.filters.strftime import strftime - from cylc.flow.jinja.filters.duration_as import duration_as - .. autosummary:: :nosignatures: