From fa3611b66a23dd3e5eb7ed317f10f2666a757c66 Mon Sep 17 00:00:00 2001 From: Tim Pillinger <26465611+wxtim@users.noreply.github.com> Date: Mon, 4 Dec 2023 11:07:21 +0000 Subject: [PATCH] Ensure that the cached globalconfig object is reloaded after the export of `CYLC_SYMLINKS` variable. This means that using the `CYLC_SYMLINKS` variable allows users to specify installation symlink locations in the `rose-suite.conf`. --- CHANGES.md | 8 ++++ cylc/rose/__init__.py | 29 +++++++++++- cylc/rose/utilities.py | 8 ++++ tests/conftest.py | 24 +++++++--- tests/functional/test_utils.py | 81 ++++++++++++++++++++++++++++++++++ 5 files changed, 143 insertions(+), 7 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 27af8c00..dc165e9c 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,6 +6,14 @@ creating a new release entry be sure to copy & paste the span tag with the updated. Only the first match gets replaced, so it's fine to leave the old ones in. --> + +## __cylc-rose-1.4.0 (Upcoming)__ + +### Features + +[#269](https://github.com/cylc/cylc-rose/pull/269) - Allow environment variables +set in ``rose-suite.conf`` to be used when parsing ``global.cylc``. + ## __cylc-rose-1.3.1 (Released 2023-10-24)__ ### Fixes diff --git a/cylc/rose/__init__.py b/cylc/rose/__init__.py index 841796a0..b8283291 100644 --- a/cylc/rose/__init__.py +++ b/cylc/rose/__init__.py @@ -70,11 +70,37 @@ for ease of porting Cylc 7 workflows. +The ``global.cylc`` file +^^^^^^^^^^^^^^^^^^^^^^^^ + +The Cylc Rose Plugin forces the reloading of the ``global.cylc`` file +to allow environment variables set by Rose to change the global configuration. + +For example you could use ``CYLC_SYMLINKS`` as a variable to control +the behaviour of ``cylc install``: + + .. code-block:: cylc + + #!jinja2 + # part of a global.cylc file + [install] + [[symlink dirs]] + [[[hpc]]] + {% if environ["CYLC_SYMLINKS"] | default("x") == "A" %} + run = $LOCATION_A + {% elif environ["CYLC_SYMLINKS"] | default("x") == "B" %} + run = $LOCATION_B + {% else %} + run = $LOCATION_C + {% endif %} + + + Special Variables ----------------- The Cylc Rose plugin provides two environment/template variables -to the Cylc scheduler: +to the Cylc scheduler. ``ROSE_ORIG_HOST`` Cylc commands (such as ``cylc install``, ``cylc validate`` and @@ -111,6 +137,7 @@ ``CYLC_VERSION`` will be removed from your configuration by the Cylc-Rose plugin, as it is now set by Cylc. + Additional CLI options ---------------------- You can use command line options to set or override diff --git a/cylc/rose/utilities.py b/cylc/rose/utilities.py index e23f6e54..515356a5 100644 --- a/cylc/rose/utilities.py +++ b/cylc/rose/utilities.py @@ -27,6 +27,7 @@ from cylc.flow import LOG from cylc.flow.exceptions import CylcError from cylc.flow.flags import cylc7_back_compat +from cylc.flow.cfgspec.glbl_cfg import glbl_cfg from cylc.flow.hostuserutil import get_host from metomi.isodatetime.datetimeoper import DateTimeOperator from metomi.rose import __version__ as ROSE_VERSION @@ -869,6 +870,13 @@ def export_environment(environment: Dict[str, str]) -> None: for key, val in environment.items(): os.environ[key] = val + # If env vars have been set we want to force reload + # the global config so that the value of this vars + # can be used by Jinja2 in the global config. + # https://github.com/cylc/cylc-rose/issues/237 + if environment: + glbl_cfg().load() + def record_cylc_install_options( srcdir: Path, diff --git a/tests/conftest.py b/tests/conftest.py index 624d7c6a..d2baf6d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -15,9 +15,11 @@ # along with this program. If not, see . from types import SimpleNamespace +from uuid import uuid4 from cylc.flow import __version__ as CYLC_VERSION from cylc.flow.option_parsers import Options +from cylc.flow.pathutil import get_workflow_run_dir from cylc.flow.scripts.install import get_option_parser as install_gop from cylc.flow.scripts.install import install_cli as cylc_install from cylc.flow.scripts.reinstall import get_option_parser as reinstall_gop @@ -27,6 +29,11 @@ import pytest +@pytest.fixture +def generate_workflow_name(): + return 'cylc-rose-test-' + str(uuid4())[:8] + + @pytest.fixture(scope='module') def mod_capsys(request): from _pytest.capture import SysCapture @@ -108,7 +115,7 @@ def _inner(srcpath, args=None): return _inner -def _cylc_install_cli(capsys, caplog): +def _cylc_install_cli(capsys, caplog, generate_workflow_name): """Access the install CLI""" def _inner(srcpath, args=None): """Install a workflow. @@ -119,9 +126,13 @@ def _inner(srcpath, args=None): """ options = Options(install_gop(), args)() output = SimpleNamespace() + if not options.workflow_name: + options.workflow_name = generate_workflow_name + if not args or args and not args.get('no_run_name', ''): + options.no_run_name = True try: - cylc_install(options, str(srcpath)) + output.name, output.id = cylc_install(options, str(srcpath)) output.ret = 0 output.exc = '' except Exception as exc: @@ -129,6 +140,7 @@ def _inner(srcpath, args=None): output.exc = exc output.logging = '\n'.join([i.message for i in caplog.records]) output.out, output.err = capsys.readouterr() + output.run_dir = get_workflow_run_dir(output.id) return output return _inner @@ -159,13 +171,13 @@ def _inner(workflow_id, opts=None): @pytest.fixture -def cylc_install_cli(capsys, caplog): - return _cylc_install_cli(capsys, caplog) +def cylc_install_cli(capsys, caplog, generate_workflow_name): + return _cylc_install_cli(capsys, caplog, generate_workflow_name) @pytest.fixture(scope='module') -def mod_cylc_install_cli(mod_capsys, mod_caplog): - return _cylc_install_cli(mod_capsys, mod_caplog) +def mod_cylc_install_cli(mod_capsys, mod_caplog, generate_workflow_name): + return _cylc_install_cli(mod_capsys, mod_caplog, generate_workflow_name) @pytest.fixture diff --git a/tests/functional/test_utils.py b/tests/functional/test_utils.py index f2b9ef07..59486f30 100644 --- a/tests/functional/test_utils.py +++ b/tests/functional/test_utils.py @@ -36,3 +36,84 @@ def test_basic(tmp_path): assert Path(tmp_path / 'src/rose-suite.conf').read_text() == ( Path(tmp_path / 'dest/rose-suite.conf').read_text() ) + + +def test_CYLC_SYMLINKS_validate(monkeypatch, tmp_path, cylc_validate_cli): + """We reload the global config after exporting env variables.""" + # Setup global config: + global_conf = """#!jinja2 + {% from "cylc.flow" import LOG %} + {% set cylc_symlinks = environ.get('CYLC_SYMLINKS', None) %} + {% do LOG.critical(cylc_symlinks) %} + """ + conf_path = tmp_path / 'conf' + conf_path.mkdir() + monkeypatch.setenv('CYLC_CONF_PATH', conf_path) + + # Setup workflow config: + (conf_path / 'global.cylc').write_text(global_conf) + (tmp_path / 'rose-suite.conf').write_text( + '[env]\nCYLC_SYMLINKS="Foo"\n') + (tmp_path / 'flow.cylc').write_text(""" + [scheduling] + initial cycle point = now + [[graph]] + R1 = x + [runtime] + [[x]] + """) + + # Validate the config: + output = cylc_validate_cli(tmp_path) + assert output.ret == 0 + + # CYLC_SYMLINKS == None the first time the global.cylc + # is loaded and "Foo" the second time. + assert output.logging == 'None\n"Foo"' + + +def test_CYLC_SYMLINKS_install(monkeypatch, tmp_path, cylc_install_cli): + """We reload the global config after exporting env variables.""" + # Setup global config: + global_conf = ( + '#!jinja2\n' + '[install]\n' + ' [[symlink dirs]]\n' + ' [[[localhost]]]\n' + '{% set cylc_symlinks = environ.get(\'CYLC_SYMLINKS\', None) %}\n' + '{% if cylc_symlinks == "foo" %}\n' + f'log = {str(tmp_path)}/foo\n' + '{% else %}\n' + f'log = {str(tmp_path)}/bar\n' + '{% endif %}\n' + ) + glbl_conf_path = tmp_path / 'conf' + glbl_conf_path.mkdir() + (glbl_conf_path / 'global.cylc').write_text(global_conf) + monkeypatch.setenv('CYLC_CONF_PATH', glbl_conf_path) + + # Setup workflow config: + (tmp_path / 'rose-suite.conf').write_text( + '[env]\nCYLC_SYMLINKS=foo\n') + (tmp_path / 'flow.cylc').write_text(""" + [scheduling] + initial cycle point = now + [[graph]] + R1 = x + [runtime] + [[x]] + """) + + # Install the config: + output = cylc_install_cli(tmp_path) + import sys + for i in output.logging.split('\n'): + print(i, file=sys.stderr) + assert output.ret == 0 + + # Assert symlink created back to test_path/foo: + expected_msg = ( + f'Symlink created: {output.run_dir}/log -> ' + f'{tmp_path}/foo/cylc-run/{output.id}/log' + ) + assert expected_msg in output.logging.split('\n')[0]