From d99106c0572159c55accc53aa34a84e492fa4f5c Mon Sep 17 00:00:00 2001 From: Micael Oliveira Date: Wed, 21 Feb 2024 16:42:35 +1100 Subject: [PATCH 1/5] In nuopc.runconfig files, always return variables that are not part of a table as lists. --- om3utils/nuopc_config.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/om3utils/nuopc_config.py b/om3utils/nuopc_config.py index 8008fc0..7ee253e 100644 --- a/om3utils/nuopc_config.py +++ b/om3utils/nuopc_config.py @@ -85,10 +85,7 @@ def read_nuopc_config(file_name: str) -> dict: elif re.match(label_value_pattern, line): match = re.match(label_value_pattern, line) - if len(match.group(2).split()) > 1: - config[match.group(1)] = [_convert_from_string(string) for string in match.group(2).split()] - else: - config[match.group(1)] = _convert_from_string(match.group(2)) + config[match.group(1)] = [_convert_from_string(string) for string in match.group(2).split()] return config From c6925d51b3d36597debfe5abc40e1fb329e996f5 Mon Sep 17 00:00:00 2001 From: Micael Oliveira Date: Wed, 21 Feb 2024 16:43:05 +1100 Subject: [PATCH 2/5] Add unit tests for several exceptions and a couple of missing cases. --- tests/test_mom6_input.py | 9 +++++++++ tests/test_nuopc_config.py | 23 +++++++++++++++++++++++ tests/test_payu_config_yaml.py | 5 +++++ 3 files changed, 37 insertions(+) diff --git a/tests/test_mom6_input.py b/tests/test_mom6_input.py index 1048c11..aa981d9 100644 --- a/tests/test_mom6_input.py +++ b/tests/test_mom6_input.py @@ -93,7 +93,16 @@ def test_write_mom6_input(tmp_path, simple_mom6_input, simple_mom6_input_file): def test_round_trip_mom6_input(tmp_path, complex_mom6_input_file, modified_mom6_input_file): mom6_input_from_file = Mom6Input(file_name=complex_mom6_input_file.file) mom6_input_from_file["dt"] = 900.0 + mom6_input_from_file["ADDED_VAR"] = 1 + del mom6_input_from_file["ADDED_VAR"] mom6_input_from_file["ADDED_VAR"] = 32 + write_mom6_input(mom6_input_from_file, tmp_path / "MOM_input_new") + assert mom6_input_from_file["ADDED_VAR"] == 32 assert filecmp.cmp(tmp_path / "MOM_input_new", modified_mom6_input_file.file) + + +def test_read_missing_mom6_file(): + with pytest.raises(FileNotFoundError): + Mom6Input(file_name="garbage") diff --git a/tests/test_nuopc_config.py b/tests/test_nuopc_config.py index e19d8e6..539e518 100644 --- a/tests/test_nuopc_config.py +++ b/tests/test_nuopc_config.py @@ -56,6 +56,19 @@ def simple_nuopc_config_file(tmp_path): return MockFile(file, resource_file_str) +@pytest.fixture() +def invalid_nuopc_config_file(tmp_path): + file = tmp_path / "invalid_config_file" + resource_file_str = """DRIVER_attributes:: + Verbosity: off + cime_model - cesm +:: + +COMPONENTS::: atm ocn +""" + return MockFile(file, resource_file_str) + + def test_read_nuopc_config(tmp_path, simple_nuopc_config, simple_nuopc_config_file): config_from_file = read_nuopc_config(file_name=simple_nuopc_config_file.file) @@ -67,3 +80,13 @@ def test_write_nuopc_config(tmp_path, simple_nuopc_config, simple_nuopc_config_f write_nuopc_config(simple_nuopc_config, file) assert filecmp.cmp(file, simple_nuopc_config_file.file) + + +def test_read_invalid_nuopc_config_file(tmp_path, invalid_nuopc_config_file): + with pytest.raises(ValueError): + read_nuopc_config(file_name=invalid_nuopc_config_file.file) + + +def test_read_missing_nuopc_config_file(): + with pytest.raises(FileNotFoundError): + read_nuopc_config(file_name="garbage") diff --git a/tests/test_payu_config_yaml.py b/tests/test_payu_config_yaml.py index 0794d86..0f745dd 100644 --- a/tests/test_payu_config_yaml.py +++ b/tests/test_payu_config_yaml.py @@ -125,3 +125,8 @@ def test_round_trip_payu_config(tmp_path, complex_payu_config_file, modified_pay write_payu_config_yaml(config, tmp_path / "config.yaml") assert filecmp.cmp(tmp_path / "config.yaml", modified_payu_config_file.file) + + +def test_read_missing_payu_config(): + with pytest.raises(FileNotFoundError): + read_payu_config_yaml(file_name="garbage") From 3e24bf97063b901af113cf14c4620f225639509b Mon Sep 17 00:00:00 2001 From: Micael Oliveira Date: Wed, 21 Feb 2024 16:51:33 +1100 Subject: [PATCH 3/5] Omit some files from coverage report. --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1b3cccd..c4d4334 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,9 @@ test = [ addopts = ["--cov=om3utils", "--cov-report=term", "--cov-report=xml"] testpaths = ["tests"] +[tool.coverage.run] +omit = ["om3utils/__init__.py", "om3utils/_version.py"] + [tool.black] line-length = 120 From 1f015c2d82b16afda72f023df28d8435c643d33f Mon Sep 17 00:00:00 2001 From: Micael Oliveira Date: Thu, 22 Feb 2024 14:10:13 +1100 Subject: [PATCH 4/5] Improve documentation of functions and classes. --- om3utils/mom6_input.py | 115 +++++++++++++++++++---------------- om3utils/nuopc_config.py | 95 ++++++++++++++++++++++++++--- om3utils/payu_config_yaml.py | 27 +++++--- 3 files changed, 167 insertions(+), 70 deletions(-) diff --git a/om3utils/mom6_input.py b/om3utils/mom6_input.py index a4c5dce..4960ee5 100644 --- a/om3utils/mom6_input.py +++ b/om3utils/mom6_input.py @@ -1,4 +1,4 @@ -"""MOM6 input +"""Utilities to handle MOM6 parameter files. The MOM6 parameter file format is described here: @@ -33,8 +33,8 @@ We then have utility functions to convert from one representation to another: - nml_str -> mom6_input (_nml_str_to_mom6_input) - mom6_input -> nml_str (_mom6_input_to_nml_str) - - mom6_input_str -> nml_str (_mom6_input_str_to_nml_str + patch_mom6_input_str) - - nml_str -> mom6_input_str (_nml_str_to_mom6_input_str + unpatch_mom6_input_str) + - mom6_input_str -> nml_str (_mom6_input_str_to_nml_str + _patch_mom6_input_str) + - nml_str -> mom6_input_str (_nml_str_to_mom6_input_str + _unpatch_mom6_input_str) For round-trip parsing, one needs to keep track of the changes done to the file to make it a conforming Fortran namelist and then undo those changes. Since we use the f90mnml parser ability to patch a file as it is read, we also @@ -67,10 +67,12 @@ def _patch_mom6_input_str(mom6_input_str: str) -> tuple[str, dict]: The changes are recorded as a "patch", which is a dictionary: the keys are the line numbers where changes were made, while the values are tuples containing a keyword describing the type of change and, optionally, a string. - :param mom6_input_str: - :return: - """ + Args: + mom6_input_str (str): Contents of the MOM6 parameter file to patch. + Returns: + tuple: Contents of the patched MOM6 parameter file and the patch that was applied. + """ # Define several patterns that need to be matched comment_pattern = re.compile(r"/\*.*?\*/", flags=re.DOTALL) zstar_pattern = re.compile(r"Z\*") @@ -116,11 +118,14 @@ def replace_comment(match): def _unpatch_mom6_input_str(mom6_input_str: str, patch: dict = None) -> str: - """Undo the changes that were done to a MOM6 parameter file to make it into a conforming Fortran namelist + """Undo the changes that were done to a MOM6 parameter file to make it into a conforming Fortran namelist. - :param mom6_input_str: - :param patch: - :return: + Args: + mom6_input_str (str): Contents of the MOM6 parameter file to unpatch. + patch (dict): A dict containing the patch to revert. + + Returns: + str: Unpatched contents of the MOM6 parameter file. """ output = "" lines = mom6_input_str.split("\n")[1:-2] @@ -144,19 +149,25 @@ def _unpatch_mom6_input_str(mom6_input_str: str, patch: dict = None) -> str: def _mom6_input_str_to_nml_str(mom6_input_str: str) -> str: - """ + """Convert the MOM6 parameter file to a conforming Fortran namelist. - :param mom6_input_str: - :return: + Args: + mom6_input_str (str): Contents of the MOM6 parameter file. + + Returns: + str: Fortran namelist. """ return "&mom6\n" + mom6_input_str + "\n/" def _nml_str_to_mom6_input_str(nml_str: str) -> str: - """ + """Convert a Fortran namelist into a MOM6 parameter file. + + Args: + nml_str (str): Fortran namelist. - :param nml_str: - :return: + Returns: + str: MOM6 parameter file. """ lines = nml_str.split("\n") lines = lines[1:-2] @@ -164,10 +175,13 @@ def _nml_str_to_mom6_input_str(nml_str: str) -> str: def _mom6_input_to_nml_str(mom6_input: dict) -> str: - """ + """Convert MOM6 parameters stored in a dictionary into a Fortran namelist. + + Args: + mom6_input (dict): Dictionary of MOM6 parameters. - :param mom6_input: - :return: + Returns: + str: Fortran namelist. """ output_file = StringIO("") nml = f90nml.Namelist({"mom6": mom6_input}) @@ -180,10 +194,13 @@ def _mom6_input_to_nml_str(mom6_input: dict) -> str: def _nml_str_to_mom6_input(nml_str: str) -> dict: - """ + """Convert MOM6 parameters stored as a Fortran namelist into a dictionary. - :param nml_str: - :return: + Args: + nml_str (str): Fortran namelist. + + Returns: + dict: Dictionary of MOM6 parameters. """ parser = f90nml.Parser() nml = parser.reads(nml_str) @@ -199,7 +216,7 @@ class Mom6Input(dict): - stored all the keys in upper case - keep track of the changes done to the original dictionary - It also stores a "patch" that was applied to the mom6_input_str to convert it to a conforming Fortran namelist. + It also stores the "patch" that was applied to the mom6_input_str to convert it to a conforming Fortran namelist. This is used to "undo" the changes when writing the file. """ @@ -213,9 +230,10 @@ class Mom6Input(dict): _nml_patch = None def __init__(self, file_name: str = None): - """ + """Read NOM6 parameters from file. - :param file_name: + Args: + file_name (str): Name of file to read. """ # Open file and read contents file = Path(file_name) @@ -238,37 +256,28 @@ def __init__(self, file_name: str = None): self._nml_patch = {"mom6": {}} def __setitem__(self, key, value): - """ + """Override method to add item to dict. - :param key: - :param value: - :return: + This method takes into account that all keys should be stored in uppercase. It also adds the new item to the + namelist patch used for round-trip parsing. """ super().__setitem__(key.upper(), value) if self._nml_patch: self._nml_patch["mom6"][key.upper()] = value def __getitem__(self, key): - """ - - :param key: - :return: - """ + """Override method to get item from dict, taking into account all keys are stored in uppercase.""" return super().__getitem__(key.upper()) def __delitem__(self, key): - """ - - :param key: - :return: - """ + """Override method to delete item from dict, so that all keys are stored in uppercase.""" super().__delitem__(key.upper()) def write(self, file: Path): - """ + """Write contents of MOM6Input to a file. - :param file: - :return: + Args: + file (Path): File to write to. """ # Streams to pass to f90nml nml_file = StringIO(_mom6_input_str_to_nml_str(self._mom6_input_str_patched)) @@ -280,30 +289,30 @@ def write(self, file: Path): file.write_text(mom6_input_str) def _keys_to_upper(self): - """ - - :return: - """ + """Change all keys in dictionary to uppercase.""" for key in list(self.keys()): if not key.isupper(): self[key.upper()] = self.pop(key) def read_mom6_input(file_name: str) -> Mom6Input: - """ + """Read the contents of a MOM6 parameter file and return its contents as an instance of the MOM6Input class. + + Args: + file_name: Name of MOM6 parameter file to read. - :param file_name: - :return: + Returns: + MOM6Input: Contents of parameter file. """ return Mom6Input(file_name) def write_mom6_input(mom_input: [dict | Mom6Input], file: Path): - """ + """Write MOM6 parameters stored either as a dict of a MOM6Input to a file. - :param mom_input: - :param file: - :return: + Args: + mom_input (dict|MOM6Input): MOM6 parameters. + file (Path): File to write to. """ if isinstance(mom_input, Mom6Input): mom_input.write(file) diff --git a/om3utils/nuopc_config.py b/om3utils/nuopc_config.py index 7ee253e..6d7f51a 100644 --- a/om3utils/nuopc_config.py +++ b/om3utils/nuopc_config.py @@ -1,13 +1,82 @@ -"""NUOPC configuration""" +"""Utilities to handle NUOPC configuration files. + +The `nuopc.runconfig` files use by the CESM driver, and thus by ACCESS-OM3, are a mixture of two file formats: Resource +Files and Fortran Namelists. + +At the top-level, one has the Resource Files as implemented in ESMF. From the ESMF documentation: + + A Resource File (RF) is a text file consisting of list of label-value pairs. There is a limit of 1024 characters per + line and the Resource File can contain a maximum of 200 records. Each label should be followed by some data, the + value. An example Resource File follows. It is the file used in the example below. + + # This is an example Resource File. + # It contains a list of pairs. + # The colon after the label is required. + + # The values after the label can be an list. + # Multiple types are authorized. + + my_file_names: jan87.dat jan88.dat jan89.dat # all strings + constants: 3.1415 25 # float and integer + my_favorite_colors: green blue 022 + + + # Or, the data can be a list of single value pairs. + # It is simplier to retrieve data in this format: + + radius_of_the_earth: 6.37E6 + parameter_1: 89 + parameter_2: 78.2 + input_file_name: dummy_input.nc + + # Or, the data can be located in a table using the following + # syntax: + + my_table_name:: + 1000 3000 263.0 + 925 3000 263.0 + 850 3000 263.0 + 700 3000 269.0 + 500 3000 287.0 + 400 3000 295.8 + 300 3000 295.8 + :: + + Note that the colon after the label is required and that the double colon is required to declare tabular data. + +See https://earthsystemmodeling.org/docs/release/ESMF_8_6_0/ESMF_refdoc/node6.html#SECTION06090000000000000000 for +further details. + +The CESM driver then uses tables as defined in Resource Files to store Fortran Namelists instead of simple values: + + DRIVER_attributes:: + Verbosity = off + cime_model = cesm + logFilePostFix = .log + pio_blocksize = -1 + pio_rearr_comm_enable_hs_comp2io = .true. + pio_rearr_comm_enable_hs_io2comp = .false. + reprosum_diffmax = -1.000000D-08 + :: + + ALLCOMP_attributes:: + ATM_model = datm + GLC_model = sglc + OCN_model = mom + ocn2glc_levels = 1:10:19:26:30:33:35 + :: + +""" from pathlib import Path import re def _convert_from_string(value: str): - """Tries to convert a string to the most appropriate type. Leaves the string unchanged if not conversion succeeds. + """Tries to convert a string to the most appropriate type. Leaves it unchanged if conversion does not succeed. - :param value: value to convert. + Note that booleans use the Fortran syntax and real numbers in double precision can use the "old" Fortran `D` + delimiter. """ # Start by trying to convert from a Fortran logical to a Python bool if value.lower() == ".true.": @@ -30,9 +99,10 @@ def _convert_from_string(value: str): def _convert_to_string(value) -> str: - """Converts values to a string. + """Convert a value to a string. - :param value: value to convert. + Note that booleans are converted using the Fortran syntax and real numbers in double precision use the "old" Fortran + `D` delimiter for backward compatibility. """ if isinstance(value, bool): return ".true." if value else ".false." @@ -43,9 +113,13 @@ def _convert_to_string(value) -> str: def read_nuopc_config(file_name: str) -> dict: - """Read a NUOPC config file. + """Read a NUOPC config file and return its contents as a dictionary. + + Args: + file_name (str): File to read. - :param file_name: File name. + Returns: + dict: Contents of file. """ fname = Path(file_name) if not fname.is_file(): @@ -91,10 +165,11 @@ def read_nuopc_config(file_name: str) -> dict: def write_nuopc_config(config: dict, file: Path): - """Write a NUOPC config dictionary as a Resource File. + """Write a dictionary to a NUOPC config file. - :param config: Dictionary holding the NUOPC configuration to write - :param file: File to write to. + Args: + config (dict): NUOPC configuration to write. + file (Path): File to write to. """ with open(file, "w") as stream: for key, item in config.items(): diff --git a/om3utils/payu_config_yaml.py b/om3utils/payu_config_yaml.py index e2dbc34..9bdd837 100644 --- a/om3utils/payu_config_yaml.py +++ b/om3utils/payu_config_yaml.py @@ -1,13 +1,26 @@ -"""NUOPC configuration""" +"""Utilities to handle payu configuration files. + +Configuration for payu experiments are stored using YAML. Documentation about these files can be found here: + + https://payu.readthedocs.io/en/latest/config.html + +Round-trip parsing is supported by using the ruamel.yaml parser. +""" from pathlib import Path from ruamel.yaml import YAML, CommentedMap def read_payu_config_yaml(file_name: str) -> CommentedMap: - """Read a payu config file. + """Read a payu configuration file. - :param file_name: File name. + This function uses ruamel to parse the YAML file, so that we can do round-trip parsing. + + Args: + file_name: Name of file to read. + + Returns: + dict: Payu configuration. """ fname = Path(file_name) if not fname.is_file(): @@ -19,10 +32,10 @@ def read_payu_config_yaml(file_name: str) -> CommentedMap: def write_payu_config_yaml(config: [dict | CommentedMap], file: Path): - """Write a NUOPC config dictionary as a Resource File. + """Write a Payu configuration to a file. - :param config: Dictionary holding the payu config file to write - :param file: File to write to. + Args: + config (dict| CommentedMap): Payu configuration. + file(Path): File to write to. """ - YAML().dump(config, file) From f345c850e12de9c79392f0af8cc25e6a780f82e8 Mon Sep 17 00:00:00 2001 From: Micael Oliveira Date: Wed, 28 Feb 2024 13:58:41 +1100 Subject: [PATCH 5/5] Add a bare-bones README file. To be improved when the project is more mature. --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..472929e --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +# OM3Utils + +*A Python package of various utilities for the [ACCESS-OM3](https://github.com/COSIMA/access-om3) coupled ocean - sea ice - wave model.* + +![CI](https://github.com/COSIMA/om3-utils/actions/workflows/ci.yml/badge.svg) [![License](https://img.shields.io/badge/License-MPL2.0-a05a3f?style=flat-square)](https://opensource.org/licenses/MPL-2.0) [![codecov](https://codecov.io/gh/COSIMA/om3-utils/graph/badge.svg?token=gWLm5kXMcb)](https://codecov.io/gh/COSIMA/om3-utils) [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) + +Collection of utilities aimed at simplifying the creation and handling of ACCESS-OM3 runs. It currently includes: + - functions to read and write ACCESS-OM3 configuration files + - functions to read and process profiling data