From ac076aa2f9105f8cbdba1b0187df8eff194553ce Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Sat, 7 Sep 2024 09:22:22 -0600 Subject: [PATCH 01/10] Include script from I. Carlson This is the script that generates a CUPiD config file based on an existing yaml file. I'm committing it as it was in Ingrid's cesm2_3_beta17 sandbox, for use with CUPiD at hash b402eba; I think it will need to be updated for more recent versions. --- cesm_scripts/generate_cupid_config_file.py | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100755 cesm_scripts/generate_cupid_config_file.py diff --git a/cesm_scripts/generate_cupid_config_file.py b/cesm_scripts/generate_cupid_config_file.py new file mode 100755 index 0000000..fb4760f --- /dev/null +++ b/cesm_scripts/generate_cupid_config_file.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 + +#import packages +import yaml +from standard_script_setup import * +import os +from CIME.case import Case + +#Create variable for the caseroot environment variable +caseroot = os.getcwd() +#Open the config.yml file and create a dictionary with safe_load from the yaml package +with open('config_template.yml') as f: + my_dict = yaml.safe_load(f) + +# get environment cesm case variables +with Case(caseroot, read_only=False, record=True) as case: + cime_case = case.get_value('CASE') + #create variable to access cesm_output_dir + outdir = case.get_value('DOUT_S_ROOT') + +my_dict['global_params']['case_name'] = cime_case +my_dict['timeseries']['case_name'] = cime_case + +#create variable user to access the user environment variable +user = os.environ['USER'] +#replace USER with the environment variable +#my_dict['data_sources']['nb_path_root'] = f'/glade/u/home/{user}/CUPiD/examples/nblibrary' + +#replace with environment variable +my_dict['global_params']['CESM_output_dir'] = outdir + +#create new file, make it writeable +with open('config.yml', "w") as f: + #write a comment + f.write("# sample comment\n") + #enter in each element of the dictionary into the new file + yaml.dump(my_dict, f) \ No newline at end of file From 2b3687430173f23cdff5d476543c6181c0a9818a Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Sat, 7 Sep 2024 09:34:16 -0600 Subject: [PATCH 02/10] Clean-up to pass pre-commit checks --- cesm_scripts/generate_cupid_config_file.py | 45 +++++++++++----------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/cesm_scripts/generate_cupid_config_file.py b/cesm_scripts/generate_cupid_config_file.py index fb4760f..8dd3c46 100755 --- a/cesm_scripts/generate_cupid_config_file.py +++ b/cesm_scripts/generate_cupid_config_file.py @@ -1,37 +1,38 @@ #!/usr/bin/env python3 +from __future__ import annotations -#import packages -import yaml -from standard_script_setup import * import os + +import yaml from CIME.case import Case +from standard_script_setup import * # noqa: F401,F403 -#Create variable for the caseroot environment variable +# Create variable for the caseroot environment variable caseroot = os.getcwd() -#Open the config.yml file and create a dictionary with safe_load from the yaml package -with open('config_template.yml') as f: +# Open the config.yml file and create a dictionary with safe_load from the yaml package +with open("config_template.yml") as f: my_dict = yaml.safe_load(f) # get environment cesm case variables with Case(caseroot, read_only=False, record=True) as case: - cime_case = case.get_value('CASE') - #create variable to access cesm_output_dir - outdir = case.get_value('DOUT_S_ROOT') + cime_case = case.get_value("CASE") + # create variable to access cesm_output_dir + outdir = case.get_value("DOUT_S_ROOT") -my_dict['global_params']['case_name'] = cime_case -my_dict['timeseries']['case_name'] = cime_case +my_dict["global_params"]["case_name"] = cime_case +my_dict["timeseries"]["case_name"] = cime_case -#create variable user to access the user environment variable -user = os.environ['USER'] -#replace USER with the environment variable -#my_dict['data_sources']['nb_path_root'] = f'/glade/u/home/{user}/CUPiD/examples/nblibrary' +# create variable user to access the user environment variable +user = os.environ["USER"] +# replace USER with the environment variable +# my_dict['data_sources']['nb_path_root'] = f'/glade/u/home/{user}/CUPiD/examples/nblibrary' -#replace with environment variable -my_dict['global_params']['CESM_output_dir'] = outdir +# replace with environment variable +my_dict["global_params"]["CESM_output_dir"] = outdir -#create new file, make it writeable -with open('config.yml', "w") as f: - #write a comment +# create new file, make it writeable +with open("config.yml", "w") as f: + # write a comment f.write("# sample comment\n") - #enter in each element of the dictionary into the new file - yaml.dump(my_dict, f) \ No newline at end of file + # enter in each element of the dictionary into the new file + yaml.dump(my_dict, f) From 97fd4d42da292c54250d57a47a6c6a04532abb81 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Sun, 8 Sep 2024 12:32:22 -0600 Subject: [PATCH 03/10] Pass commandline arguments instead of using CIME generate_cupid_config_file.py now relies on argparse to get a handful of commandline arguments instead of importing some CIME functions and determining these values on the fly (it'll be easy to set all these arguments when creating the template) --- cesm_scripts/generate_cupid_config_file.py | 93 ++++++++++++++-------- 1 file changed, 61 insertions(+), 32 deletions(-) diff --git a/cesm_scripts/generate_cupid_config_file.py b/cesm_scripts/generate_cupid_config_file.py index 8dd3c46..0a5d433 100755 --- a/cesm_scripts/generate_cupid_config_file.py +++ b/cesm_scripts/generate_cupid_config_file.py @@ -1,38 +1,67 @@ #!/usr/bin/env python3 from __future__ import annotations +import argparse import os import yaml -from CIME.case import Case -from standard_script_setup import * # noqa: F401,F403 - -# Create variable for the caseroot environment variable -caseroot = os.getcwd() -# Open the config.yml file and create a dictionary with safe_load from the yaml package -with open("config_template.yml") as f: - my_dict = yaml.safe_load(f) - -# get environment cesm case variables -with Case(caseroot, read_only=False, record=True) as case: - cime_case = case.get_value("CASE") - # create variable to access cesm_output_dir - outdir = case.get_value("DOUT_S_ROOT") - -my_dict["global_params"]["case_name"] = cime_case -my_dict["timeseries"]["case_name"] = cime_case - -# create variable user to access the user environment variable -user = os.environ["USER"] -# replace USER with the environment variable -# my_dict['data_sources']['nb_path_root'] = f'/glade/u/home/{user}/CUPiD/examples/nblibrary' - -# replace with environment variable -my_dict["global_params"]["CESM_output_dir"] = outdir - -# create new file, make it writeable -with open("config.yml", "w") as f: - # write a comment - f.write("# sample comment\n") - # enter in each element of the dictionary into the new file - yaml.dump(my_dict, f) + + +def _parse_args(): + """Parse command line arguments""" + + parser = argparse.ArgumentParser( + description="Generate config.yml based on an existing CUPID example", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + + # Command line argument for name of case (required) + parser.add_argument("--case", action="store", dest="case", help="Name of CESM case") + + # Command line argument for name of case (required) + parser.add_argument( + "--cesm-output-dir", + action="store", + dest="cesm_output_dir", + help="Location of CESM history files (short-term archive)", + ) + + parser.add_argument( + "--cupid-root", + action="store", + dest="cupid_root", + help="Location of CUPiD in file system", + ) + + parser.add_argument( + "--cupid-example", + action="store", + dest="cupid_example", + default="key_metrics", + help="CUPiD example to use as template for config.yml", + ) + + return parser.parse_args() + + +def generate_cupid_config(case, cesm_output_dir, cupid_root, cupid_example): + with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: + my_dict = yaml.safe_load(f) + + my_dict["global_params"]["case_name"] = case + my_dict["timeseries"]["case_name"] = case + + # replace with environment variable + my_dict["global_params"]["CESM_output_dir"] = cesm_output_dir + + # create new file, make it writeable + with open("config.yml", "w") as f: + # write a comment + f.write("# sample comment\n") + # enter in each element of the dictionary into the new file + yaml.dump(my_dict, f, sort_keys=False) + + +if __name__ == "__main__": + args = vars(_parse_args()) + generate_cupid_config(**args) From a40ef5de3addff90c227167ca1a8243a67957ba1 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 9 Sep 2024 16:49:24 -0600 Subject: [PATCH 04/10] Rename script and directory --- .../generate_cupid_config_for_cesm_case.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cesm_scripts/generate_cupid_config_file.py => helper_scripts/generate_cupid_config_for_cesm_case.py (100%) diff --git a/cesm_scripts/generate_cupid_config_file.py b/helper_scripts/generate_cupid_config_for_cesm_case.py similarity index 100% rename from cesm_scripts/generate_cupid_config_file.py rename to helper_scripts/generate_cupid_config_for_cesm_case.py From 18426491fd6b0fd2783a2d6af930b8ad5db08b75 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Mon, 9 Sep 2024 17:08:53 -0600 Subject: [PATCH 05/10] Clean up based on review 1. Only required command line argument is cesm root directory; also allows optional arguments for case directory (default is current dir) and cupid example (default is key_metrics) 2. Now gets case name and short-term archive directory from CESM Case class 3. Cleaned up comment inserted at top of config.yml --- .../generate_cupid_config_for_cesm_case.py | 44 ++++++++++++------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 0a5d433..a2fbb41 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -3,6 +3,7 @@ import argparse import os +import sys import yaml @@ -15,24 +16,15 @@ def _parse_args(): formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - # Command line argument for name of case (required) - parser.add_argument("--case", action="store", dest="case", help="Name of CESM case") - - # Command line argument for name of case (required) - parser.add_argument( - "--cesm-output-dir", - action="store", - dest="cesm_output_dir", - help="Location of CESM history files (short-term archive)", - ) - + # Command line argument for location of CESM source code (required) parser.add_argument( - "--cupid-root", + "--cesm-root", action="store", - dest="cupid_root", - help="Location of CUPiD in file system", + dest="cesm_root", + help="Location of CESM source code", ) + # Command line argument for CUPiD example from which to get config.yml parser.add_argument( "--cupid-example", action="store", @@ -41,10 +33,27 @@ def _parse_args(): help="CUPiD example to use as template for config.yml", ) + # Command line argument location of CESM case directory + parser.add_argument( + "--case-root", + action="store", + dest="case_root", + default=os.getcwd(), + help="CESM case directory", + ) + return parser.parse_args() -def generate_cupid_config(case, cesm_output_dir, cupid_root, cupid_example): +def generate_cupid_config(case_root, cesm_root, cupid_example): + sys.path.append(os.path.join(cesm_root, "cime")) + from CIME.case import Case + + with Case(case_root, read_only=False, record=True) as cesm_case: + case = cesm_case.get_value("CASE") + dout_s_root = cesm_case.get_value("DOUT_S_ROOT") + cupid_root = os.path.join(cesm_root, "tools", "CUPiD") + with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) @@ -52,12 +61,13 @@ def generate_cupid_config(case, cesm_output_dir, cupid_root, cupid_example): my_dict["timeseries"]["case_name"] = case # replace with environment variable - my_dict["global_params"]["CESM_output_dir"] = cesm_output_dir + my_dict["global_params"]["CESM_output_dir"] = dout_s_root # create new file, make it writeable with open("config.yml", "w") as f: # write a comment - f.write("# sample comment\n") + f.write(f"# This file has been auto-generated for use with {case}\n") + f.write(f"# It is based off of examples/{cupid_example}/config.yml\n") # enter in each element of the dictionary into the new file yaml.dump(my_dict, f, sort_keys=False) From 1466ad40a727909179511b75bfbe439cc6700822 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 19 Sep 2024 15:35:00 -0600 Subject: [PATCH 06/10] Change more config values generate_cupid_config_for_cesm_case.py now updates more global parameters (start_date, end_date, climo_nyears, as well as the base_ copies) and the time-series end_years. For now these values are hard-coded in with placeholders until we can get them from env_cupid.xml --- .../generate_cupid_config_for_cesm_case.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index a2fbb41..c799b7e 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -54,11 +54,28 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): dout_s_root = cesm_case.get_value("DOUT_S_ROOT") cupid_root = os.path.join(cesm_root, "tools", "CUPiD") + # Additional options we need to get from env_cupid.xml + nyears = 1 + start_date = "0001-01-01" + end_date = f"{nyears+1:04d}-01-01" + climo_nyears = 1 + base_case_output_dir = "/glade/campaign/cesm/development/cross-wg/diagnostic_framework/CESM_output_for_testing" + base_nyears = 100 + base_end_date = f"{base_nyears+1:04d}-01-01" + base_climo_nyears = 40 + with open(os.path.join(cupid_root, "examples", cupid_example, "config.yml")) as f: my_dict = yaml.safe_load(f) my_dict["global_params"]["case_name"] = case + my_dict["global_params"]["start_date"] = start_date + my_dict["global_params"]["end_date"] = end_date + my_dict["global_params"]["climo_nyears"] = climo_nyears + my_dict["global_params"]["base_case_output_dir"] = base_case_output_dir + my_dict["global_params"]["base_end_date"] = base_end_date + my_dict["global_params"]["base_climo_nyears"] = base_climo_nyears my_dict["timeseries"]["case_name"] = case + my_dict["timeseries"]["atm"]["end_years"] = [nyears, base_nyears] # replace with environment variable my_dict["global_params"]["CESM_output_dir"] = dout_s_root From a52a042173a53ed77425a56f5d17521a50479243 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 19 Sep 2024 15:44:39 -0600 Subject: [PATCH 07/10] generate_cupid_config... requires --cesm-root I thought that not specifying a default value made it required, but really it just made the default None (which had all sorts of fun implications in the code) --- helper_scripts/generate_cupid_config_for_cesm_case.py | 1 + 1 file changed, 1 insertion(+) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index c799b7e..2e0d12f 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -21,6 +21,7 @@ def _parse_args(): "--cesm-root", action="store", dest="cesm_root", + required=True, help="Location of CESM source code", ) From bb3f37d8f538b705727cf29b35306909f52dbaf8 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Thu, 19 Sep 2024 16:04:48 -0600 Subject: [PATCH 08/10] Add valid values for --cupid-example We don't know the full list of valid values when parsing command line arguments, so this check needs to be done in the body of the python script --- .../generate_cupid_config_for_cesm_case.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 2e0d12f..5900232 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -50,10 +50,22 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): sys.path.append(os.path.join(cesm_root, "cime")) from CIME.case import Case + # Is cupid_example a valid value? + cupid_root = os.path.join(cesm_root, "tools", "CUPiD") + cupid_examples = os.path.join(cupid_root, "examples") + valid_examples = [ + example + for example in next(os.walk(cupid_examples))[1] + if example not in ["ilamb", "nblibrary"] + ] + if cupid_example not in valid_examples: + raise KeyError( + f"--cupid-example must be a subdirectory of {cupid_examples} ({valid_examples})", + ) + with Case(case_root, read_only=False, record=True) as cesm_case: case = cesm_case.get_value("CASE") dout_s_root = cesm_case.get_value("DOUT_S_ROOT") - cupid_root = os.path.join(cesm_root, "tools", "CUPiD") # Additional options we need to get from env_cupid.xml nyears = 1 From 0888657f348856200f26edf00b9e5e6f117fc51a Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 20 Sep 2024 10:17:22 -0600 Subject: [PATCH 09/10] Better error message for bad --cupid-example --- helper_scripts/generate_cupid_config_for_cesm_case.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index 5900232..c739e3e 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -59,8 +59,9 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): if example not in ["ilamb", "nblibrary"] ] if cupid_example not in valid_examples: + error_msg = f"argument --cupid-example: invalid choice '{cupid_example}'" raise KeyError( - f"--cupid-example must be a subdirectory of {cupid_examples} ({valid_examples})", + f"{error_msg} (choose from subdirectories of {cupid_examples}: {valid_examples})", ) with Case(case_root, read_only=False, record=True) as cesm_case: From dba6bea202b05e7785b11cc342a05423c1f3e7e8 Mon Sep 17 00:00:00 2001 From: Michael Levy Date: Fri, 20 Sep 2024 10:35:42 -0600 Subject: [PATCH 10/10] Note script arguments in generated yaml file The header of the YAML file is a little bit bigger; in addition to mentioning the file was auto-generated, it includes each of the three arguments (cesm_root, case_root, and cupid_example) --- helper_scripts/generate_cupid_config_for_cesm_case.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/helper_scripts/generate_cupid_config_for_cesm_case.py b/helper_scripts/generate_cupid_config_for_cesm_case.py index c739e3e..ce2c87d 100755 --- a/helper_scripts/generate_cupid_config_for_cesm_case.py +++ b/helper_scripts/generate_cupid_config_for_cesm_case.py @@ -96,9 +96,14 @@ def generate_cupid_config(case_root, cesm_root, cupid_example): # create new file, make it writeable with open("config.yml", "w") as f: - # write a comment + # Header of file is a comment logging provenance f.write(f"# This file has been auto-generated for use with {case}\n") f.write(f"# It is based off of examples/{cupid_example}/config.yml\n") + f.write("# Arguments used:\n") + f.write(f"# cesm_root = {cesm_root}\n") + f.write(f"# case_root = {case_root}\n") + f.write(f"# cupid_example= {cupid_example}\n") + # enter in each element of the dictionary into the new file yaml.dump(my_dict, f, sort_keys=False)