From 2744f7943f4c4029f7fbee8e7a0e784358f95e05 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 13 Feb 2024 10:21:15 -0700 Subject: [PATCH 01/10] working version of refactoring --- .gitignore | 1 + cime_config/buildnml | 4 +- python/ctsm/site_and_regional/neon_site.py | 198 ++-------------- python/ctsm/site_and_regional/run_neon.py | 2 +- python/ctsm/site_and_regional/tower_site.py | 246 ++++++++++++++++++++ 5 files changed, 270 insertions(+), 181 deletions(-) create mode 100644 python/ctsm/site_and_regional/tower_site.py diff --git a/.gitignore b/.gitignore index ca701132a7..ec2a3b17f6 100644 --- a/.gitignore +++ b/.gitignore @@ -111,6 +111,7 @@ unit_test_build /tools/site_and_regional/????.ad/ /tools/site_and_regional/????.postad/ /tools/site_and_regional/????.transient/ +/tools/site_and_regional/archive/ # build output *.o diff --git a/cime_config/buildnml b/cime_config/buildnml index 84e1581406..0521830616 100755 --- a/cime_config/buildnml +++ b/cime_config/buildnml @@ -144,8 +144,8 @@ def buildnml(case, caseroot, compname): or clm_usrdat_name is "NEON.PRISM" ): logger.warning( - "WARNING: Do you have approriprate initial conditions for this simulation?" - + " Check that the finidat file used in the lnd_in namelist is apprporiately spunup for your case" + "WARNING: Do you have appropriate initial conditions for this simulation?" + + " Check that the finidat file used in the lnd_in namelist is appropriately spunup for your case" ) if comp_atm != "datm": diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 31ae78f5ad..707c27e575 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -15,6 +15,10 @@ _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) sys.path.insert(1, _CTSM_PYTHON) +# -- import local classes for this script +# pylint: disable=wrong-import-position +from ctsm.site_and_regional.tower_site import TowerSite + # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root @@ -27,126 +31,23 @@ # pylint: disable=too-many-instance-attributes -class NeonSite: +class NeonSite(TowerSite): """ A class for encapsulating neon sites. """ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): - self.name = name - self.start_year = int(start_year) - self.end_year = int(end_year) - self.start_month = int(start_month) - self.end_month = int(end_month) - self.cesmroot = path_to_ctsm_root() - self.finidat = finidat + super().__init__(name, start_year, end_year, start_month, end_month, finidat) def build_base_case( self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False ): - """ - Function for building a base_case to clone. - To spend less time on building ctsm for the neon cases, - all the other cases are cloned from this case - - Args: - self: - The NeonSite object - base_root (str): - root of the base_case CIME - res (str): - base_case resolution or gridname - compset (str): - base case compset - overwrite (bool) : - Flag to overwrite the case if exists - """ - print("---- building a base case -------") - # pylint: disable=attribute-defined-outside-init - self.base_case_root = output_root - # pylint: enable=attribute-defined-outside-init - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] - if not output_root: - output_root = os.getcwd() - case_path = os.path.join(output_root, self.name) - - logger.info("base_case_name : %s", self.name) - logger.info("user_mods_dir : %s", user_mods_dirs[0]) - - if overwrite and os.path.isdir(case_path): - print("Removing the existing case at: {}".format(case_path)) - shutil.rmtree(case_path) - - with Case(case_path, read_only=False) as case: - if not os.path.isdir(case_path): - print("---- creating a base case -------") + case_path = super().build_base_case(cesmroot, output_root, res, compset) - case.create( - case_path, - cesmroot, - compset, - res, - run_unsupported=True, - answer="r", - output_root=output_root, - user_mods_dirs=user_mods_dirs, - driver="nuopc", - ) - - print("---- base case created ------") - - # --change any config for base_case: - # case.set_value("RUN_TYPE","startup") - print("---- base case setup ------") - case.case_setup() - else: - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - # reset the case - case.case_setup(reset=True) - case_path = case.get_value("CASEROOT") - - if setup_only: - return case_path - - print("---- base case build ------") - print("--- This may take a while and you may see WARNING messages ---") - # always walk through the build process to make sure it's up to date. - initial_time = time.time() - build.case_build(case_path, case=case) - end_time = time.time() - total = end_time - initial_time - print("Time required to building the base case: {} s.".format(total)) - # update case_path to be the full path to the base case return case_path - # pylint: disable=no-self-use def get_batch_query(self, case): - """ - Function for querying the batch queue query command for a case, depending on the - user's batch system. - - Args: - case: - case object - """ - - if case.get_value("BATCH_SYSTEM") == "none": - return "none" - return case.get_value("batch_query") + return super().get_batch_query(case) # pylint: disable=too-many-statements def run_case( @@ -190,7 +91,7 @@ def run_case( """ user_mods_dirs = [ os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) - ] + ] #TODO: bring into neon_site expect( os.path.isdir(base_case_root), "Error base case does not exist in {}".format(base_case_root), @@ -272,9 +173,9 @@ def run_case( case.set_value("STOP_OPTION", "ndays") case.set_value("REST_OPTION", "end") case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) + case.set_value("NEONVERSION", version) #TODO: put in neon_site if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") + case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") #TODO: put in neon_site if run_type == "ad": case.set_value("CLM_FORCE_COLDSTART", "on") @@ -321,74 +222,15 @@ def run_case( print(f"Use {batch_query} to check its run status") def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True + super().set_ref_case(case) + return True ### Check if super returns false, if this will still return True? def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", - """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', - 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO'""", - ] + # TODO: include neon-specific user namelist lines, using this as just an example currently + site_lines = [ + """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', + 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO',""" + ] + super().modify_user_nl(case_root, run_type, rundir, site_lines) + - if user_nl_lines: - with open(user_nl_fname, "a") as nl_file: - for line in user_nl_lines: - nl_file.write("{}\n".format(line)) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 72bf3fdfb4..7e9195413a 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -176,7 +176,7 @@ def main(description): # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob( os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*") - ) + ) # TODO: This is currently including FATES and a secondary list of all sites... valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) ( diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py new file mode 100644 index 0000000000..1d1bf453be --- /dev/null +++ b/python/ctsm/site_and_regional/tower_site.py @@ -0,0 +1,246 @@ +""" +This module includes the definition for the TowerSite class, +which has NeonSite and Plumber2Site child classes. This class defines common +functionalities that are in both NeonSite and Plumber2Site classes. +""" +# -- Import libraries + +# -- standard libraries +import os.path +import glob +import logging +import re +import shutil +import sys +import time + +# Get the ctsm util tools and then the cime tools. +_CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) +sys.path.insert(1, _CTSM_PYTHON) + +# pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order +from ctsm import add_cime_to_path +from ctsm.path_utils import path_to_ctsm_root + +from CIME import build +from CIME.case import Case +from CIME.utils import safe_copy, expect, symlink_force + +logger = logging.getLogger(__name__) + + +# pylint: disable=too-many-instance-attributes +class TowerSite: + """ + Parent class to NeonSite and Plumber2Site classes. + ... + Attributes + ---------- + Methods + ------- + """ + + def __init__(self, name, start_year, end_year, start_month, end_month, finidat): + """ + Initializes TowerSite with the given arguments. + Parameters + ---------- + """ + self.name = name + self.start_year = int(start_year) + self.end_year = int(end_year) + self.start_month = int(start_month) + self.end_month = int(end_month) + self.cesmroot = path_to_ctsm_root() + self.finidat = finidat + + def __str__(self): + """ + Converts ingredients of the TowerSite to string for printing. + """ + return "{}\n{}".format( + str(self.__class__), + "\n".join( + ( + "{} = {}".format(str(key), str(self.__dict__[key])) + for key in sorted(self.__dict__) + ) + ), + ) + + def build_base_case( + self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + ): + """ + Function for building a base_case to clone. + To spend less time on building ctsm for the neon cases, + all the other cases are cloned from this case + Args: + self: + The NeonSite object + base_root (str): + root of the base_case CIME + res (str): + base_case resolution or gridname + compset (str): + base case compset + overwrite (bool) : + Flag to overwrite the case if exists + """ + print("---- building a base case -------") + # pylint: disable=attribute-defined-outside-init + self.base_case_root = output_root + # pylint: enable=attribute-defined-outside-init + user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] + if not output_root: + output_root = os.getcwd() + case_path = os.path.join(output_root, self.name) + + logger.info("base_case_name : %s", self.name) + logger.info("user_mods_dir : %s", user_mods_dirs[0]) + + if overwrite and os.path.isdir(case_path): + print("Removing the existing case at: {}".format(case_path)) + shutil.rmtree(case_path) + + with Case(case_path, read_only=False) as case: + if not os.path.isdir(case_path): + print("---- creating a base case -------") + case.create( + case_path, + cesmroot, + compset, + res, + run_unsupported=True, + answer="r", + output_root=output_root, + user_mods_dirs=user_mods_dirs, + driver="nuopc", + ) + + print("---- base case created ------") + + # --change any config for base_case: + # case.set_value("RUN_TYPE","startup") + print("---- base case setup ------") + case.case_setup() + else: + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + # reset the case + case.case_setup(reset=True) + case_path = case.get_value("CASEROOT") + + if setup_only: + return case_path + + print("---- base case build ------") + print("--- This may take a while and you may see WARNING messages ---") + # always walk through the build process to make sure it's up to date. + initial_time = time.time() + build.case_build( + case_path, case=case + ) # TODO: this causes issues if run from tower_site.py + end_time = time.time() + total = end_time - initial_time + print("Time required to building the base case: {} s.".format(total)) + # update case_path to be the full path to the base case + return case_path + + # pylint: disable=no-self-use + def get_batch_query(self, case): + """ + Function for querying the batch queue query command for a case, depending on the + user's batch system. + Args: + case: + case object + """ + + if case.get_value("BATCH_SYSTEM") == "none": + return "none" + return case.get_value("batch_query") + + def modify_user_nl(self, case_root, run_type, rundir, site_lines): + """ + Modify user namelist. If transient, include finidat in user_nl; + Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. + """ + user_nl_fname = os.path.join(case_root, "user_nl_clm") + user_nl_lines = None + if run_type == "transient": + if self.finidat: + user_nl_lines = [ + "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) + ] + else: + user_nl_lines = [ + "hist_fincl2 = ''", + "hist_mfilt = 20", + "hist_nhtfrq = -8760", + "hist_empty_htapes = .true.", + ] + site_lines + + if user_nl_lines: + with open(user_nl_fname, "a") as nl_file: + for line in user_nl_lines: + nl_file.write("{}\n".format(line)) + + def set_ref_case(self, case): + """ + Set an existing case as the reference case, eg for use with spinup. + """ + rundir = case.get_value("RUNDIR") + case_root = case.get_value("CASEROOT") + if case_root.endswith(".postad"): + ref_case_root = case_root.replace(".postad", ".ad") + root = ".ad" + else: + ref_case_root = case_root.replace(".transient", ".postad") + root = ".postad" + if not os.path.isdir(ref_case_root): + logger.warning( + "ERROR: spinup must be completed first, could not find directory %s", ref_case_root + ) + return False + + with Case(ref_case_root) as refcase: + refrundir = refcase.get_value("RUNDIR") + case.set_value("RUN_REFDIR", refrundir) + case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) + refdate = None + for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): + m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) + if m_searched: + refdate = m_searched.group(1) + symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) + logger.info("Found refdate of %s", refdate) + if not refdate: + logger.warning("Could not find refcase for %s", case_root) + return False + + for rpfile in glob.iglob(refrundir + "/rpointer*"): + safe_copy(rpfile, rundir) + if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( + os.path.join(refrundir, "inputdata") + ): + symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) + + case.set_value("RUN_REFDATE", refdate) + if case_root.endswith(".postad"): + case.set_value("RUN_STARTDATE", refdate) + # NOTE: if start options are set, RUN_STARTDATE should be modified here + return True From 68f69576beddbd726b81182c402351dcfce04a37 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 13 Feb 2024 11:51:26 -0700 Subject: [PATCH 02/10] extended functions --- python/ctsm/site_and_regional/neon_site.py | 139 +-------------- python/ctsm/site_and_regional/run_neon.py | 2 +- python/ctsm/site_and_regional/tower_site.py | 177 +++++++++++++++++++- 3 files changed, 181 insertions(+), 137 deletions(-) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 707c27e575..ded08585e1 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -42,7 +42,10 @@ def __init__(self, name, start_year, end_year, start_month, end_month, finidat): def build_base_case( self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False ): - case_path = super().build_base_case(cesmroot, output_root, res, compset) + user_mods_dirs = [ + os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) + ] + case_path = super().build_base_case(cesmroot, output_root, res, compset, user_mods_dirs) return case_path @@ -91,135 +94,11 @@ def run_case( """ user_mods_dirs = [ os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) - ] #TODO: bring into neon_site - expect( - os.path.isdir(base_case_root), - "Error base case does not exist in {}".format(base_case_root), + ] + tower_type = "NEON" + super().run_case( + base_case_root, run_type, prism, run_length, tower_type, user_mods_dirs, user_version ) - # -- if user gives a version: - if user_version: - version = user_version - else: - version = "latest" - - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - # pylint: disable=undefined-variable - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - # pylint: enable=undefined-variable - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) #TODO: put in neon_site - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") #TODO: put in neon_site - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") def set_ref_case(self, case): super().set_ref_case(case) @@ -232,5 +111,3 @@ def modify_user_nl(self, case_root, run_type, rundir): 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO',""" ] super().modify_user_nl(case_root, run_type, rundir, site_lines) - - diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 7e9195413a..39a7806eec 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -176,7 +176,7 @@ def main(description): # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob( os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*") - ) # TODO: This is currently including FATES and a secondary list of all sites... + ) # TODO: This is currently including FATES and a secondary list of all sites... valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) ( diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index 1d1bf453be..1d07fec816 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -69,7 +69,7 @@ def __str__(self): ) def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + self, cesmroot, output_root, res, compset, user_mods_dirs, overwrite=False, setup_only=False ): """ Function for building a base_case to clone. @@ -91,7 +91,6 @@ def build_base_case( # pylint: disable=attribute-defined-outside-init self.base_case_root = output_root # pylint: enable=attribute-defined-outside-init - user_mods_dirs = [os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", self.name)] if not output_root: output_root = os.getcwd() case_path = os.path.join(output_root, self.name) @@ -151,9 +150,7 @@ def build_base_case( print("--- This may take a while and you may see WARNING messages ---") # always walk through the build process to make sure it's up to date. initial_time = time.time() - build.case_build( - case_path, case=case - ) # TODO: this causes issues if run from tower_site.py + build.case_build(case_path, case=case) end_time = time.time() total = end_time - initial_time print("Time required to building the base case: {} s.".format(total)) @@ -244,3 +241,173 @@ def set_ref_case(self, case): case.set_value("RUN_STARTDATE", refdate) # NOTE: if start options are set, RUN_STARTDATE should be modified here return True + + # pylint: disable=too-many-statements + def run_case( + self, + base_case_root, + run_type, + prism, + run_length, + tower_type, + user_mods_dirs, + user_version, + overwrite=False, + setup_only=False, + no_batch=False, + rerun=False, + experiment=False, + ): + """ + Run case. + + Args: + self + base_case_root: str, opt + file path of base case + run_type: str, opt + transient, post_ad, or ad case, default transient + prism: bool, opt + if True, use PRISM precipitation, default False + run_length: str, opt + length of run, default '4Y' + user_version: str, opt + default 'latest' + overwrite: bool, opt + default False + setup_only: bool, opt + default False; if True, set up but do not run case + no_batch: bool, opt + default False + rerun: bool, opt + default False + experiment: str, opt + name of experiment, default False + """ + expect( + os.path.isdir(base_case_root), + "Error base case does not exist in {}".format(base_case_root), + ) + # -- if user gives a version: + if user_version: + version = user_version + else: + version = "latest" + + print("using this version:", version) + + if experiment is not False: + self.name = self.name + "." + experiment + case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) + + rundir = None + if os.path.isdir(case_root): + if overwrite: + print("---- removing the existing case -------") + shutil.rmtree(case_root) + elif rerun: + with Case(case_root, read_only=False) as case: + rundir = case.get_value("RUNDIR") + # For existing case check that the compset name is correct + existingcompname = case.get_value("COMPSET") + match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) + # pylint: disable=undefined-variable + if re.search("^HIST", compset, flags=re.IGNORECASE) is None: + expect( + match is None, + """Existing base case is a historical type and should not be + --rerun with the --overwrite option""", + ) + # pylint: enable=undefined-variable + else: + expect( + match is not None, + """Existing base case should be a historical type and is not + --rerun with the --overwrite option""", + ) + if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): + print("Case {} appears to be complete, not rerunning.".format(case_root)) + elif not setup_only: + print("Resubmitting case {}".format(case_root)) + case.submit(no_batch=no_batch) + print("-----------------------------------") + print("Successfully submitted case!") + batch_query = self.get_batch_query(case) + if batch_query != "none": + print(f"Use {batch_query} to check its run status") + return + else: + logger.warning("Case already exists in %s, not overwritting", case_root) + return + if run_type == "postad": + adcase_root = case_root.replace(".postad", ".ad") + if not os.path.isdir(adcase_root): + logger.warning("postad requested but no ad case found in %s", adcase_root) + return + + if not os.path.isdir(case_root): + # read_only = False should not be required here + with Case(base_case_root, read_only=False) as basecase: + print("---- cloning the base case in {}".format(case_root)) + # + # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although + # it causes some of the user_nl_* files to have duplicated inputs. It also ensures + # that the shell_commands file is copied, as well as taking care of the DATM inputs. + # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 + # + basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) + + with Case(case_root, read_only=False) as case: + if run_type != "transient": + # in order to avoid the complication of leap years, + # we always set the run_length in units of days. + case.set_value("STOP_OPTION", "ndays") + case.set_value("REST_OPTION", "end") + case.set_value("CONTINUE_RUN", False) + if tower_type == "NEON": + case.set_value("NEONVERSION", version) + if prism: + case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") + + if run_type == "ad": + case.set_value("CLM_FORCE_COLDSTART", "on") + case.set_value("CLM_ACCELERATED_SPINUP", "on") + case.set_value("RUN_REFDATE", "0018-01-01") + case.set_value("RUN_STARTDATE", "0018-01-01") + case.set_value("RESUBMIT", 1) + case.set_value("STOP_N", run_length) + + else: + case.set_value("CLM_FORCE_COLDSTART", "off") + case.set_value("CLM_ACCELERATED_SPINUP", "off") + case.set_value("RUN_TYPE", "hybrid") + + if run_type == "postad": + self.set_ref_case(case) + case.set_value("STOP_N", run_length) + + # For transient cases STOP will be set in the user_mod_directory + if run_type == "transient": + if self.finidat: + case.set_value("RUN_TYPE", "startup") + else: + if not self.set_ref_case(case): + return + case.set_value("CALENDAR", "GREGORIAN") + case.set_value("RESUBMIT", 0) + case.set_value("STOP_OPTION", "nmonths") + if not rundir: + rundir = case.get_value("RUNDIR") + + self.modify_user_nl(case_root, run_type, rundir) + + case.create_namelists() + # explicitly run check_input_data + case.check_all_input_data() + if not setup_only: + case.submit(no_batch=no_batch) + print("-----------------------------------") + print("Successfully submitted case!") + batch_query = self.get_batch_query(case) + if batch_query != "none": + print(f"Use {batch_query} to check its run status") From f48bb75b0f2c9e49eaf069f0e9a342c3c4e26860 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 13 Feb 2024 14:11:27 -0700 Subject: [PATCH 03/10] fix argument errors --- python/ctsm/site_and_regional/neon_site.py | 42 ++++++++++++--------- python/ctsm/site_and_regional/run_neon.py | 3 +- python/ctsm/site_and_regional/tower_site.py | 6 +-- 3 files changed, 29 insertions(+), 22 deletions(-) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index ded08585e1..75dca35691 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -3,13 +3,9 @@ """ # Import libraries -import glob import logging import os -import re -import shutil import sys -import time # Get the ctsm util tools and then the cime tools. _CTSM_PYTHON = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "python")) @@ -36,21 +32,28 @@ class NeonSite(TowerSite): A class for encapsulating neon sites. """ - def __init__(self, name, start_year, end_year, start_month, end_month, finidat): - super().__init__(name, start_year, end_year, start_month, end_month, finidat) - def build_base_case( - self, cesmroot, output_root, res, compset, overwrite=False, setup_only=False + self, + cesmroot, + output_root, + res, + compset, + user_mods_dirs=None, + overwrite=False, + setup_only=False, ): - user_mods_dirs = [ - os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) - ] + if user_mods_dirs is None: + user_mods_dirs = [ + os.path.join(self.cesmroot, "cime_config", "usermods_dirs", "NEON", self.name) + ] + print("in neonsite adding usermodsdirs") + print("usermodsdirs: {}".format(user_mods_dirs)) case_path = super().build_base_case(cesmroot, output_root, res, compset, user_mods_dirs) return case_path - def get_batch_query(self, case): - return super().get_batch_query(case) + # def get_batch_query(self, case): + # return super().get_batch_query(case) # pylint: disable=too-many-statements def run_case( @@ -60,6 +63,8 @@ def run_case( prism, run_length, user_version, + tower_type=None, + user_mods_dirs=None, overwrite=False, setup_only=False, no_batch=False, @@ -97,17 +102,18 @@ def run_case( ] tower_type = "NEON" super().run_case( - base_case_root, run_type, prism, run_length, tower_type, user_mods_dirs, user_version + base_case_root, run_type, prism, run_length, user_version, tower_type, user_mods_dirs ) def set_ref_case(self, case): super().set_ref_case(case) return True ### Check if super returns false, if this will still return True? - def modify_user_nl(self, case_root, run_type, rundir): + def modify_user_nl(self, case_root, run_type, rundir, site_lines=None): # TODO: include neon-specific user namelist lines, using this as just an example currently - site_lines = [ - """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', + if site_lines is None: + site_lines = [ + """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO',""" - ] + ] super().modify_user_nl(case_root, run_type, rundir, site_lines) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 39a7806eec..e4dcbb2596 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -220,8 +220,9 @@ def main(description): if run_from_postad: neon_site.finidat = None if not base_case_root: + user_mods_dirs = None base_case_root = neon_site.build_base_case( - cesmroot, output_root, res, compset, overwrite, setup_only + cesmroot, output_root, res, compset, user_mods_dirs, overwrite, setup_only ) logger.info("-----------------------------------") logger.info("Running CTSM for neon site : %s", neon_site.name) diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index 1d07fec816..82bd7f872d 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -171,7 +171,7 @@ def get_batch_query(self, case): return "none" return case.get_value("batch_query") - def modify_user_nl(self, case_root, run_type, rundir, site_lines): + def modify_user_nl(self, case_root, run_type, rundir, site_lines=None): """ Modify user namelist. If transient, include finidat in user_nl; Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. @@ -249,9 +249,9 @@ def run_case( run_type, prism, run_length, + user_version, tower_type, user_mods_dirs, - user_version, overwrite=False, setup_only=False, no_batch=False, @@ -399,7 +399,7 @@ def run_case( if not rundir: rundir = case.get_value("RUNDIR") - self.modify_user_nl(case_root, run_type, rundir) + self.modify_user_nl(case_root, run_type, rundir) # TODO: add site_lines argument? case.create_namelists() # explicitly run check_input_data From e7d1505765c49b6197c657a387a327e30dde0bea Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 13 Feb 2024 14:29:32 -0700 Subject: [PATCH 04/10] fix FATES being listed as a valid neon site... --- python/ctsm/site_and_regional/run_neon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index e4dcbb2596..6d0108bf95 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -175,8 +175,8 @@ def main(description): cesmroot = path_to_ctsm_root() # Get the list of supported neon sites from usermods valid_neon_sites = glob.glob( - os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!d]*") - ) # TODO: This is currently including FATES and a secondary list of all sites... + os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!Fd]*") + ) valid_neon_sites = sorted([v.split("/")[-1] for v in valid_neon_sites]) ( From faa18ab699ada92ee3f98ce34019034b17a32621 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Tue, 13 Feb 2024 14:31:18 -0700 Subject: [PATCH 05/10] todo update --- python/ctsm/site_and_regional/tower_site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index 82bd7f872d..af3a04e93e 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -399,7 +399,7 @@ def run_case( if not rundir: rundir = case.get_value("RUNDIR") - self.modify_user_nl(case_root, run_type, rundir) # TODO: add site_lines argument? + self.modify_user_nl(case_root, run_type, rundir) case.create_namelists() # explicitly run check_input_data From fed0836121f3aba682f1ff96c849283f4463a3a2 Mon Sep 17 00:00:00 2001 From: Teagan King Date: Fri, 23 Feb 2024 15:50:04 -0700 Subject: [PATCH 06/10] updates from Erik's comments --- python/ctsm/site_and_regional/neon_site.py | 7 ------- python/ctsm/site_and_regional/run_neon.py | 2 ++ python/ctsm/site_and_regional/tower_site.py | 1 + 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index 75dca35691..b3944bb95d 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -52,9 +52,6 @@ def build_base_case( return case_path - # def get_batch_query(self, case): - # return super().get_batch_query(case) - # pylint: disable=too-many-statements def run_case( self, @@ -105,10 +102,6 @@ def run_case( base_case_root, run_type, prism, run_length, user_version, tower_type, user_mods_dirs ) - def set_ref_case(self, case): - super().set_ref_case(case) - return True ### Check if super returns false, if this will still return True? - def modify_user_nl(self, case_root, run_type, rundir, site_lines=None): # TODO: include neon-specific user namelist lines, using this as just an example currently if site_lines is None: diff --git a/python/ctsm/site_and_regional/run_neon.py b/python/ctsm/site_and_regional/run_neon.py index 6d0108bf95..d831f8dba2 100755 --- a/python/ctsm/site_and_regional/run_neon.py +++ b/python/ctsm/site_and_regional/run_neon.py @@ -174,6 +174,8 @@ def main(description): """ cesmroot = path_to_ctsm_root() # Get the list of supported neon sites from usermods + # The [!Fd]* portion means that we won't retrieve cases that start with: + # F (FATES) or d (default). We should be aware of adding cases that start with these. valid_neon_sites = glob.glob( os.path.join(cesmroot, "cime_config", "usermods_dirs", "NEON", "[!Fd]*") ) diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index af3a04e93e..0145aedeee 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -243,6 +243,7 @@ def set_ref_case(self, case): return True # pylint: disable=too-many-statements + # TODO: This code should be broken up into smaller pieces def run_case( self, base_case_root, From e58cc394d09b2e15c318789577fcd56fa6fb930b Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Mar 2024 00:25:47 -0700 Subject: [PATCH 07/10] Save the current directory at setup and return to it before deleting the temporary directory, this is handling for #2405 --- python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py | 2 ++ python/ctsm/test/test_sys_fsurdat_modifier.py | 2 ++ python/ctsm/test/test_sys_lilac_build_ctsm.py | 2 ++ python/ctsm/test/test_sys_modify_singlept_site_neon.py | 2 ++ python/ctsm/test/test_sys_regrid_ggcmi_shdates.py | 2 ++ python/ctsm/test/test_unit_fsurdat_modifier.py | 2 ++ python/ctsm/test/test_unit_modify_singlept_site_neon.py | 2 ++ python/ctsm/test/test_unit_neon_arg_parse.py | 2 ++ python/ctsm/test/test_unit_neon_site.py | 2 ++ python/ctsm/test/test_unit_path_utils.py | 2 ++ python/ctsm/test/test_unit_run_neon.py | 2 ++ python/ctsm/test/test_unit_utils.py | 2 ++ python/ctsm/test/test_unit_utils_import_coord.py | 2 ++ 13 files changed, 26 insertions(+) diff --git a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py index 46b59f4e76..59610751bc 100755 --- a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py +++ b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py @@ -21,9 +21,11 @@ class TestJobLauncherNoBatch(unittest.TestCase): """Tests of job_launcher_no_batch""" def setUp(self): + self._previous_dir = os.getcwd() self._testdir = tempfile.mkdtemp() def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._testdir, ignore_errors=True) def assertFileContentsEqual(self, expected, filepath, msg=None): diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 1a5045c14d..1c8a8a0e04 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -38,6 +38,7 @@ def setUp(self): - modify_fsurdat.cfg - fsurdat_out.nc """ + self._previous_dir = os.getcwd() self._cfg_template_path = os.path.join( path_to_ctsm_root(), "tools/modify_input_files/modify_fsurdat_template.cfg" ) @@ -55,6 +56,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_no_files_given_fail(self): diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index d773749bf7..3d17b24427 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -26,6 +26,7 @@ class TestSysBuildCtsm(unittest.TestCase): """System tests for lilac_build_ctsm""" def setUp(self): + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() self.assertTrue(os.path.isdir(self._tempdir)) @@ -42,6 +43,7 @@ def setUp(self): self._ncarhost = None def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) if self._ncarhost is not None: os.environ["NCAR_HOST"] = self._ncarhost diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 74362be4cd..036e8f39a4 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -27,6 +27,7 @@ def setUp(self): Make /_tempdir for use by these tests. Check tempdir for history files """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") self._cfg_file_path = os.path.join( @@ -37,6 +38,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_site(self): diff --git a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py index 6c2e230481..e2b304070d 100755 --- a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py +++ b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py @@ -38,6 +38,7 @@ def setUp(self): self._testinputs_cc_path = testinputs_cc_path # Make /_tempdir for use by these tests. + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() # Obtain path for the directory being created in /_tempdir @@ -73,6 +74,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_regrid_ggcmi_shdates(self): diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 166924903b..25fcc19fa2 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -46,6 +46,7 @@ def setUp(self): testinputs_path, "surfdata_5x5_amazon_16pfts_Irrig_CMIP6_simyr2000_c171214.nc", ) + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() self._fsurdat_in = os.path.join( testinputs_path, @@ -80,6 +81,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_subgrid_and_idealized_fails(self): diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index 3a9d7d424c..d74d7d6237 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -43,12 +43,14 @@ def setUp(self): Make /_tempdir for use by these tests. Check tempdir for history files """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_get_neon(self): diff --git a/python/ctsm/test/test_unit_neon_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py index 7bae337709..68de8d3c77 100755 --- a/python/ctsm/test/test_unit_neon_arg_parse.py +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -34,12 +34,14 @@ def setUp(self): """ Make /_tempdir for use by these tests. """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_function(self): diff --git a/python/ctsm/test/test_unit_neon_site.py b/python/ctsm/test/test_unit_neon_site.py index 4828718272..cf10b7c132 100755 --- a/python/ctsm/test/test_unit_neon_site.py +++ b/python/ctsm/test/test_unit_neon_site.py @@ -33,12 +33,14 @@ def setUp(self): """ Make /_tempdir for use by these tests. """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_user_nl_transient(self): diff --git a/python/ctsm/test/test_unit_path_utils.py b/python/ctsm/test/test_unit_path_utils.py index 566c458acc..e4905b18b2 100755 --- a/python/ctsm/test/test_unit_path_utils.py +++ b/python/ctsm/test/test_unit_path_utils.py @@ -22,9 +22,11 @@ class TestPathUtils(unittest.TestCase): """Tests of path_utils""" def setUp(self): + self._previous_dir = os.getcwd() self._testdir = tempfile.mkdtemp() def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._testdir, ignore_errors=True) def _ctsm_path_in_cesm(self): diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index a35608e249..9a5bac457d 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -32,12 +32,14 @@ def setUp(self): """ Make /_tempdir for use by these tests. """ + self._previous_dir = os.getcwd() self._tempdir = tempfile.mkdtemp() def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_check_neon_listing(self): diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index a78928abcc..fe63e01883 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -21,9 +21,11 @@ class TestUtilsFillTemplateFile(unittest.TestCase): """Tests of utils: fill_template_file""" def setUp(self): + self._previous_dir = os.getcwd() self._testdir = tempfile.mkdtemp() def tearDown(self): + os.chdir(self._previous_dir ) shutil.rmtree(self._testdir, ignore_errors=True) def test_fillTemplateFile_basic(self): diff --git a/python/ctsm/test/test_unit_utils_import_coord.py b/python/ctsm/test/test_unit_utils_import_coord.py index 6e339a913f..6e62e72966 100755 --- a/python/ctsm/test/test_unit_utils_import_coord.py +++ b/python/ctsm/test/test_unit_utils_import_coord.py @@ -39,6 +39,7 @@ class TestUtilsImportCoord(unittest.TestCase): def setUp(self): """Setup for trying out the methods""" + self._previous_dir = os.getcwd() testinputs_path = os.path.join(path_to_ctsm_root(), "python/ctsm/test/testinputs") self._testinputs_path = testinputs_path self._tempdir = tempfile.mkdtemp() @@ -55,6 +56,7 @@ def tearDown(self): """ Remove temporary directory """ + os.chdir(self._previous_dir ) shutil.rmtree(self._tempdir, ignore_errors=True) def test_importcoord1d(self): From 78d05967c2b027dc9776a884716597db6ef7f57c Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Mar 2024 00:32:20 -0700 Subject: [PATCH 08/10] Black update --- python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py | 2 +- python/ctsm/test/test_sys_fsurdat_modifier.py | 2 +- python/ctsm/test/test_sys_lilac_build_ctsm.py | 2 +- python/ctsm/test/test_sys_modify_singlept_site_neon.py | 2 +- python/ctsm/test/test_sys_regrid_ggcmi_shdates.py | 2 +- python/ctsm/test/test_unit_fsurdat_modifier.py | 2 +- python/ctsm/test/test_unit_modify_singlept_site_neon.py | 2 +- python/ctsm/test/test_unit_neon_arg_parse.py | 2 +- python/ctsm/test/test_unit_neon_site.py | 2 +- python/ctsm/test/test_unit_path_utils.py | 2 +- python/ctsm/test/test_unit_run_neon.py | 2 +- python/ctsm/test/test_unit_utils.py | 2 +- python/ctsm/test/test_unit_utils_import_coord.py | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py index 59610751bc..4e22e3c085 100755 --- a/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py +++ b/python/ctsm/test/joblauncher/test_unit_job_launcher_no_batch.py @@ -25,7 +25,7 @@ def setUp(self): self._testdir = tempfile.mkdtemp() def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._testdir, ignore_errors=True) def assertFileContentsEqual(self, expected, filepath, msg=None): diff --git a/python/ctsm/test/test_sys_fsurdat_modifier.py b/python/ctsm/test/test_sys_fsurdat_modifier.py index 1c8a8a0e04..0e11a204a8 100755 --- a/python/ctsm/test/test_sys_fsurdat_modifier.py +++ b/python/ctsm/test/test_sys_fsurdat_modifier.py @@ -56,7 +56,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_no_files_given_fail(self): diff --git a/python/ctsm/test/test_sys_lilac_build_ctsm.py b/python/ctsm/test/test_sys_lilac_build_ctsm.py index 3d17b24427..a8af9b2eab 100755 --- a/python/ctsm/test/test_sys_lilac_build_ctsm.py +++ b/python/ctsm/test/test_sys_lilac_build_ctsm.py @@ -43,7 +43,7 @@ def setUp(self): self._ncarhost = None def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) if self._ncarhost is not None: os.environ["NCAR_HOST"] = self._ncarhost diff --git a/python/ctsm/test/test_sys_modify_singlept_site_neon.py b/python/ctsm/test/test_sys_modify_singlept_site_neon.py index 036e8f39a4..76a78c3db5 100755 --- a/python/ctsm/test/test_sys_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_sys_modify_singlept_site_neon.py @@ -38,7 +38,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_site(self): diff --git a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py index e2b304070d..9209ebfe78 100755 --- a/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py +++ b/python/ctsm/test/test_sys_regrid_ggcmi_shdates.py @@ -74,7 +74,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_regrid_ggcmi_shdates(self): diff --git a/python/ctsm/test/test_unit_fsurdat_modifier.py b/python/ctsm/test/test_unit_fsurdat_modifier.py index 25fcc19fa2..bcb9099f61 100755 --- a/python/ctsm/test/test_unit_fsurdat_modifier.py +++ b/python/ctsm/test/test_unit_fsurdat_modifier.py @@ -81,7 +81,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_subgrid_and_idealized_fails(self): diff --git a/python/ctsm/test/test_unit_modify_singlept_site_neon.py b/python/ctsm/test/test_unit_modify_singlept_site_neon.py index d74d7d6237..db1fc1966d 100755 --- a/python/ctsm/test/test_unit_modify_singlept_site_neon.py +++ b/python/ctsm/test/test_unit_modify_singlept_site_neon.py @@ -50,7 +50,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_get_neon(self): diff --git a/python/ctsm/test/test_unit_neon_arg_parse.py b/python/ctsm/test/test_unit_neon_arg_parse.py index 68de8d3c77..4a5b0b9e6c 100755 --- a/python/ctsm/test/test_unit_neon_arg_parse.py +++ b/python/ctsm/test/test_unit_neon_arg_parse.py @@ -41,7 +41,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_function(self): diff --git a/python/ctsm/test/test_unit_neon_site.py b/python/ctsm/test/test_unit_neon_site.py index cf10b7c132..8ef6034f94 100755 --- a/python/ctsm/test/test_unit_neon_site.py +++ b/python/ctsm/test/test_unit_neon_site.py @@ -40,7 +40,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_modify_user_nl_transient(self): diff --git a/python/ctsm/test/test_unit_path_utils.py b/python/ctsm/test/test_unit_path_utils.py index e4905b18b2..067809cbf6 100755 --- a/python/ctsm/test/test_unit_path_utils.py +++ b/python/ctsm/test/test_unit_path_utils.py @@ -26,7 +26,7 @@ def setUp(self): self._testdir = tempfile.mkdtemp() def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._testdir, ignore_errors=True) def _ctsm_path_in_cesm(self): diff --git a/python/ctsm/test/test_unit_run_neon.py b/python/ctsm/test/test_unit_run_neon.py index 9a5bac457d..904db885a9 100755 --- a/python/ctsm/test/test_unit_run_neon.py +++ b/python/ctsm/test/test_unit_run_neon.py @@ -39,7 +39,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_check_neon_listing(self): diff --git a/python/ctsm/test/test_unit_utils.py b/python/ctsm/test/test_unit_utils.py index fe63e01883..85ba2515dd 100755 --- a/python/ctsm/test/test_unit_utils.py +++ b/python/ctsm/test/test_unit_utils.py @@ -25,7 +25,7 @@ def setUp(self): self._testdir = tempfile.mkdtemp() def tearDown(self): - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._testdir, ignore_errors=True) def test_fillTemplateFile_basic(self): diff --git a/python/ctsm/test/test_unit_utils_import_coord.py b/python/ctsm/test/test_unit_utils_import_coord.py index 6e62e72966..11ff3c1b54 100755 --- a/python/ctsm/test/test_unit_utils_import_coord.py +++ b/python/ctsm/test/test_unit_utils_import_coord.py @@ -56,7 +56,7 @@ def tearDown(self): """ Remove temporary directory """ - os.chdir(self._previous_dir ) + os.chdir(self._previous_dir) shutil.rmtree(self._tempdir, ignore_errors=True) def test_importcoord1d(self): From 72d2944c783992be941e06684aa41a46852baeee Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Thu, 7 Mar 2024 00:33:28 -0700 Subject: [PATCH 09/10] Add black commit to git-blame file --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 8708f8e0c2..a5783fdb9f 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -30,3 +30,4 @@ d866510188d26d51bcd6d37239283db690af7e82 8a168bb0895f4f2421608dd2589398e13a6663e6 183fc26a6691bbdf87f515dc47924a64be3ced9b 6fccf682eaf718615407d9bacdd3903b8786a03d +78d05967c2b027dc9776a884716597db6ef7f57c From 5150c501b509dbcd690fd0116cb50af81a5d60c6 Mon Sep 17 00:00:00 2001 From: Erik Kluzek Date: Tue, 19 Mar 2024 14:36:17 -0600 Subject: [PATCH 10/10] Don't run stest in the python Makefile, add a note to run it by hand, fix the conflict that ended up being committed, and make sure abort is added --- python/Makefile | 13 +- python/ctsm/site_and_regional/neon_site.py | 371 -------------------- python/ctsm/site_and_regional/tower_site.py | 1 + 3 files changed, 13 insertions(+), 372 deletions(-) diff --git a/python/Makefile b/python/Makefile index b43e1c5e53..9645242111 100644 --- a/python/Makefile +++ b/python/Makefile @@ -28,6 +28,10 @@ PYLINT_SRC = \ # ../cime_config/buildnml all: test black lint +# ---------------------------------------------------------------- +# See the stest target about this issue + @echo "Run './run_ctsm_py_tests --sys' by hand afterwards" +# ---------------------------------------------------------------- @echo @echo @echo "Successfully ran all standard tests" @@ -40,7 +44,14 @@ utest: FORCE .PHONY: stest stest: FORCE - $(PYTHON) ./run_ctsm_py_tests $(TEST_ARGS) --sys +# ---------------------------------------------------------------- +# EBK 2024-03-19: Comment out running here because of this issue: +# https://github.com/ESCOMP/CTSM/pull/2363#issuecomment-1967884908 +#$(PYTHON) ./run_ctsm_py_tests $(TEST_ARGS) --sys +# Instead run by hand which seems to be working for now... +# ---------------------------------------------------------------- + @echo "System tests currently don't run under Make so..." + @echo "Run './run_ctsm_py_tests --sys' by hand afterwards" .PHONY: lint lint: FORCE diff --git a/python/ctsm/site_and_regional/neon_site.py b/python/ctsm/site_and_regional/neon_site.py index ca77faa32c..4af8e66fdd 100755 --- a/python/ctsm/site_and_regional/neon_site.py +++ b/python/ctsm/site_and_regional/neon_site.py @@ -107,377 +107,6 @@ def modify_user_nl(self, case_root, run_type, rundir, site_lines=None): # TODO: include neon-specific user namelist lines, using this as just an example currently if site_lines is None: site_lines = [ -||||||| dac9bad4a - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - # pylint: disable=undefined-variable - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - # pylint: enable=undefined-variable - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - - def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", -======= - print("using this version:", version) - - if experiment is not None: - self.name = self.name + "." + experiment - case_root = os.path.abspath(os.path.join(base_case_root, "..", self.name + "." + run_type)) - - rundir = None - if os.path.isdir(case_root): - if overwrite: - print("---- removing the existing case -------") - if os.getcwd() == case_root: - abort("Trying to remove the directory tree that we are in") - shutil.rmtree(case_root) - elif rerun: - with Case(case_root, read_only=False) as case: - rundir = case.get_value("RUNDIR") - # For existing case check that the compset name is correct - existingcompname = case.get_value("COMPSET") - match = re.search("^HIST", existingcompname, flags=re.IGNORECASE) - # pylint: disable=undefined-variable - if re.search("^HIST", compset, flags=re.IGNORECASE) is None: - expect( - match is None, - """Existing base case is a historical type and should not be - --rerun with the --overwrite option""", - ) - # pylint: enable=undefined-variable - else: - expect( - match is not None, - """Existing base case should be a historical type and is not - --rerun with the --overwrite option""", - ) - if os.path.isfile(os.path.join(rundir, "ESMF_Profile.summary")): - print("Case {} appears to be complete, not rerunning.".format(case_root)) - elif not setup_only: - print("Resubmitting case {}".format(case_root)) - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - return - else: - logger.warning("Case already exists in %s, not overwritting", case_root) - return - - if run_type == "postad": - adcase_root = case_root.replace(".postad", ".ad") - if not os.path.isdir(adcase_root): - logger.warning("postad requested but no ad case found in %s", adcase_root) - return - - if not os.path.isdir(case_root): - # read_only = False should not be required here - with Case(base_case_root, read_only=False) as basecase: - print("---- cloning the base case in {}".format(case_root)) - # - # EBK: 11/05/2022 -- Note keeping the user_mods_dirs argument is important. Although - # it causes some of the user_nl_* files to have duplicated inputs. It also ensures - # that the shell_commands file is copied, as well as taking care of the DATM inputs. - # See https://github.com/ESCOMP/CTSM/pull/1872#pullrequestreview-1169407493 - # - basecase.create_clone(case_root, keepexe=True, user_mods_dirs=user_mods_dirs) - - with Case(case_root, read_only=False) as case: - if run_type != "transient": - # in order to avoid the complication of leap years, - # we always set the run_length in units of days. - case.set_value("STOP_OPTION", "ndays") - case.set_value("REST_OPTION", "end") - case.set_value("CONTINUE_RUN", False) - case.set_value("NEONVERSION", version) - if prism: - case.set_value("CLM_USRDAT_NAME", "NEON.PRISM") - - if run_type == "ad": - case.set_value("CLM_FORCE_COLDSTART", "on") - case.set_value("CLM_ACCELERATED_SPINUP", "on") - case.set_value("RUN_REFDATE", "0018-01-01") - case.set_value("RUN_STARTDATE", "0018-01-01") - case.set_value("RESUBMIT", 1) - case.set_value("STOP_N", run_length) - - else: - case.set_value("CLM_FORCE_COLDSTART", "off") - case.set_value("CLM_ACCELERATED_SPINUP", "off") - case.set_value("RUN_TYPE", "hybrid") - - if run_type == "postad": - self.set_ref_case(case) - case.set_value("STOP_N", run_length) - - # For transient cases STOP will be set in the user_mod_directory - if run_type == "transient": - if self.finidat: - case.set_value("RUN_TYPE", "startup") - else: - if not self.set_ref_case(case): - return - case.set_value("CALENDAR", "GREGORIAN") - case.set_value("RESUBMIT", 0) - case.set_value("STOP_OPTION", "nmonths") - - if not rundir: - rundir = case.get_value("RUNDIR") - - self.modify_user_nl(case_root, run_type, rundir) - - case.create_namelists() - # explicitly run check_input_data - case.check_all_input_data() - if not setup_only: - case.submit(no_batch=no_batch) - print("-----------------------------------") - print("Successfully submitted case!") - batch_query = self.get_batch_query(case) - if batch_query != "none": - print(f"Use {batch_query} to check its run status") - - def set_ref_case(self, case): - """ - Set an existing case as the reference case, eg for use with spinup. - """ - rundir = case.get_value("RUNDIR") - case_root = case.get_value("CASEROOT") - if case_root.endswith(".postad"): - ref_case_root = case_root.replace(".postad", ".ad") - root = ".ad" - else: - ref_case_root = case_root.replace(".transient", ".postad") - root = ".postad" - if not os.path.isdir(ref_case_root): - logger.warning( - "ERROR: spinup must be completed first, could not find directory %s", ref_case_root - ) - return False - - with Case(ref_case_root) as refcase: - refrundir = refcase.get_value("RUNDIR") - case.set_value("RUN_REFDIR", refrundir) - case.set_value("RUN_REFCASE", os.path.basename(ref_case_root)) - refdate = None - for reffile in glob.iglob(refrundir + "/{}{}.clm2.r.*.nc".format(self.name, root)): - m_searched = re.search(r"(\d\d\d\d-\d\d-\d\d)-\d\d\d\d\d.nc", reffile) - if m_searched: - refdate = m_searched.group(1) - symlink_force(reffile, os.path.join(rundir, os.path.basename(reffile))) - logger.info("Found refdate of %s", refdate) - if not refdate: - logger.warning("Could not find refcase for %s", case_root) - return False - - for rpfile in glob.iglob(refrundir + "/rpointer*"): - safe_copy(rpfile, rundir) - if not os.path.isdir(os.path.join(rundir, "inputdata")) and os.path.isdir( - os.path.join(refrundir, "inputdata") - ): - symlink_force(os.path.join(refrundir, "inputdata"), os.path.join(rundir, "inputdata")) - - case.set_value("RUN_REFDATE", refdate) - if case_root.endswith(".postad"): - case.set_value("RUN_STARTDATE", refdate) - # NOTE: if start options are set, RUN_STARTDATE should be modified here - return True - - def modify_user_nl(self, case_root, run_type, rundir): - """ - Modify user namelist. If transient, include finidat in user_nl; - Otherwise, adjust user_nl to include different mfilt, nhtfrq, and variables in hist_fincl1. - """ - user_nl_fname = os.path.join(case_root, "user_nl_clm") - user_nl_lines = None - if run_type == "transient": - if self.finidat: - user_nl_lines = [ - "finidat = '{}/inputdata/lnd/ctsm/initdata/{}'".format(rundir, self.finidat) - ] - else: - user_nl_lines = [ - "hist_fincl2 = ''", - "hist_mfilt = 20", - "hist_nhtfrq = -8760", - "hist_empty_htapes = .true.", ->>>>>>> escomp/b4b-dev """hist_fincl1 = 'TOTECOSYSC', 'TOTECOSYSN', 'TOTSOMC', 'TOTSOMN', 'TOTVEGC', 'TOTVEGN', 'TLAI', 'GPP', 'CPOOL', 'NPP', 'TWS', 'H2OSNO',""" ] diff --git a/python/ctsm/site_and_regional/tower_site.py b/python/ctsm/site_and_regional/tower_site.py index d92df0fec1..1679df83e9 100644 --- a/python/ctsm/site_and_regional/tower_site.py +++ b/python/ctsm/site_and_regional/tower_site.py @@ -21,6 +21,7 @@ # pylint: disable=wrong-import-position, import-error, unused-import, wrong-import-order from ctsm import add_cime_to_path from ctsm.path_utils import path_to_ctsm_root +from ctsm.utils import abort from CIME import build from CIME.case import Case