Merge branch 'master' into update-mvk-config

.github/workflows/testing.yml

@@ -113,14 +113,13 @@ jobs:
           export CIME_REMOTE=https://github.com/${{ github.event.pull_request.head.repo.full_name || github.repository }}
           export CIME_BRANCH=${GITHUB_HEAD_REF:-${GITHUB_REF##*/}}
 
-          mamba install -y python=${{ matrix.python-version }}
-
           pip install -r test-requirements.txt
 
-          export DEBUG=true
-
           source /entrypoint.sh
 
+          # from 'entrypoint.sh', create and activate new environment
+          create_environment ${{ matrix.python-version }}
+
           # GitHub runner home is different than container
           cp -rf /root/.cime /github/home/
 

CIME/BuildTools/configure.py

@@ -28,7 +28,7 @@
 from CIME.XML.files import Files
 from CIME.build import CmakeTmpBuildDir
 
-import shutil
+import shutil, glob
 
 logger = logging.getLogger(__name__)
 
@@ -72,6 +72,12 @@ def configure(
         output_cmake_macros_dir = os.path.join(output_dir, "cmake_macros")
         if not os.path.exists(output_cmake_macros_dir):
             shutil.copytree(new_cmake_macros_dir, output_cmake_macros_dir)
+            ccs_mach_dir = os.path.join(
+                new_cmake_macros_dir, "..", machobj.get_machine_name()
+            )
+            for f in glob.iglob(os.path.join(ccs_mach_dir, "*.cmake")):
+                print(f"copying {f} to {output_dir}")
+                safe_copy(f, output_dir)
 
         copy_local_macros_to_dir(
             output_cmake_macros_dir, extra_machdir=extra_machines_dir

CIME/ParamGen/paramgen.py

@@ -279,7 +279,7 @@ def _expand_vars(expr, expand_func):
         return expr
 
     @staticmethod
-    def _is_guarded_dict(data_dict):
+    def is_guarded_dict(data_dict):
         """Returns true if all the keys of a dictionary are logical expressions, i.e., guards.
 
         Parameters
@@ -289,11 +289,11 @@ def _is_guarded_dict(data_dict):
 
         Example
         -------
-        >>> ParamGen._is_guarded_dict({True: 'x', 'False': 'y'})
+        >>> ParamGen.is_guarded_dict({True: 'x', 'False': 'y'})
         True
-        >>> ParamGen._is_guarded_dict({ "'tx0.66v1' == 'tx0.66v1'": 'x', False: 'y'})
+        >>> ParamGen.is_guarded_dict({ "'tx0.66v1' == 'tx0.66v1'": 'x', False: 'y'})
         True
-        >>> ParamGen._is_guarded_dict({'i':'x', 'j':'y'})
+        >>> ParamGen.is_guarded_dict({'i':'x', 'j':'y'})
         False
         """
         if not isinstance(data_dict, dict):
@@ -351,7 +351,7 @@ def _eval_guard(guard):
             )
             return guard_evaluated
 
-        if not ParamGen._is_guarded_dict(data_dict):
+        if not ParamGen.is_guarded_dict(data_dict):
             return data_dict
 
         guards_eval_true = []  # list of guards that evaluate to true.
@@ -370,64 +370,36 @@ def _eval_guard(guard):
             return data_dict[guards_eval_true[-1]]
         raise RuntimeError("Unknown match option.")
 
-    def _reduce_recursive(self, data_dict, expand_func=None):
-
+    def _reduce_recursive(self, data, expand_func=None):
         """A recursive method to reduce a given data_dict. This method is intended to be called by the reduce method
         only. Check the docstring of the reduce method for more information."""
 
-        # (1) Expand variables in keys, .e.g, "$OCN_GRID" to "gx1v7":
-        def _expand_vars_in_keys(data_dict):
-            if expand_func is not None:
-                new_data_dict = {}
-                for key in data_dict:
-                    new_key = key
-                    if has_unexpanded_var(key):
-                        new_key = ParamGen._expand_vars(key, expand_func)
-                    new_data_dict[new_key] = data_dict[key]
-                return new_data_dict
-            return data_dict
-
-        data_dict = _expand_vars_in_keys(data_dict)
-
-        # (2) Evaluate the keys if they are all logical expressions, i.e., guards.
-        # Pick the value of the first or last key evaluating to True and drop everything else.
-        while ParamGen._is_guarded_dict(data_dict):
-            data_dict = self._impose_guards(data_dict)
-            if isinstance(data_dict, dict):
-                data_dict = _expand_vars_in_keys(data_dict)
+        if isinstance(data, dict):
 
-        # If the data_dict is reduced to a string, expand vars and eval formulas as a last step.
-        if isinstance(data_dict, str):
-
-            # (3) Expand variables in values, .e.g, "$OCN_GRID" to "gx1v7":
-            data_dict = ParamGen._expand_vars(data_dict, expand_func)
-
-            # (4) Evaluate the formulas as values, e.g., "= 3+4", "= [i for i in range(5)]", etc.
-            if is_formula(data_dict):
-                data_dict = eval_formula(data_dict.strip()[1:])
+            # (1) Expand vars in keys
+            if expand_func is not None:
+                data = {
+                    ParamGen._expand_vars(key, expand_func): data[key] for key in data
+                }
 
-        # Continue reducing the data if it is still a dictionary (nested or not).
-        elif isinstance(data_dict, dict):
+            # (2) Evaulate guards (if applicable)
+            if ParamGen.is_guarded_dict(data):
+                data = self._reduce_recursive(self._impose_guards(data), expand_func)
 
-            # (3) Expand variables in values, .e.g, "$OCN_GRID" to "gx1v7":
-            for key, val in data_dict.copy().items():
-                if isinstance(val, str):
-                    data_dict[key] = ParamGen._expand_vars(val, expand_func)
+            # (3) Call _reduce_recursive for all branches of dict
+            else:
+                for key in data:
+                    data[key] = self._reduce_recursive(data[key], expand_func)
 
-            # (4) Evaluate the formulas as values, e.g., "= 3+4", "= [i for i in range(5)]", etc.
-            for key, val in data_dict.copy().items():
-                if is_formula(val):
-                    data_dict[key] = eval_formula(val.strip()[1:])
+        else:  # data is not a dict, and so is a value.
 
-            # (5) Recursively call _reduce_recursive for the remaining nested dicts before returning
-            keys_of_nested_dicts = []  # i.e., keys of values that are of type dict
-            for key, val in data_dict.items():
-                if isinstance(val, dict):
-                    keys_of_nested_dicts.append(key)
-            for key in keys_of_nested_dicts:
-                data_dict[key] = self._reduce_recursive(data_dict[key], expand_func)
+            # (4) Finally, process values by expanding vars and applying formulas
+            if isinstance(data, str):
+                data = ParamGen._expand_vars(data, expand_func)
+            if is_formula(data):
+                data = eval_formula(data.strip()[1:])
 
-        return data_dict
+        return data
 
     def reduce(self, expand_func=None):
         """

CIME/Servers/__init__.py

@@ -1,9 +1,9 @@
 # pylint: disable=import-error
-from distutils.spawn import find_executable
+from shutil import which
 
-has_gftp = find_executable("globus-url-copy")
-has_svn = find_executable("svn")
-has_wget = find_executable("wget")
+has_gftp = which("globus-url-copy")
+has_svn = which("svn")
+has_wget = which("wget")
 has_ftp = True
 try:
     from ftplib import FTP

CIME/SystemTests/ers.py

@@ -16,29 +16,18 @@ def __init__(self, case, **kwargs):
         SystemTestsCommon.__init__(self, case, **kwargs)
 
     def _ers_first_phase(self):
-        stop_n = self._case.get_value("STOP_N")
-        stop_option = self._case.get_value("STOP_OPTION")
-        rest_n = self._case.get_value("REST_N")
-        expect(stop_n > 0, "Bad STOP_N: {:d}".format(stop_n))
-
-        expect(stop_n > 2, "ERROR: stop_n value {:d} too short".format(stop_n))
-        logger.info(
-            "doing an {0} {1} initial test with restart file at {2} {1}".format(
-                str(stop_n), stop_option, str(rest_n)
-            )
-        )
+        self._rest_n = self._set_restart_interval()
         self.run_indv()
 
     def _ers_second_phase(self):
         stop_n = self._case.get_value("STOP_N")
         stop_option = self._case.get_value("STOP_OPTION")
 
-        rest_n = int(stop_n / 2 + 1)
-        stop_new = stop_n - rest_n
+        stop_new = stop_n - self._rest_n
         expect(
             stop_new > 0,
             "ERROR: stop_n value {:d} too short {:d} {:d}".format(
-                stop_new, stop_n, rest_n
+                stop_new, stop_n, self._rest_n
             ),
         )
         rundir = self._case.get_value("RUNDIR")

CIME/SystemTests/funit.py

@@ -5,7 +5,8 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.build import post_build
-from CIME.utils import append_testlog, get_cime_root
+from CIME.status import append_testlog
+from CIME.utils import get_cime_root
 from CIME.test_status import *
 
 logger = logging.getLogger(__name__)

CIME/SystemTests/hommebaseclass.py

@@ -4,11 +4,11 @@
 from CIME.XML.standard_module_setup import *
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.build import post_build
-from CIME.utils import append_testlog, SharedArea
+from CIME.status import append_testlog
+from CIME.utils import SharedArea
 from CIME.test_status import *
 
 import shutil
-from distutils import dir_util
 
 logger = logging.getLogger(__name__)
 
@@ -97,10 +97,9 @@ def run_phase(self):
                     shutil.rmtree(full_baseline_dir)
 
                 with SharedArea():
-                    dir_util.copy_tree(
+                    shutil.copytree(
                         os.path.join(exeroot, "tests", "baseline"),
                         full_baseline_dir,
-                        preserve_mode=False,
                     )
 
         elif compare:

CIME/SystemTests/mvk.py

@@ -9,7 +9,7 @@
 import os
 import json
 import logging
-from distutils import dir_util
+from shutils import copytree
 
 from CIME import test_status
 from CIME import utils
@@ -272,7 +272,7 @@ def update_testlog(self, test_name, case_name, evv_out_dir):
             urlroot = utils.get_urlroot(mach_obj)
 
             with utils.SharedArea():
-                dir_util.copy_tree(
+                copytree(
                     evv_out_dir,
                     os.path.join(htmlroot, "evv", case_name),
                     preserve_mode=False,

CIME/SystemTests/pgn.py

@@ -16,14 +16,15 @@
 import logging
 
 from collections import OrderedDict
-from distutils import dir_util
+from shutils import copytree
 
 import pandas as pd
 import numpy as np
 
 
 import CIME.test_status
 import CIME.utils
+from CIME.status import append_testlog
 from CIME.SystemTests.system_tests_common import SystemTestsCommon
 from CIME.case.case_setup import case_setup
 from CIME.XML.machines import Machines
@@ -80,7 +81,7 @@ def build_phase(self, sharedlib_only=False, model_only=False):
             if not model_only:
                 # Lay all of the components out concurrently
                 logger.debug(
-                    "PGN_INFO: Updating NINST for multi-instance in " "env_mach_pes.xml"
+                    "PGN_INFO: Updating NINST for multi-instance in env_mach_pes.xml"
                 )
                 for comp in ["ATM", "OCN", "WAV", "GLC", "ICE", "ROF", "LND"]:
                     ntasks = self._case.get_value("NTASKS_{}".format(comp))
@@ -224,10 +225,9 @@ def _compare_baseline(self):
             urlroot = CIME.utils.get_urlroot(mach_obj)
             if htmlroot is not None:
                 with CIME.utils.SharedArea():
-                    dir_util.copy_tree(
+                    copytree(
                         evv_out_dir,
                         os.path.join(htmlroot, "evv", case_name),
-                        preserve_mode=False,
                     )
                 if urlroot is None:
                     urlroot = "[{}_URL]".format(mach_name.capitalize())
@@ -253,7 +253,7 @@ def _compare_baseline(self):
                 )
             )
 
-            CIME.utils.append_testlog(comments, self._orig_caseroot)
+            append_testlog(comments, self._orig_caseroot)
 
     def run_phase(self):
         logger.debug("PGN_INFO: RUN PHASE")

CIME/SystemTests/restart_tests.py

@@ -1,6 +1,5 @@
 """
 Abstract class for restart tests
-
 """
 
 from CIME.SystemTests.system_tests_compare_two import SystemTestsCompareTwo
@@ -34,6 +33,7 @@ def __init__(
     def _case_one_setup(self):
         stop_n = self._case1.get_value("STOP_N")
         expect(stop_n >= 3, "STOP_N must be at least 3, STOP_N = {}".format(stop_n))
+        self._set_restart_interval()
 
     def _case_two_setup(self):
         rest_n = self._case1.get_value("REST_N")