Merge branch 'NOAA-GFDL:main' into update-ci-for-compilation

fre/make/tests/test_create_makefile.py

@@ -0,0 +1,72 @@
+"""
+Test fre make create-makefile
+"""
+import os
+import shutil
+from pathlib import Path
+from fre.make import createMakefile
+
+# SET-UP
+test_dir = Path("fre/make/tests")
+NM_EXAMPLE = Path("null_example")
+YAMLFILE = "null_model.yaml"
+BM_PLATFORM = ["ncrc5.intel23"]
+CONTAINER_PLATFORM = ["hpcme.2023"]
+TARGET = ["debug"]
+EXPERIMENT = "null_model_full"
+
+# Create output location
+out = f"{test_dir}/makefile_out"
+if Path(out).exists():
+    # remove
+    shutil.rmtree(out)
+    # create output directory
+    Path(out).mkdir(parents=True,exist_ok=True)
+else:
+    Path(out).mkdir(parents=True,exist_ok=True)
+
+# Set output directory as home for fre make output
+#os.environ["HOME"]=str(Path(out))
+
+def test_modelyaml_exists():
+    """
+    Check the model yaml exists
+    """
+    assert Path(f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}").exists()
+
+def test_compileyaml_exists():
+    """
+    Check the compile yaml exists
+    """
+    assert Path(f"{test_dir}/{NM_EXAMPLE}/compile.yaml").exists()
+
+def test_platformyaml_exists():
+    """
+    Check the platform yaml exists
+    """
+    assert Path(f"{test_dir}/{NM_EXAMPLE}/platforms.yaml").exists()
+
+def test_bm_makefile_creation():
+    """
+    Check the makefile is created when a bare-metal platform is used
+    """
+    # Set output directory as home for fre make output
+    os.environ["HOME"]=str(Path(out))
+
+    bm_plat = BM_PLATFORM[0]
+    targ = TARGET[0]
+    yamlfile_path = f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}"
+
+    createMakefile.makefile_create(yamlfile_path,BM_PLATFORM,TARGET)
+
+    assert Path(f"{out}/fremake_canopy/test/{EXPERIMENT}/{bm_plat}-{targ}/exec/Makefile").exists()
+
+def test_container_makefile_creation():
+    """
+    Check the makefile is created when the container platform is used
+    """
+    container_plat = CONTAINER_PLATFORM[0]
+    yamlfile_path = f"{test_dir}/{NM_EXAMPLE}/{YAMLFILE}"
+    createMakefile.makefile_create(yamlfile_path,CONTAINER_PLATFORM,TARGET)
+
+    assert Path(f"tmp/{container_plat}/Makefile").exists()

fre/pp/configure_script_yaml.py

@@ -147,7 +147,7 @@ def set_rose_apps(yamlfile,rose_regrid,rose_remap):
                             value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}')
 
 ####################
-def _yamlInfo(yamlfile,experiment,platform,target):
+def yamlInfo(yamlfile,experiment,platform,target):
     """
     Using a valid pp.yaml, the rose-app and rose-suite
     configuration files are created in the cylc-src
@@ -200,12 +200,12 @@ def _yamlInfo(yamlfile,experiment,platform,target):
     print("  " + outfile)
 
 @click.command()
-def yamlInfo(yamlfile,experiment,platform,target):
+def _yamlInfo(yamlfile,experiment,platform,target):
     '''
     Wrapper script for calling yamlInfo - allows the decorated version
     of the function to be separate from the undecorated version
     '''
-    return _yamlInfo(yamlfile,experiment,platform,target)
+    return yamlInfo(yamlfile,experiment,platform,target)
 
 # Use parseyaml function to parse created edits.yaml
 if __name__ == '__main__':

fre/pp/tests/test_configure_script_yaml.py

@@ -1,49 +1,46 @@
+"""
+Test configure_script_yaml
+"""
 import os
 from pathlib import Path
 from fre.pp import configure_script_yaml as csy
 
 # Set what would be click options
-experiment = "c96L65_am5f7b12r1_amip"
-platform = "gfdl.ncrc5-intel22-classic"
-target = "prod-openmp"
+EXPERIMENT = "c96L65_am5f7b12r1_amip"
+PLATFORM = "gfdl.ncrc5-intel22-classic"
+TARGET = "prod-openmp"
 
 # Set example yaml paths, input directory
-CWD = Path.cwd()
 test_dir = Path("fre/pp/tests")
-test_yaml = Path(f"AM5_example/am5.yaml")
-
-# Set home for ~/cylc-src location in script
-os.environ["HOME"]=str(Path(f"{CWD}/{test_dir}/configure_yaml_out"))
+test_yaml = Path("AM5_example/am5.yaml")
 
 def test_combinedyaml_exists():
     """
     Make sure combined yaml file exists
     """
-    assert Path(f"{CWD}/{test_dir}/{test_yaml}").exists()
+    assert Path(f"{test_dir}/{test_yaml}").exists()
 
 def test_configure_script():
     """ 
     Tests success of confgure yaml script
     Creates rose-suite, regrid rose-app, remap rose-app
     TO-DO: will break this up for better tests
     """
-    os.chdir(f"{CWD}/{test_dir}/AM5_example")
+    # Set home for ~/cylc-src location in script
+    os.environ["HOME"]=str(Path(f"{test_dir}/configure_yaml_out"))
 
     # Set output directory
-    out_dir = Path(f"{os.getenv('HOME')}/cylc-src/{experiment}__{platform}__{target}")
+    out_dir = Path(f"{os.getenv('HOME')}/cylc-src/{EXPERIMENT}__{PLATFORM}__{TARGET}")
     Path(out_dir).mkdir(parents=True,exist_ok=True)
 
     # Define combined yaml
-    model_yaml = str(Path(f"{CWD}/{test_dir}/{test_yaml}"))
+    model_yaml = str(Path(f"{test_dir}/{test_yaml}"))
 
     # Invoke configure_yaml_script.py
-    csy._yamlInfo(model_yaml,experiment,platform,target)
+    csy.yamlInfo(model_yaml,EXPERIMENT,PLATFORM,TARGET)
 
     # Check for configuration creation and final combined yaml
-    assert all([Path(f"{out_dir}/{experiment}.yaml").exists(),
+    assert all([Path(f"{out_dir}/{EXPERIMENT}.yaml").exists(),
                 Path(f"{out_dir}/rose-suite.conf").exists(),
                 Path(f"{out_dir}/app/regrid-xy/rose-app.conf").exists(),
                 Path(f"{out_dir}/app/remap-pp-components/rose-app.conf").exists()])
-
-    # Go back to original directory
-    os.chdir(CWD)

fre/yamltools/combine_yamls.py

@@ -90,7 +90,7 @@ def experiment_check(mainyaml_dir,comb,experiment):
             if expyaml is not None:
                 ey_path=[]
                 for e in expyaml:
-                    if Path(e).exists():
+                    if Path(os.path.join(mainyaml_dir,e)).exists():
                         ey=Path(os.path.join(mainyaml_dir,e))
                         ey_path.append(ey)
                     else:
@@ -115,7 +115,7 @@ def experiment_check(mainyaml_dir,comb,experiment):
 class init_compile_yaml():
   def __init__(self,yamlfile,platform,target):
     """
-    Process to combine yamls appllicable to compilation
+    Process to combine yamls applicable to compilation
     """
     self.yml = yamlfile
     self.name = yamlfile.split(".")[0]

fre/yamltools/tests/test_combine_yamls.py

@@ -13,13 +13,13 @@
 
 ## SET-UP
 # Set example yaml paths, input directory, output directory
-CWD = Path.cwd()
+#CWD = Path.cwd()
 TEST_DIR = Path("fre/yamltools/tests")
-IN_DIR = Path(f"{CWD}/{TEST_DIR}/AM5_example")
+IN_DIR = Path(f"{TEST_DIR}/AM5_example")
 
 # Create output directories
-COMP_OUT_DIR = Path(f"{CWD}/{TEST_DIR}/combine_yamls_out/compile")
-PP_OUT_DIR = Path(f"{CWD}/{TEST_DIR}/combine_yamls_out/pp")
+COMP_OUT_DIR = Path(f"{TEST_DIR}/combine_yamls_out/compile")
+PP_OUT_DIR = Path(f"{TEST_DIR}/combine_yamls_out/pp")
 
 # If output directory exists, remove and create again
 for out in [COMP_OUT_DIR, PP_OUT_DIR]:
@@ -63,25 +63,19 @@ def test_merged_compile_yamls():
     Check for the creation of the combined-[experiment] yaml
     Check that the model yaml was merged into the combined yaml
     """
-    # Go into the input directory
-    os.chdir(IN_DIR)
-
     # Model yaml path
-    modelyaml = "am5.yaml"
+    modelyaml = str(Path(f"{IN_DIR}/am5.yaml"))
     use = "compile"
 
     # Merge the yamls
     cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use)
 
     # Move combined yaml to output location
-    shutil.move("combined-am5.yaml", COMP_OUT_DIR)
+    shutil.move(f"{IN_DIR}/combined-am5.yaml", COMP_OUT_DIR)
 
     # Check that the combined yaml exists
     assert Path(f"{COMP_OUT_DIR}/combined-{COMP_EXPERIMENT}.yaml").exists()
 
-    # Go back to original directory
-    os.chdir(CWD)
-
 def test_combined_compileyaml_validation():
     """
     Validate the combined compile yaml
@@ -108,44 +102,35 @@ def test_combined_compileyaml_combinefail():
     Check to test if compile yaml is incorrect/does not exist,
     the combine fails. (compile yaml path misspelled)
     """
-    # Go into the input directory
-    os.chdir(f"{IN_DIR}/compile_yamls/compile_fail")
-
     # Model yaml path
-    modelyaml = "am5-wrong_compilefile.yaml"
+    modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_compilefile.yaml"))
     use = "compile"
 
     # Merge the yamls - should fail since there is no compile yaml specified in the model yaml
     try:
         cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use)
         # Move combined yaml to output location
-        shutil.move("combined-am5-wrong_compilefile.yaml", COMP_OUT_DIR)
+        shutil.move(f"{IN_DIR}/compile_yamls/compile_fail/combined-am5-wrong_compilefile.yaml", COMP_OUT_DIR)
     except:
         print("EXPECTED FAILURE")
         # Move combined yaml to output location
-        shutil.move("combined-am5-wrong_compilefile.yaml", COMP_OUT_DIR)
+        shutil.move(f"{IN_DIR}/compile_yamls/compile_fail/combined-am5-wrong_compilefile.yaml", COMP_OUT_DIR)
         assert True
 
-    # Go back to original directory
-    os.chdir(CWD)
-
 def test_combined_compileyaml_validatefail():
     """
     Check if the schema is validating correctly
     Branch should be string
     """
-    # Go into the input directory
-    os.chdir(f"{IN_DIR}/compile_yamls/compile_fail")
-
     # Model yaml path
-    modelyaml = "am5-wrong_datatype.yaml"
+    modelyaml = str(Path(f"{IN_DIR}/compile_yamls/compile_fail/am5-wrong_datatype.yaml"))
     use = "compile"
 
     # Merge the yamls
     cy.consolidate_yamls(modelyaml, COMP_EXPERIMENT, COMP_PLATFORM, COMP_TARGET, use)
 
     # Move combined yaml to output location
-    shutil.move("combined-am5-wrong_datatype.yaml", COMP_OUT_DIR)
+    shutil.move(f"{IN_DIR}/compile_yamls/compile_fail/combined-am5-wrong_datatype.yaml", COMP_OUT_DIR)
 
     # Validate against schema; should fail
     wrong_combined = Path(f"{COMP_OUT_DIR}/combined-am5-wrong_datatype.yaml")
@@ -166,9 +151,6 @@ def test_combined_compileyaml_validatefail():
     except:
         assert True
 
-    # Go back to original directory
-    os.chdir(CWD)
-
 ############ PP ############
 def test_expyaml_exists():
     """
@@ -188,9 +170,6 @@ def test_merged_pp_yamls():
     Check for the creation of the combined-[experiment] yaml
     Check that the model yaml was merged into the combined yaml
     """
-    # Go into the input directory
-    os.chdir(IN_DIR)
-
     # Model yaml path
     modelyaml = Path(f"{IN_DIR}/am5.yaml")
     use = "pp"
@@ -204,9 +183,6 @@ def test_merged_pp_yamls():
     # Check that the combined yaml exists
     assert Path(f"{PP_OUT_DIR}/combined-{PP_EXPERIMENT}.yaml").exists()
 
-    # Go back to original directory
-    os.chdir(CWD)
-
 def test_combined_ppyaml_validation():
     """
     Validate the combined compile yaml