Merge branch 'precompute-workflow.247' of github.com:NOAA-GFDL/fre-cl…

…i into checkout_edits

fre/cmor/README.md

@@ -29,18 +29,39 @@ this subtool's help, and command-specific `run` help:
 
 
 # subtool command-specific help, e.g. for run
-> fre cmor run --help
-    Usage: fre cmor run [OPTIONS]
-
-      Rewrite climate model output
-
-    Options:
-      -d, --indir TEXT         Input directory  [required]
-      -l, --varlist TEXT       Variable list  [required]
-      -r, --table_config TEXT  Table configuration  [required]
-      -p, --exp_config TEXT    Experiment configuration  [required]
-      -o, --outdir TEXT        Output directory  [required]
-      --help                   Show this message and exit.
+> fre cmor run --help                        
+Usage: fre cmor run [OPTIONS]                                                 
+                                                                              
+  Rewrite climate model output files with CMIP-compliant metadata for down-   
+  stream publishing                                                           
+                                                                              
+Options:                                                                      
+  -d, --indir TEXT         directory containing netCDF files. keys specified  
+                           in json_var_list are local variable names used for 
+                           targeting specific files in this directory         
+                           [required]                                         
+  -l, --varlist TEXT       path pointing to a json file containing directory  
+                           of key/value pairs. the keys are the 'local' names 
+                           used in the filename, and the values pointed to by 
+                           those keys are strings representing the name of the
+                           variable contained in targeted files. the key and  
+                           value are often the same, but it is not required.  
+                           [required]                                         
+  -r, --table_config TEXT  json file containing CMIP-compliant per-           
+                           variable/metadata for specific MIP table. The MIP  
+                           table can generally be identified by the specific  
+                           filename (e.g. 'Omon')  [required]                 
+  -p, --exp_config TEXT    json file containing metadata dictionary for       
+                           CMORization. this metadata is effectively appended 
+                           to the final output file's header  [required]      
+  -o, --outdir TEXT        directory root that will contain the full output   
+                           and output directory structure generated by the    
+                           cmor module upon request.  [required]              
+  -v, --opt_var_name TEXT  optional, specify a variable name to specifically  
+                           process only filenames matching that variable name.
+                           I.e., this string help target local_vars, not      
+                           target_vars.                                       
+  --help                   Show this message and exit.                        
 ```
 
 

fre/cmor/frecmor.py

@@ -11,35 +11,53 @@ def cmor_cli():
 @cmor_cli.command()
 @click.option("-d", "--indir",
               type=str,
-              help="Input directory",
+              help="directory containing netCDF files. keys specified in json_var_list are local " + \
+                   "variable names used for targeting specific files in this directory",
               required=True)
 @click.option("-l", "--varlist",
               type=str,
-              help="Variable list",
+              help="path pointing to a json file containing directory of key/value pairs. " + \
+                   "the keys are the \'local\' names used in the filename, and the values " + \
+                   "pointed to by those keys are strings representing the name of the variable " + \
+                   "contained in targeted files. the key and value are often the same, " + \
+                   "but it is not required.",
               required=True)
 @click.option("-r", "--table_config",
               type=str,
-              help="Table configuration",
+              help="json file containing CMIP-compliant per-variable/metadata for specific " + \
+                   "MIP table. The MIP table can generally be identified by the specific " + \
+                   "filename (e.g. \'Omon\')",
               required=True)
 @click.option("-p", "--exp_config",
               type=str,
-              help="Experiment configuration",
+              help="json file containing metadata dictionary for CMORization. this metadata is " + \
+                   "effectively appended to the final output file's header",
               required=True)
 @click.option("-o", "--outdir",
               type=str,
-              help="Output directory",
+              help="directory root that will contain the full output and output directory " + \
+                   "structure generated by the cmor module upon request.",
               required=True)
+@click.option('-v', "--opt_var_name",
+              type = str,
+              help="optional, specify a variable name to specifically process only filenames " + \
+                   "matching that variable name. I.e., this string help target local_vars, not " + \
+                   "target_vars.",
+              required=False)
 @click.pass_context
-def run(context, indir, varlist, table_config, exp_config, outdir):
+def run(context, indir, varlist, table_config, exp_config, outdir, opt_var_name):
     # pylint: disable=unused-argument
-    """Rewrite climate model output"""
+    """ 
+    Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing 
+    """ 
     context.invoke(
         _cmor_run_subtool,
         indir = indir,
         json_var_list = varlist,
         json_table_config = table_config,
         json_exp_config = exp_config,
-        outdir = outdir
+        outdir = outdir,
+        opt_var_name = opt_var_name
     )
     #    context.forward(
     #        _cmor_run_subtool() )

fre/make/createCheckout.py

@@ -5,10 +5,9 @@
 import logging
 import sys
 import click
-from .gfdlfremake import varsfre, yamlfre, checkout, targetfre
 import fre.yamltools.combine_yamls as cy
+from .gfdlfremake import varsfre, yamlfre, checkout, targetfre
 
-@click.command()
 def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
     # Define variables
     yml = yamlfile
@@ -65,7 +64,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
            RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName)
 
     ## Create the source directory for the platform
-         if iscontainer == False:
+         if iscontainer is False:
               srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
               # if the source directory does not exist, it is created
               if not os.path.exists(srcDir):
@@ -80,7 +79,7 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
                    print("\nCheckout script created in "+ srcDir + "/checkout.sh \n")
 
                    # Run the checkout script
-                   if run == True:
+                   if run is True:
                         freCheckout.run()
                    else:
                         sys.exit()
@@ -105,6 +104,13 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v
               freCheckout.finish(pc)
               print("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n")
 
+@click.command()
+def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
+    '''
+    Decorator for calling checkout_create - allows the decorated version
+    of the function to be separate from the undecorated version
+    '''
+    return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose)
 
 if __name__ == "__main__":
     checkout_create()

fre/make/createCompile.py

@@ -9,7 +9,6 @@
 from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal
 import fre.yamltools.combine_yamls as cy
 
-@click.command()
 def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
     # Define variables
     yml = yamlfile
@@ -60,10 +59,10 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
               raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml"))
 
          (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName)
-    ## Make the bldDir based on the modelRoot, the platform, and the target
+         ## Make the bldDir based on the modelRoot, the platform, and the target
          srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
          ## Check for type of build
-         if iscontainer == False:
+         if iscontainer is False:
               baremetalRun = True
               bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec"
               os.system("mkdir -p " + bldDir)
@@ -82,14 +81,19 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
               fremakeBuildList.append(fremakeBuild)
               click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
     if run:
-        #print("ITS GONNA RUN")
         if baremetalRun:
             pool = Pool(processes=nparallel)                         # Create a multiprocessing Pool
             pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)  # process data_inputs iterable with pool
-#        else:
-#            fremakeBuild.run()
     else:
         sys.exit()
 
+@click.command()
+def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose):
+    '''
+    Decorator for calling compile_create - allows the decorated version
+    of the function to be separate from the undecorated version
+    '''
+    return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose)
+
 if __name__ == "__main__":
     compile_create()

fre/make/createDocker.py

@@ -8,7 +8,6 @@
 from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker
 import fre.yamltools.combine_yamls as cy
 
-@click.command()
 def dockerfile_create(yamlfile,platform,target,execute):
     srcDir="src"
     checkoutScriptName = "checkout.sh"
@@ -52,7 +51,7 @@ def dockerfile_create(yamlfile,platform,target,execute):
             ## Make the bldDir based on the modelRoot, the platform, and the target
             srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
             ## Check for type of build
-            if iscontainer == True:
+            if iscontainer is True:
                 image="ecpe4s/noaa-intel-prototype:2023.09.25"
                 bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec"
                 tmpDir = "tmp/"+platformName
@@ -78,5 +77,13 @@ def dockerfile_create(yamlfile,platform,target,execute):
             else:
                 sys.exit()
 
+@click.command()
+def _dockerfile_create(yamlfile,platform,target,execute):
+    '''
+    Decorator for calling dockerfile_create - allows the decorated version
+    of the function to be separate from the undecorated version
+    '''
+    return dockerfile_create(yamlfile,platform,target,execute)
+
 if __name__ == "__main__":
     dockerfile_create()

fre/make/createMakefile.py

@@ -8,7 +8,6 @@
 from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre
 import fre.yamltools.combine_yamls as cy
 
-@click.command()
 def makefile_create(yamlfile,platform,target):
     srcDir="src"
     checkoutScriptName = "checkout.sh"
@@ -46,7 +45,7 @@ def makefile_create(yamlfile,platform,target):
   ## Make the bldDir based on the modelRoot, the platform, and the target
             srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src"
             ## Check for type of build
-            if iscontainer == False:
+            if iscontainer is False:
                 baremetalRun = True
                 bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + targetObject.gettargetName() + "/exec"
                 os.system("mkdir -p " + bldDir)
@@ -78,5 +77,13 @@ def makefile_create(yamlfile,platform,target):
                 freMakefile.writeMakefile()
                 click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n")
 
+@click.command()
+def _makefile_create(yamlfile,platform,target):
+    '''
+    Decorator for calling makefile_create - allows the decorated version
+    of the function to be separate from the undecorated version
+    '''
+    return makefile_create(yamlfile,platform,target)
+
 if __name__ == "__main__":
     makefile_create()

fre/make/fremake.py

@@ -1,9 +1,9 @@
 import click
-from .createCheckout import checkout_create
-from .createCompile import compile_create
-from .createDocker import dockerfile_create
-from .createMakefile import makefile_create
-from .runFremake import fremake_run
+from fre.make import createCheckout
+from fre.make import createMakefile
+from fre.make import createCompile
+from fre.make import createDocker
+from fre.make import runFremake
 
 yamlfile_opt_help = """Experiment yaml compile FILE
 """
@@ -73,7 +73,7 @@ def make_cli():
 @click.pass_context
 def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose):
     """ - Perform all fremake functions to run checkout and compile model"""
-    context.forward(fremake_run)
+    context.forward(runFremake._fremake_run)
 
 ####
 @make_cli.command()
@@ -114,7 +114,7 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel
 @click.pass_context
 def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose):
     """ - Write the checkout script """
-    context.forward(checkout_create)
+    context.forward(createCheckout._checkout_create)
 
 #####
 @make_cli.command
@@ -136,7 +136,7 @@ def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,e
 @click.pass_context
 def create_makefile(context,yamlfile,platform,target):
     """ - Write the makefile """
-    context.forward(makefile_create)
+    context.forward(createMakefile._makefile_create)
 
 #####
 
@@ -178,7 +178,7 @@ def create_makefile(context,yamlfile,platform,target):
 @click.pass_context
 def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose):
     """ - Write the compile script """
-    context.forward(compile_create)
+    context.forward(createCompile._compile_create)
 
 @make_cli.command
 @click.option("-y",
@@ -202,8 +202,7 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos
 @click.pass_context
 def create_dockerfile(context,yamlfile,platform,target,execute):
     """ - Write the dockerfile """
-    context.forward(dockerfile_create)
-
+    context.forward(createDocker._dockerfile_create)
 
 if __name__ == "__main__":
     make_cli()

fre/make/gfdlfremake/checkout.py

@@ -111,7 +111,7 @@ def finish (self,pc):
 ## TODO: batch script building
     def run (self):
         """
-        Brief: Changes the permission on the checkout script and runs it
+        Brief: Runs the checkout script
         Param:
             - self The checkout script object
         """

fre/make/runFremake.py

@@ -9,15 +9,12 @@
 import logging
 from multiprocessing.dummy import Pool
 from pathlib import Path
-
 import click
-
 import fre.yamltools.combine_yamls as cy
 from .gfdlfremake import (
     targetfre, varsfre, yamlfre, checkout,
     makefilefre, buildDocker, buildBaremetal )
 
-@click.command()
 def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
     ''' run fremake via click'''
     yml = yamlfile
@@ -89,6 +86,7 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
                 freCheckout = checkout.checkout("checkout.sh",srcDir)
                 freCheckout.writeCheckout(modelYaml.compile.getCompileYaml(),jobs,pc)
                 freCheckout.finish(pc)
+                os.chmod(srcDir+"/checkout.sh", 0o744)
                 ## TODO: Options for running on login cluster?
                 freCheckout.run()
 
@@ -202,6 +200,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb
             # process data_inputs iterable with pool
             pool.map(buildBaremetal.fremake_parallel,fremakeBuildList)
 
+@click.command()
+def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose):
+    '''
+    Decorator for calling _fremake_run - allows the decorated version
+    of the function to be separate from the undecorated version
+    '''
+    return fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose)
 
 if __name__ == "__main__":
     fremake_run()

fre/tests/test_files/CMORbite_var_list.json

@@ -0,0 +1,11 @@
+{
+	"lai": "lai",
+	"t_ref": "t_ref",
+	"cl": "cl",
+	"mc": "mc",
+	"ta": "ta",
+	"sos": "sos",
+	"so": "so",
+	"ch4global": "ch4global",
+	"gppLut": "gppLut"
+}

fre/tests/test_fre_app_cli.py

@@ -1,6 +1,7 @@
 """ test "fre app" calls """
 
 import os
+import subprocess
 from pathlib import Path
 
 import click