From 4a008b9046da016f00cd8f3b5eeb838fb214ab0b Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 29 Oct 2024 17:17:34 -0400 Subject: [PATCH 1/4] #230 Add decorator functions to pass click options correctly - this was needed in order to import fre make scripts successfully when creating pytests --- fre/make/createCheckout.py | 10 ++++++++-- fre/make/createCompile.py | 16 ++++++++++------ fre/make/createDocker.py | 10 +++++++++- fre/make/createMakefile.py | 11 +++++++++-- fre/make/runFremake.py | 12 ++++++++---- 5 files changed, 44 insertions(+), 15 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index 595c3054..d94a021a 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -8,8 +8,7 @@ from .gfdlfremake import varsfre, yamlfre, checkout, targetfre import fre.yamltools.combine_yamls as cy -@click.command() -def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -104,6 +103,13 @@ def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,v freCheckout.finish(pc) click.echo("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n") +@click.command() +def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): + ''' + Decorator for calling _checkout_create - allows the decorated version + of the function to be separate from the undecorated version + ''' + return _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) if __name__ == "__main__": checkout_create() diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 90fc8127..2a354230 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -9,8 +9,7 @@ from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal import fre.yamltools.combine_yamls as cy -@click.command() -def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -60,7 +59,7 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): raise ValueError (platformName + " does not exist in " + modelYaml.combined.get("compile").get("platformYaml")) (compiler,modules,modulesInit,fc,cc,modelRoot,iscontainer,mkTemplate,containerBuild,ContainerRun,RUNenv)=modelYaml.platforms.getPlatformFromName(platformName) - ## Make the bldDir based on the modelRoot, the platform, and the target + ## Make the bldDir based on the modelRoot, the platform, and the target srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build if iscontainer == False: @@ -82,14 +81,19 @@ def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): fremakeBuildList.append(fremakeBuild) click.echo("\nCompile script created at " + bldDir + "/compile.sh" + "\n") if run: - #print("ITS GONNA RUN") if baremetalRun: pool = Pool(processes=nparallel) # Create a multiprocessing Pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) # process data_inputs iterable with pool -# else: -# fremakeBuild.run() else: sys.exit() +@click.command() +def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): + ''' + Decorator for calling _compile_create - allows the decorated version + of the function to be separate from the undecorated version + ''' + return _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose) + if __name__ == "__main__": compile_create() diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index 27639751..0dc698e7 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -9,7 +9,7 @@ import fre.yamltools.combine_yamls as cy @click.command() -def dockerfile_create(yamlfile,platform,target,execute): +def _dockerfile_create(yamlfile,platform,target,execute): srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -78,5 +78,13 @@ def dockerfile_create(yamlfile,platform,target,execute): else: sys.exit() +@click.command() +def dockerfile_create(yamlfile,platform,target,execute): + ''' + Decorator for calling _dockerfile_create - allows the decorated version + of the function to be separate from the undecorated version + ''' + return _dockerfile_create(yamlfile,platform,target,execute) + if __name__ == "__main__": dockerfile_create() diff --git a/fre/make/createMakefile.py b/fre/make/createMakefile.py index 0232bf4b..33170391 100644 --- a/fre/make/createMakefile.py +++ b/fre/make/createMakefile.py @@ -8,8 +8,7 @@ from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre import fre.yamltools.combine_yamls as cy -@click.command() -def makefile_create(yamlfile,platform,target): +def _makefile_create(yamlfile,platform,target): srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -78,5 +77,13 @@ def makefile_create(yamlfile,platform,target): freMakefile.writeMakefile() click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n") +@click.command() +def makefile_create(yamlfile,platform,target): + ''' + Decorator for calling _makefile_create - allows the decorated version + of the function to be separate from the undecorated version + ''' + return _makefile_create(yamlfile,platform,target) + if __name__ == "__main__": makefile_create() diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 934a07fd..14fa5ec1 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -9,16 +9,13 @@ import logging from multiprocessing.dummy import Pool from pathlib import Path - import click - import fre.yamltools.combine_yamls as cy from .gfdlfremake import ( targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal ) -@click.command() -def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): +def _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): ''' run fremake via click''' yml = yamlfile name = yamlfile.split(".")[0] @@ -202,6 +199,13 @@ def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verb # process data_inputs iterable with pool pool.map(buildBaremetal.fremake_parallel,fremakeBuildList) +@click.command() +def fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose): + ''' + Decorator for calling _fremake_run - allows the decorated version + of the function to be separate from the undecorated version + ''' + return _fremake_run(yamlfile,platform,target,parallel,jobs,no_parallel_checkout,verbose) if __name__ == "__main__": fremake_run() From 43475ab8ddb81c0a9b5b9a72142ad047362f63aa Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 29 Oct 2024 17:29:23 -0400 Subject: [PATCH 2/4] #230 Fix little pylint messages --- fre/make/createCheckout.py | 8 ++++---- fre/make/createCompile.py | 2 +- fre/make/createDocker.py | 2 +- fre/make/createMakefile.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index d94a021a..6217a2c1 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -5,8 +5,8 @@ import logging import sys import click -from .gfdlfremake import varsfre, yamlfre, checkout, targetfre import fre.yamltools.combine_yamls as cy +from .gfdlfremake import varsfre, yamlfre, checkout, targetfre def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): # Define variables @@ -64,7 +64,7 @@ def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute, RUNenv ) = modelYaml.platforms.getPlatformFromName(platformName) ## Create the source directory for the platform - if iscontainer == False: + if iscontainer is False: srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" # if the source directory does not exist, it is created if not os.path.exists(srcDir): @@ -77,13 +77,13 @@ def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute, click.echo("\nCheckout script created in "+ srcDir + "/checkout.sh \n") # Run the checkout script - if run == True: + if run is True: freCheckout.run() else: sys.exit() else: print("\nCheckout script PREVIOUSLY created in "+ srcDir + "/checkout.sh \n") - if run == True: + if run is True: os.chmod(srcDir+"/checkout.sh", 0o744) try: subprocess.run(args=[srcDir+"/checkout.sh"], check=True) diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 2a354230..85edb171 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -62,7 +62,7 @@ def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): ## Make the bldDir based on the modelRoot, the platform, and the target srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build - if iscontainer == False: + if iscontainer is False: baremetalRun = True bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + target.gettargetName() + "/exec" os.system("mkdir -p " + bldDir) diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index 0dc698e7..0ff042f4 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -52,7 +52,7 @@ def _dockerfile_create(yamlfile,platform,target,execute): ## Make the bldDir based on the modelRoot, the platform, and the target srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build - if iscontainer == True: + if iscontainer is True: image="ecpe4s/noaa-intel-prototype:2023.09.25" bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/exec" tmpDir = "tmp/"+platformName diff --git a/fre/make/createMakefile.py b/fre/make/createMakefile.py index 33170391..57b27577 100644 --- a/fre/make/createMakefile.py +++ b/fre/make/createMakefile.py @@ -45,7 +45,7 @@ def _makefile_create(yamlfile,platform,target): ## Make the bldDir based on the modelRoot, the platform, and the target srcDir = modelRoot + "/" + fremakeYaml["experiment"] + "/src" ## Check for type of build - if iscontainer == False: + if iscontainer is False: baremetalRun = True bldDir = modelRoot + "/" + fremakeYaml["experiment"] + "/" + platformName + "-" + targetObject.gettargetName() + "/exec" os.system("mkdir -p " + bldDir) From c7f896359cd42b8bba5264bdcb63e2e4f9ccfe4d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 30 Oct 2024 12:53:19 -0400 Subject: [PATCH 3/4] #230 Reverse the underscore convention --- fre/make/createCheckout.py | 8 ++++---- fre/make/createCompile.py | 8 ++++---- fre/make/createDocker.py | 9 ++++----- fre/make/createMakefile.py | 8 ++++---- fre/make/fremake.py | 21 ++++++++++----------- 5 files changed, 26 insertions(+), 28 deletions(-) diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index 6217a2c1..c665e9bf 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -8,7 +8,7 @@ import fre.yamltools.combine_yamls as cy from .gfdlfremake import varsfre, yamlfre, checkout, targetfre -def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -104,12 +104,12 @@ def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute, click.echo("\nCheckout script created at " + tmpDir + "/checkout.sh" + "\n") @click.command() -def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): +def _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): ''' - Decorator for calling _checkout_create - allows the decorated version + Decorator for calling checkout_create - allows the decorated version of the function to be separate from the undecorated version ''' - return _checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) + return checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose) if __name__ == "__main__": checkout_create() diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 85edb171..36068329 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -9,7 +9,7 @@ from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal import fre.yamltools.combine_yamls as cy -def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): # Define variables yml = yamlfile name = yamlfile.split(".")[0] @@ -88,12 +88,12 @@ def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): sys.exit() @click.command() -def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): +def _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): ''' - Decorator for calling _compile_create - allows the decorated version + Decorator for calling compile_create - allows the decorated version of the function to be separate from the undecorated version ''' - return _compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose) + return compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose) if __name__ == "__main__": compile_create() diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index 0ff042f4..59b73ee9 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -8,8 +8,7 @@ from .gfdlfremake import varsfre, targetfre, yamlfre, buildDocker import fre.yamltools.combine_yamls as cy -@click.command() -def _dockerfile_create(yamlfile,platform,target,execute): +def dockerfile_create(yamlfile,platform,target,execute): srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -79,12 +78,12 @@ def _dockerfile_create(yamlfile,platform,target,execute): sys.exit() @click.command() -def dockerfile_create(yamlfile,platform,target,execute): +def _dockerfile_create(yamlfile,platform,target,execute): ''' - Decorator for calling _dockerfile_create - allows the decorated version + Decorator for calling dockerfile_create - allows the decorated version of the function to be separate from the undecorated version ''' - return _dockerfile_create(yamlfile,platform,target,execute) + return dockerfile_create(yamlfile,platform,target,execute) if __name__ == "__main__": dockerfile_create() diff --git a/fre/make/createMakefile.py b/fre/make/createMakefile.py index 57b27577..eaf340dd 100644 --- a/fre/make/createMakefile.py +++ b/fre/make/createMakefile.py @@ -8,7 +8,7 @@ from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre import fre.yamltools.combine_yamls as cy -def _makefile_create(yamlfile,platform,target): +def makefile_create(yamlfile,platform,target): srcDir="src" checkoutScriptName = "checkout.sh" baremetalRun = False # This is needed if there are no bare metal runs @@ -78,12 +78,12 @@ def _makefile_create(yamlfile,platform,target): click.echo("\nMakefile created at " + bldDir + "/Makefile" + "\n") @click.command() -def makefile_create(yamlfile,platform,target): +def _makefile_create(yamlfile,platform,target): ''' - Decorator for calling _makefile_create - allows the decorated version + Decorator for calling makefile_create - allows the decorated version of the function to be separate from the undecorated version ''' - return _makefile_create(yamlfile,platform,target) + return makefile_create(yamlfile,platform,target) if __name__ == "__main__": makefile_create() diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 54946349..8764a8aa 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -1,9 +1,9 @@ import click -from .createCheckout import checkout_create -from .createCompile import compile_create -from .createDocker import dockerfile_create -from .createMakefile import makefile_create -from .runFremake import fremake_run +from .createCheckout import _checkout_create +from .createCompile import _compile_create +from .createDocker import _dockerfile_create +from .createMakefile import _makefile_create +from .runFremake import _fremake_run yamlfile_opt_help = """Experiment yaml compile FILE """ @@ -73,7 +73,7 @@ def make_cli(): @click.pass_context def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): """ - Perform all fremake functions to run checkout and compile model""" - context.forward(fremake_run) + context.forward(_fremake_run) #### @make_cli.command() @@ -114,7 +114,7 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel @click.pass_context def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): """ - Write the checkout script """ - context.forward(checkout_create) + context.forward(_checkout_create) ##### @make_cli.command @@ -136,7 +136,7 @@ def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,e @click.pass_context def create_makefile(context,yamlfile,platform,target): """ - Write the makefile """ - context.forward(makefile_create) + context.forward(_makefile_create) ##### @@ -178,7 +178,7 @@ def create_makefile(context,yamlfile,platform,target): @click.pass_context def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose): """ - Write the compile script """ - context.forward(compile_create) + context.forward(_compile_create) @make_cli.command @click.option("-y", @@ -202,8 +202,7 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos @click.pass_context def create_dockerfile(context,yamlfile,platform,target,execute): """ - Write the dockerfile """ - context.forward(dockerfile_create) - + context.forward(_dockerfile_create) if __name__ == "__main__": make_cli() From 7032e241a1d6e654b383acccd28a9a23d85d9598 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 30 Oct 2024 16:04:44 -0400 Subject: [PATCH 4/4] #230 Make relative imports absolute --- fre/make/fremake.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 8764a8aa..b2053e5f 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -1,9 +1,9 @@ import click -from .createCheckout import _checkout_create -from .createCompile import _compile_create -from .createDocker import _dockerfile_create -from .createMakefile import _makefile_create -from .runFremake import _fremake_run +from fre.make import createCheckout +from fre.make import createMakefile +from fre.make import createCompile +from fre.make import createDocker +from fre.make import runFremake yamlfile_opt_help = """Experiment yaml compile FILE """ @@ -73,7 +73,7 @@ def make_cli(): @click.pass_context def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): """ - Perform all fremake functions to run checkout and compile model""" - context.forward(_fremake_run) + context.forward(runfremake._fremake_run) #### @make_cli.command() @@ -114,7 +114,7 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel @click.pass_context def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): """ - Write the checkout script """ - context.forward(_checkout_create) + context.forward(createCheckout._checkout_create) ##### @make_cli.command @@ -136,7 +136,7 @@ def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,e @click.pass_context def create_makefile(context,yamlfile,platform,target): """ - Write the makefile """ - context.forward(_makefile_create) + context.forward(createMakefile._makefile_create) ##### @@ -178,7 +178,7 @@ def create_makefile(context,yamlfile,platform,target): @click.pass_context def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbose): """ - Write the compile script """ - context.forward(_compile_create) + context.forward(createCompile._compile_create) @make_cli.command @click.option("-y", @@ -202,7 +202,7 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos @click.pass_context def create_dockerfile(context,yamlfile,platform,target,execute): """ - Write the dockerfile """ - context.forward(_dockerfile_create) + context.forward(createDocker._dockerfile_create) if __name__ == "__main__": make_cli()