675.force compile option

fre/__init__.py

@@ -10,7 +10,7 @@
 
 fre_logger = logging.getLogger(__name__)
 
-FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s"
+FORMAT = "[%(levelname)7s:%(filename)25s:%(funcName)24s] %(message)s"
 logging.basicConfig(level = logging.WARNING,
                     format = FORMAT,
                     filename = None,

fre/make/create_checkout_script.py

@@ -24,7 +24,7 @@ def baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute):
 
     # Make checkout script executable
     os.chmod(src_dir+"/checkout.sh", 0o744)
-    fre_logger.info("Checkout script created in %s/checkout.sh", src_dir )
+    fre_logger.info("Checkout script created: %s/checkout.sh", src_dir )
 
     if execute:
         fre_checkout.run()
@@ -36,7 +36,7 @@ def container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc):
     fre_checkout = checkout.checkoutForContainer("checkout.sh", src_dir, tmp_dir)
     fre_checkout.writeCheckout(model_yaml.compile.getCompileYaml(), jobs, pc)
     fre_checkout.finish(model_yaml.compile.getCompileYaml(), pc)
-    fre_logger.info("Checkout script created in ./%s/checkout.sh", tmp_dir)
+    fre_logger.info("Checkout script created: ./%s/checkout.sh", tmp_dir)
 
 def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_checkout: Optional[bool] = None,
                     njobs: int = 4, execute: Optional[bool] = False, force_checkout: Optional[bool] = False):
@@ -106,6 +106,7 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
         target = targetfre.fretarget(target_name)
 
     fre_logger.setLevel(level = logging.INFO)
+#    fre_logger.info("")
     ## Loop through the platforms specified on the command line
     ## If the platform is a baremetal platform, write the checkout script and run it once
     ## This should be done separately and serially because bare metal platforms should all be using
@@ -129,25 +130,22 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
                 # Create and run (if --execute passed) the checkout script
                 baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute)
             elif os.path.exists(f"{src_dir}/checkout.sh") and force_checkout:
-                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir)
-                fre_logger.info("*** REMOVING CHECKOUT SCRIPT ***")
+                fre_logger.warning("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir)
+                fre_logger.warning("*** REMOVING CHECKOUT SCRIPT ***")
                 # Remove the checkout script
                 os.remove(f"{src_dir}/checkout.sh")
 
                 # Re-create and run (if --execute passed) the checkout script
                 baremetal_checkout_write(model_yaml, src_dir, jobs, pc, execute)
 
             elif os.path.exists(f"{src_dir}/checkout.sh") and not force_checkout:
-                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir)
+                fre_logger.warning("Checkout script PREVIOUSLY created in %s/checkout.sh", src_dir)
                 if execute:
                     try:
                         subprocess.run(args=[src_dir+"/checkout.sh"], check=True)
                     except Exception as exc:
                         raise OSError(f"\nThere was an error with the checkout script {src_dir}/checkout.sh.",
                                       f"\nTry removing test folder: {platform['modelRoot']}\n") from exc
-                else:
-                    return
-
         else:
             src_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/src'
             tmp_dir = f"tmp/{platform_name}"
@@ -159,12 +157,13 @@ def checkout_create(yamlfile: str, platform: str, target: str, no_parallel_check
                 container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc)
             # If checkout script exists and force_checkout is used:
             elif os.path.exists(f"{tmp_dir}/checkout.sh") and force_checkout:
-                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", tmp_dir)
+                fre_logger.info("Checkout script PREVIOUSLY created: %s/checkout.sh", tmp_dir)
                 fre_logger.info("*** REMOVING CHECKOUT SCRIPT ***")
                 # remove
                 os.remove(f"{tmp_dir}/checkout.sh")
                 # recreate
                 container_checkout_write(model_yaml, src_dir, tmp_dir, jobs, pc)
             # If checkout script exists but force_checkout is not used
             elif os.path.exists(f"{tmp_dir}/checkout.sh") and not force_checkout:
-                fre_logger.info("Checkout script PREVIOUSLY created in %s/checkout.sh", tmp_dir)
+                fre_logger.info("Checkout script PREVIOUSLY created: %s/checkout.sh", tmp_dir)
+    fre_logger.info("")

fre/make/create_compile_script.py

@@ -1,23 +1,55 @@
 '''
 Creates a compile script to compile the model and generate a model executable.
 '''
-
-import os
 import logging
 
 from pathlib import Path
 from multiprocessing.dummy import Pool
+#import filecmp
+#import difflib
 
-import fre.yamltools.combine_yamls_script as cy
 from typing import Optional
+import fre.yamltools.combine_yamls_script as cy
 from fre.make.make_helpers import get_mktemplate_path
 from .gfdlfremake import varsfre, yamlfre, targetfre, buildBaremetal
 
 fre_logger = logging.getLogger(__name__)
 
+def compile_call(fremake_yaml, template_path, src_dir, bld_dir, target, platform, jobs):
+    """
+    lkjsdlfkjs
+
+    :param fremake_yaml:
+    :type fremake_yaml:
+    :param template_path:
+    :type template_path:
+    :param src_dir:
+    :type src_dir:
+    :param bld_dir:
+    :type bld_dir:
+    :param target:
+    :type target:
+    :param platform:
+    :type platform:
+    :param jobs:
+    :type jobs:
+    """
+    fremake_build = buildBaremetal.buildBaremetal(exp=fremake_yaml["experiment"],
+                                                 mkTemplatePath=template_path,
+                                                 srcDir=src_dir,
+                                                 bldDir=bld_dir,
+                                                 target=target,
+                                                 env_setup=platform["envSetup"],
+                                                 jobs=jobs)
+    for c in fremake_yaml['src']:
+        fremake_build.writeBuildComponents(c)
+    fremake_build.writeScript()
+    fre_logger.info("Compile script created: %s/compile.sh", bld_dir)
+
 def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4,
                    nparallel: int = 1, execute: Optional[bool] = False,
-                   verbose: Optional[bool] = None):
+                   verbose: Optional[bool] = False,
+                   force_compile: Optional[bool] = False):
     """
     Creates the compile script for bare-metal build
 
@@ -28,14 +60,17 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4,
     :type platform: str
     :param target: Predefined FRE targets; options include [prod/debug/repro]-openmp
     :type target: str
-    :param njobs: Used for parallelism with make; number of files to build simultaneously; on a per-build basis (default 4)
+    :param njobs: Used for parallelism with make; number of files to build simultaneously;
+                  on a per-build basis (default 4)
     :type njobs: int
     :param nparallel: Number of concurrent model builds (default 1)
     :type nparallel: int
     :param execute: Run the created compile script to build a model executable
     :type execute: bool
     :param verbose: Increase verbosity output
     :type verbose: bool
+    :param force_compile: Re-create compile script if specified
+    :type force_compile: bool
     :raises ValueError:
         - Error if platform does not exist in platforms yaml configuration 
         - Error if the mkmf template defined in platforms yaml does not exist
@@ -44,23 +79,12 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4,
     # Define variables
     yml = yamlfile
     name = yamlfile.split(".")[0]
-    nparallel = nparallel
     jobs = str(njobs)
 
     if verbose:
         fre_logger.setLevel(level=logging.DEBUG)
-    else:
-        fre_logger.setLevel(level=logging.INFO)
-
-    srcDir = "src"
-    baremetalRun = False  # This is needed if there are no bare metal runs
-
-    ## Split and store the platforms and targets in a list
-    plist = platform
-    tlist = target
-
-    # Combined compile yaml file
-    # combined = Path(f"combined-{name}.yaml")
+#    else:
+#        fre_logger.setLevel(level=logging.INFO)
 
     # Combine model, compile, and platform yamls
     full_combined = cy.consolidate_yamls(yamlfile=yml,
@@ -74,52 +98,88 @@ def compile_create(yamlfile:str, platform:str, target:str, njobs: int = 4,
     fre_vars = varsfre.frevars(full_combined)
 
     ## Open the yaml file, validate the yaml, and parse as fremake_yaml
-    modelYaml = yamlfre.freyaml(full_combined, fre_vars)
-    fremakeYaml = modelYaml.getCompileYaml()
+    model_yaml = yamlfre.freyaml(full_combined, fre_vars)
+    fremake_yaml = model_yaml.getCompileYaml()
 
+    tlist = target
     ## Error checking the targets
-    for targetName in tlist:
-        target = targetfre.fretarget(targetName)
+    for target_name in tlist:
+        target = targetfre.fretarget(target_name)
 
-    fremakeBuildList = []
+    fremake_build_list = []
     ## Loop through platforms and targets
-    for platformName in plist:
-        for targetName in tlist:
-            target = targetfre.fretarget(targetName)
-            if not modelYaml.platforms.hasPlatform(platformName):
-                raise ValueError(f"{platformName} does not exist in platforms.yaml")
-
-            platform = modelYaml.platforms.getPlatformFromName(platformName)
-            ## Make the bldDir based on the modelRoot, the platform, and the target
-            srcDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/src"
+    for platform_name in platform:
+        for target_name in tlist:
+            target = targetfre.fretarget(target_name)
+            if not model_yaml.platforms.hasPlatform(platform_name):
+                raise ValueError(f"{platform_name} does not exist in platforms.yaml")
+
+            platform = model_yaml.platforms.getPlatformFromName(platform_name)
+            ## Make the bld_dir based on the modelRoot, the platform, and the target
+            src_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/src'
             ## Check for type of build
             if platform["container"] is False:
-                baremetalRun = True
-                bldDir = f'{platform["modelRoot"]}/{fremakeYaml["experiment"]}/' + \
-                         f'{platformName}-{target.gettargetName()}/exec'
-                os.system("mkdir -p " + bldDir)
+#                baremetalRun = True
+                bld_dir = f'{platform["modelRoot"]}/{fremake_yaml["experiment"]}/' + \
+                         f'{platform_name}-{target.gettargetName()}/exec'
+                Path(bld_dir).mkdir(parents=True, exist_ok=True)
 
                 template_path = get_mktemplate_path(mk_template = platform["mkTemplate"],
                                                     model_root = platform["modelRoot"],
                                                     container_flag = platform["container"])
 
-                ## Create a list of compile scripts to run in parallel
-                fremakeBuild = buildBaremetal.buildBaremetal(exp=fremakeYaml["experiment"],
-                                                             mkTemplatePath=template_path,
-                                                             srcDir=srcDir,
-                                                             bldDir=bldDir,
-                                                             target=target,
-                                                             env_setup=platform["envSetup"],
-                                                             jobs=jobs)
-                for c in fremakeYaml['src']:
-                    fremakeBuild.writeBuildComponents(c)
-                fremakeBuild.writeScript()
-                fremakeBuildList.append(fremakeBuild)
-                fre_logger.info("\nCompile script created at " + bldDir + "/compile.sh" + "\n")
-
+                if not Path(f"{bld_dir}/compile.sh").exists():
+                    ## Create a list of compile scripts to run in parallel
+                    compile_call(fremake_yaml = fremake_yaml,
+                                 template_path = template_path,
+                                 src_dir = src_dir,
+                                 bld_dir = bld_dir,
+                                 target = target,
+                                 platform = platform,
+                                 jobs = jobs)
+                    fremake_build_list.append(f"{bld_dir}/compile.sh")
+                elif Path(f"{bld_dir}/compile.sh").exists() and force_compile:
+                    # Remove old compile script
+                    fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir)
+                    fre_logger.warning("*** REMOVING COMPILE SCRIPT ***")
+                    Path(f"{bld_dir}/compile.sh").unlink()
+
+                    # Re-create compile script
+                    compile_call(fremake_yaml = fremake_yaml,
+                                 template_path = template_path,
+                                 src_dir = src_dir,
+                                 bld_dir = bld_dir,
+                                 target = target,
+                                 platform = platform,
+                                 jobs = jobs)
+
+                    fremake_build_list.append(f"{bld_dir}/compile.sh")
+                elif Path(f"{bld_dir}/compile.sh").exists() and not force_compile:
+                    fre_logger.warning("Compile script PREVIOUSLY created: %s/compile.sh", bld_dir)
+                    fremake_build_list.append(f"{bld_dir}/compile.sh")
+                    ###COMPARE THE TWO TO SEE IF IT'S CHANGED###--> filecmp or difflib
+                    ###IF CHANGED, THROW ERROR###
+                    ###SHOULD IT ALSO BE RE-CREATED IF CHECKOUT RE-CREATED?? -->
+                    ### I THINK THIS WILL BE FOR "ALL" SUBTOOL###
+
+    fre_logger.setLevel(level=logging.INFO)
+    fre_logger.info("Compile scripts available/generated with specified platform-target combination: ")
+    for i in fremake_build_list:
+        fre_logger.info("  - %s", i)
+    fre_logger.setLevel(level=logging.WARNING)
+
+    # Returns the exit status for multiprocessing pool command
     if execute:
-        if baremetalRun:
-            pool = Pool(processes=nparallel)  # Create a multiprocessing Pool
-            pool.map(buildBaremetal.fremake_parallel, fremakeBuildList)  # process data_inputs iterable with pool
-    else:
-        return
+#        if baremetalRun:
+        # Create a multiprocessing Pool
+        pool = Pool(processes=nparallel)
+        # process data_inputs iterable with pool
+        results = pool.map(buildBaremetal.fremake_parallel, fremake_build_list)
+
+        for r in results:
+            for key,value in r.items():
+                if key == 1:
+                    fre_logger.error("ERROR: compile NOT successful")
+                    fre_logger.error("Check the generated log: %s", value)
+                elif key == 0:
+                    fre_logger.info("Compile successful")

fre/make/create_makefile_script.py

@@ -93,7 +93,7 @@ def makefile_create(yamlfile: str, platform: str, target:str):
                 freMakefile.writeMakefile()
                 former_log_level = fre_logger.level
                 fre_logger.setLevel(logging.INFO)
-                fre_logger.info("Makefile created in " + bldDir + "/Makefile")
+                fre_logger.info("Makefile created: " + bldDir + "/Makefile")
                 fre_logger.setLevel(former_log_level)
             else:
                 bldDir = platform["modelRoot"] + "/" + fremakeYaml["experiment"] + "/exec"
@@ -115,5 +115,6 @@ def makefile_create(yamlfile: str, platform: str, target:str):
                 freMakefile.writeMakefile()
                 former_log_level = fre_logger.level
                 fre_logger.setLevel(logging.INFO)
-                fre_logger.info("Makefile created in " + tmpDir + "/Makefile")
+                fre_logger.info("Makefile created: " + tmpDir + "/Makefile")
                 fre_logger.setLevel(former_log_level)
+    fre_logger.warning("")

fre/make/fremake.py

@@ -77,15 +77,18 @@ def make_cli():
               help = "Use this to run the created compilation script.")
 @click.option("--force-checkout",
               is_flag = True,
-              help = "Force checkout in case the source directory exists.")
+              help = "Re-create the checkout script in case the source directory exists.")
+@click.option("--force-compile",
+              is_flag = True,
+              help = "Re-create the compile script in case it exists already.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = VERBOSE_OPT_HELP)
-def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout):
+def all(yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout, force_compile):
     """ - Perform all fre make functions; run checkout and compile scripts to create model executable or container"""
     run_fremake_script.fremake_run(
-        yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout)
+        yamlfile, platform, target, nparallel, njobs, no_parallel_checkout, no_format_transfer, execute, verbose, force_checkout, force_compile)
 
 @make_cli.command()
 @click.option("-y",
@@ -177,14 +180,17 @@ def makefile(yamlfile, platform, target):
               is_flag = True,
               default = False,
               help = "Use this to run the created checkout script.")
+@click.option("--force-compile",
+              is_flag = True,
+              help = "Re-create the compile script in case it exists already.")
 @click.option("-v",
               "--verbose",
               is_flag = True,
               help = VERBOSE_OPT_HELP)
-def compile_script(yamlfile, platform, target, njobs, nparallel, execute, verbose):
+def compile_script(yamlfile, platform, target, njobs, nparallel, execute, verbose, force_compile):
     """ - Write the compile script """
     create_compile_script.compile_create(
-        yamlfile, platform, target, njobs, nparallel, execute, verbose)
+        yamlfile, platform, target, njobs, nparallel, execute, verbose, force_compile)
 
 @make_cli.command
 @click.option("-y",