697.workflow checkout tool

fre/__init__.py

@@ -10,7 +10,7 @@
 
 fre_logger = logging.getLogger(__name__)
 
-FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s"
+FORMAT = "[%(levelname)7s:%(filename)24s:%(funcName)24s] %(message)s"
 logging.basicConfig(level = logging.WARNING,
                     format = FORMAT,
                     filename = None,

fre/fre.py

@@ -22,7 +22,8 @@
 # click and lazy group loading
 @click.group(
     cls = LazyGroup,
-    lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
+    lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli",
+                       "pp": ".pp.frepp.pp_cli",
                        "catalog": ".catalog.frecatalog.catalog_cli",
                        "list": ".list_.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",

fre/tests/test_fre_cmor_cli.py

@@ -70,8 +70,8 @@ def test_cli_fre_cmor_help_and_debuglog():
     assert result.exit_code == 0
     assert Path("TEST_FOO_LOG.log").exists()
 
-    log_text_line_1='[ INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
-    log_text_line_2='[DEBUG:                  fre.py:                     fre] click entry-point function call done.\n' # pylint: disable=line-too-long
+    log_text_line_1='[   INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
+    log_text_line_2='[  DEBUG:                  fre.py:                     fre] click entry-point function call done.\n' # pylint: disable=line-too-long
     with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text:
         line_list=log_text.readlines()
         assert log_text_line_1 in line_list[0]
@@ -89,7 +89,7 @@ def test_cli_fre_cmor_help_and_infolog():
     assert result.exit_code == 0
     assert Path("TEST_FOO_LOG.log").exists()
 
-    log_text_line_1='[ INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
+    log_text_line_1='[   INFO:                  fre.py:                     fre] fre_file_handler added to base_fre_logger\n' # pylint: disable=line-too-long
     with open( "TEST_FOO_LOG.log", 'r', encoding='utf-8') as log_text:
         line_list=log_text.readlines()
         assert log_text_line_1 in line_list[0]

fre/tests/test_fre_workflow_cli.py

@@ -0,0 +1,103 @@
+"""
+CLI Tests for fre workflow *
+Tests the command-line-interface calls for tools in the fre workflow suite.
+
+Each tool generally gets 3 tests:
+  - fre workflow $tool, checking for exit code 0 (fails if cli isn't configured right)
+  - fre workflow $tool --help, checking for exit code 0 (fails if the code doesn't run)
+  - fre workflow $tool --optionDNE, checking for exit code 2; misuse of command (fails if cli isn't configured
+    right and thinks the tool has a --optionDNE option)
+"""
+import os
+from pathlib import Path
+from click.testing import CliRunner
+from fre import fre
+
+runner = CliRunner()
+
+## fre workflow subtools search for if TMPDIR is set, specifically for fre workflow checkout --target-dir
+# If a value for --target-dir is not passed --> TMPDIR will be used for --target-dir
+# If TMPDIR is not set --> a default location will be used for --target-dir 
+#-- fre workflow
+def test_cli_fre_workflow(monkeypatch, tmp_path):
+    ''' fre workflow '''
+    monkeypatch.setenv("TMPDIR", tmp_path)
+    result = runner.invoke(fre.fre, args=["workflow"])
+    assert result.exit_code == 2
+
+def test_cli_fre_workflow_help(monkeypatch, tmp_path):
+    ''' fre workflow --help '''
+    monkeypatch.setenv("TMPDIR", tmp_path)
+    result = runner.invoke(fre.fre, args=["workflow", "--help"])
+    assert result.exit_code == 0
+
+def test_cli_fre_workflow_opt_dne(monkeypatch, tmp_path):
+    ''' fre workflow optionDNE '''
+    monkeypatch.setenv("TMPDIR", tmp_path)
+    result = runner.invoke(fre.fre, args=["workflow", "optionDNE"])
+    assert result.exit_code == 2
+
+#-- fre workflow checkout
+def test_cli_fre_workflow_checkout(monkeypatch, tmp_path):
+    ''' fre workflow checkout'''
+    monkeypatch.setenv("TMPDIR", tmp_path)
+    result = runner.invoke(fre.fre, args=["workflow", "checkout"])
+    assert result.exit_code == 2
+
+def test_cli_fre_workflow_checkout_help(monkeypatch, tmp_path):
+    ''' fre workflow checkout --help '''
+    monkeypatch.setenv("TMPDIR", tmp_path)
+    result = runner.invoke(fre.fre, args=["workflow", "checkout", "--help"])
+    assert result.exit_code == 0
+
+def test_cli_fre_workflow_checkout_opt_dne(monkeypatch, tmp_path):
+    ''' fre workflow checkout optionDNE '''
+    monkeypatch.setenv("TMPDIR", tmp_path)
+    result = runner.invoke(fre.fre, args=["workflow", "checkout", "optionDNE"])
+    assert result.exit_code == 2
+
+def test_cli_fre_workflow_checkout_target_dir_set(tmp_path):
+    """
+    Test checkout in target directory if --target-dir is explicitly set.
+    """
+    experiment = "c96L65_am5f7b12r1_amip_TESTING"
+    result = runner.invoke(fre.fre, args=["workflow", "checkout",
+                                          "--yamlfile", "fre/workflow/tests/AM5_example/am5.yaml",
+                                          "--experiment", experiment,
+                                          "--application", "pp",
+                                          "--target-dir", tmp_path])
+    assert result.exit_code == 0
+    assert Path(f"{tmp_path}/cylc-src/{experiment}").exists()
+
+def test_cli_fre_workflow_checkout_TMPDIR_set(tmp_path, monkeypatch):
+    """
+    Test checkout if TMPDIR environment variable is set and --target-dir has no
+    specified value
+    """
+    experiment = "c96L65_am5f7b12r1_amip_TESTING"
+    Path(f"{tmp_path}/env_var").mkdir(parents=True)
+    monkeypatch.setenv("TMPDIR", f"{tmp_path}/env_var")
+
+    result = runner.invoke(fre.fre, args=["workflow", "checkout",
+                                          "-y", "fre/workflow/tests/AM5_example/am5.yaml",
+                                          "-e", experiment,
+                                          "-a", "pp"])
+    assert result.exit_code == 0
+    assert Path(f"{os.environ['TMPDIR']}/cylc-src/{experiment}").exists()
+
+#def test_cli_fre_workflow_checkout_default_dir():
+#    """
+#    Test checkout if TMPDIR and --target-dir is not set;
+#    use the default location: ~/.fre
+#    """
+    experiment = "c96L65_am5f7b12r1_amip_TESTING"
+    monkeypatch.setenv("TMPDIR", "")
+
+    result = runner.invoke(fre.fre, args=["workflow", "checkout",
+                                          "-y", "fre/workflow/tests/AM5_example/am5.yaml",
+                                          "-e", experiment,
+                                          "-a", "pp"])
+    #default cylc-src location
+    default_dir = os.path.expanduser("~/.fre")
+    assert result.exit_code == 0
+    assert Path(f"{default_dir}/cylc-src/{experiment}").exists()

fre/workflow/README.md

@@ -0,0 +1,19 @@
+# FRE workflow
+`fre workflow` provides subtools that help to clone, install, and run a workflow from a repository.
+
+## Quickstart
+From the root of the fre-cli repository, run:
+```
+# Checkout/clone the post-processing workflow repository
+fre workflow checkout -y fre/workflow/tests/AM5_example/am5.yaml -e c96L65_am5f7b12r1_amip_TESTING -a pp
+```
+
+## Subtools
+- `fre workflow checkout [options]`
+   - Purpose: Clone the workflow repository/branch, depending on the application passed.
+   - Options:
+        - `-y, --yamlfile [model yaml] (str; required)`
+        - `-e, --experiment [experiment name] (str; required)`
+        - `-a, --application [ run | pp ] (str; required)`
+        - `--target-dir [target directory where workflow will be cloned] (str; optional; default is ~/.fre-workflows`
+        - `--force-checkout (bool; optional)`

fre/workflow/checkout_script.py

@@ -0,0 +1,159 @@
+""" Workflow checkout """
+import os
+import subprocess
+from pathlib import Path
+import logging
+import shutil
+import json
+from jsonschema import validate, SchemaError, ValidationError
+from typing import Optional
+
+import fre.yamltools.combine_yamls_script as cy
+from fre.app.helpers import change_directory
+
+fre_logger = logging.getLogger(__name__)
+
+######VALIDATE#####
+def validate_yaml(yamlfile: dict, application: str):
+    """
+    Validate the format of the yaml file based
+    on the schema.json held in [gfdl_msd_schemas](https://github.com/NOAA-GFDL/gfdl_msd_schemas).
+
+    :param yamlfile: Model, settings, pp, and analysis yaml
+                     information combined into a dictionary
+    :type yamlfile: dict
+    :param application: type of workflow to check out/clone 
+    :type application: string
+    :raises ValueError:
+        - if gfdl_mdf_schema path is not valid
+        - combined yaml is not valid
+        - unclear error in validation
+    """
+    schema_dir = Path(__file__).resolve().parents[1]
+    schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json')
+    fre_logger.info("Using yaml schema '%s'", schema_path)
+    # Load the json schema: .load() (vs .loads()) reads and parses the json in one)
+    try:
+        with open(schema_path,'r', encoding='utf-8') as s:
+            schema = json.load(s)
+    except:
+        fre_logger.error("Schema '%s' is not valid. Contact the FRE team.", schema_path)
+        raise
+
+    # Validate yaml
+    # If the yaml is not valid, the schema validation will raise errors and exit
+    try:
+        validate(instance = yamlfile,schema=schema)
+        fre_logger.info(" ** COMBINED YAML VALID ** ")
+    except SchemaError as exc:
+        raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc
+    except ValidationError as exc:
+        raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc
+    except Exception as exc:
+        raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc
+
+def workflow_checkout(target_dir: str, yamlfile: str = None, experiment: str = None, application: str = None, force_checkout: Optional[bool] = False):
+    """
+    Create a directory and clone the workflow template files from a defined repository.
+
+    :param yamlfile: Model yaml configuration file
+    :type yamlfile: str
+    :param experiment: One of the postprocessing experiment names from the
+                       yaml displayed by fre list exps -y $yamlfile 
+                       (e.g. c96L65_am5f4b4r0_amip), default None
+    :type experiment: str
+    :param application: Which workflow will be used/cloned
+    :type application: str
+    :param target_dir: Target/base directory used for cylc-src/<workflow> creation
+    :type target_dir: str
+    :param force_checkout: re-clone the workflow repository if it exists
+    :type force_checkout: bool
+    :raises OSError: if the checkout script was not able to be created
+    :raises ValueError:
+        - if the repository and/or tag was not defined
+        - if the target directory does not exist or cannot be found
+        - if neither tag nor branch matches the git clone branch arg
+    """
+    # Used in consolidate_yamls function for now
+    platform = None
+    target = None
+
+    if application in ["run", "pp"]:
+        fre_logger.info(" ** Configuring the resolved YAML for the %s **", application)
+        yaml = cy.consolidate_yamls(yamlfile=yamlfile,
+                                    experiment=experiment,
+                                    platform=platform,
+                                    target=target,
+                                    use=application,
+                                    output="config.yaml")
+
+        validate_yaml(yamlfile = yaml, application = application)
+
+        # Reset application for pp to make it discoverable in yaml config
+        if application == "pp":
+            application = "postprocess"
+
+        workflow_info = yaml.get(application).get("workflow")
+
+    repo = workflow_info.get("repository")
+    tag = workflow_info.get("version")
+    fre_logger.info("Defined tag ==> '%s'", tag)
+
+    if None in [repo, tag]:
+        raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}")
+
+    fre_logger.info("(%s):(%s) check out for %s ==> REQUESTED", repo, tag, application)
+
+    # Create src_dir if it does not exist
+    if not Path(target_dir).exists():
+        Path(target_dir).mkdir(parents=True, exist_ok=True)
+
+    # Define cylc-src directory
+    src_dir = f"{target_dir}/cylc-src"
+    # workflow name
+    workflow_name = experiment
+
+    # create workflow in cylc-src
+    try:
+        Path(src_dir).mkdir(parents=True, exist_ok=True)
+    except Exception as exc:
+        raise OSError(
+            f"(checkoutScript) directory {src_dir} wasn't able to be created. exit!") from exc
+
+    if Path(f"{src_dir}/{workflow_name}").is_dir():
+        fre_logger.info(" *** PREVIOUS CHECKOUT FOUND: %s/%s *** ", src_dir, workflow_name)
+        if force_checkout:
+            fre_logger.warning(" *** REMOVING %s/%s *** ", src_dir, workflow_name)
+            shutil.rmtree(f"{src_dir}/{workflow_name}")
+        else:
+            with change_directory(f"{src_dir}/{workflow_name}"):
+                # capture the branch and tag
+                # if either match git_clone_branch_arg, then success. otherwise, fail.
+                current_tag = subprocess.run(["git","describe","--tags"],
+                                             capture_output = True,
+                                             text = True, check = True).stdout.strip()
+                current_branch = subprocess.run(["git", "branch", "--show-current"],
+                                                 capture_output = True,
+                                                 text = True, check = True).stdout.strip()
+
+                if tag in (current_tag, current_branch):
+                    fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", src_dir, workflow_name, tag)
+                else:
+                    fre_logger.error(
+                        "ERROR: Checkout exists ('%s/%s') and does not match '%s'", src_dir, workflow_name, tag)
+                    fre_logger.error(
+                        "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag)
+                    raise ValueError('Neither tag nor branch matches the git clone branch arg')
+    if not Path(f"{src_dir}/{workflow_name}").is_dir():
+        fre_logger.info("Workflow does not exist; will create now")
+        clone_output = subprocess.run( ["git", "clone","--recursive",
+                                        f"--branch={tag}",
+                                        repo, f"{src_dir}/{workflow_name}"],
+                                        capture_output = True, text = True, check = True)
+        fre_logger.debug(clone_output)
+        fre_logger.info("(%s):(%s) check out ==> SUCCESSFUL", repo, tag)
+
+        ## Move combined yaml to cylc-src location
+        current_dir = Path.cwd()
+        shutil.move(Path(f"{current_dir}/config.yaml"), f"{src_dir}/{workflow_name}")
+        fre_logger.info("Combined yaml file moved to %s/%s", src_dir, workflow_name)

fre/workflow/freworkflow.py

@@ -0,0 +1,41 @@
+''' fre workflow click interface for fre workflow subcommands'''
+import os
+import click
+import logging
+fre_logger = logging.getLogger(__name__)
+
+#fre tools
+from . import checkout_script
+#from . import install_script
+#from . import run_script
+
+@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210)))
+def workflow_cli():
+    ''' entry point to fre workflow click commands '''
+
+@workflow_cli.command()
+@click.option("-y", "--yamlfile", type=str,
+              help="Model yaml file",
+              required=True)
+@click.option("-e", "--experiment", type=str,
+              help="Experiment name",
+              required=True)
+@click.option("-a", "--application",
+              type=click.Choice(['run', 'pp']),
+              help="Type of workflow to check out/clone",
+              required=True)
+@click.option("--target-dir",
+              type=str,
+              default=os.path.expanduser("~/.fre-workflows"),
+              help=f"""Target directory for the workflow to be cloned into.
+                    If not defined, a default location of ~/.fre-workflows
+                    will be used""")
+@click.option("--force-checkout",
+              is_flag=True,
+              default=False,
+              help="If the checkout already, exists, remove and clone the desired repo again.")
+def checkout(target_dir, yamlfile, experiment, application, force_checkout):
+    """
+    Checkout/extract fre workflow
+    """
+    checkout_script.workflow_checkout(target_dir, yamlfile, experiment, application, force_checkout)