Adding new pyproject file + code refresh + docs

Author: Ciheim Brown

MANIFEST.in

@@ -7,31 +7,13 @@
 import os
 from pathlib import Path
 import logging
-from catalogbuilder.tests.compval import compval as cv
+from catalogbuilder.scripts.compval import compval as cv
+from catalogbuilder.intakebuilder import gfdlcrawler, CSVwriter, configparser, getinfo
 
 logger = logging.getLogger('local')
 logger.setLevel(logging.INFO)
 logging.basicConfig(stream=sys.stdout)
 
-try:
-    from catalogbuilder.intakebuilder import gfdlcrawler, CSVwriter, configparser, getinfo
-except ModuleNotFoundError:
-    logger.warning("The module intakebuilder is not installed. Do you have intakebuilder in your sys.path or have you activated the conda environment with the intakebuilder package in it? ")
-    logger.warning("Attempting again with adjusted sys.path ")
-    try:
-        sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-    except:
-        logger.error("Unable to adjust sys.path")
-    #print(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-    try:
-
-        from intakebuilder import gfdlcrawler, CSVwriter, builderconfig, configparser,getinfo
-        logger.info(gfdlcrawler.__file__)
-
-    except ModuleNotFoundError:
-        logger.error("The module 'intakebuilder' is still not installed. Do you have intakebuilder in your sys.path or have you activated the conda environment with the intakebuilder package in it?")
-        raise ImportError("The module 'intakebuilder' is still not installed. Do you have intakebuilder in your sys.path or have you activated the conda environment with the intakebuilder package in it?")
-
 package_dir = os.path.dirname(os.path.abspath(__file__))
 #template_path = os.path.join(package_dir, '../cats/gfdl_template.json')
 
@@ -45,16 +27,16 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm=
     if strict:
         logger.warning("!!!!! STRICT MODE IS ACTIVE. CATALOG GENERATION WILL FAIL IF ERRORS ARE FOUND !!!!!\n")
         time.sleep(10)
+
     configyaml = None
-    if (config is not None):
+    if config is not None:
         configyaml = configparser.Config(config,logger)
-        if(input_path is None):
+        if input_path is None:
             input_path = configyaml.input_path
-        if(output_path is None):
+        if output_path is None:
             output_path = configyaml.output_path
     else:
             # If user does not pass a config, we will use the default config with the same format to avoid special cases
-        #
         try:
             pkg = importlib_resources.files("catalogbuilder.scripts")
             config = pkg / "configs" / "config.yaml"
@@ -66,25 +48,29 @@ def create_catalog(input_path=None, output_path=None, config=None, filter_realm=
             except:
                 raise FileNotFoundError("Can't locate or read config, check --config ")
         configyaml = configparser.Config(config,logger)
-        if(input_path is None):
+
+        if input_path is None:
             input_path = configyaml.input_path
-        if(output_path is None):
+        if output_path is None:
             output_path = configyaml.output_path
-    if((input_path is None) or (output_path is None)):
+    if input_path is None or output_path is None:
         logger.error("Missing: input_path or output_path. Pass it in the config yaml or as command-line option")
         raise TypeError("Missing: input_path or output_path. Pass it in the config yaml or as command-line option")
+    
     if config is None or not configyaml.schema:
         logger.info("Default schema: catalogbuilder/cats/gfdl_template.json")
         template_path = os.path.join(package_dir, '../cats/gfdl_template.json')
     else:
         template_path = configyaml.schema
         logger.info("Using schema from config file", template_path)
+
     if not os.path.exists(input_path):
         logger.error("Input path does not exist. Adjust configuration.")
         raise FileNotFoundError("Input path does not exist. Adjust configuration.")
     if not os.path.exists(Path(output_path).parent.absolute()):
         logger.error("Output path parent directory does not exist. Adjust configuration.")
         raise ValueError("Output path parent directory does not exist. Adjust configuration.")
+
     logger.info("input path: "+ input_path)
     logger.info("output path: "+ output_path)
     project_dir = input_path

catalogbuilder/tests/compval.py catalogbuilder/scripts/compval.pycatalogbuilder/tests/compval.py renamed to catalogbuilder/scripts/compval.py

@@ -0,0 +1,80 @@
+==============
+For developers
+==============
+
+The Catalog Builder team welcomes all contributions. If you would like to help develop the package, please follow the steps outlined below.
+
+
+How to contribute
+=================
+
+Set up a clean environment
+--------------------------
+
+First, create a new environment for your Catalog Builder development work. The recommended approach is to use a `python virtual environment (venv) <https://docs.python.org/3/library/venv.html>`_. A conda environment will also work fine if such is desired.
+
+.. code-block:: console
+
+   python3 -m venv /path/to/new/virtual/environment
+
+Then, activate the environment by sourcing the activation script. The command varies by operating system and shell:
+
+* **Linux/macOS (bash/zsh):**
+
+.. code-block:: console
+
+   source /path/to/new/virtual/environment/bin/activate
+
+* **Linux/macOS (csh):**
+
+.. code-block:: console
+
+   source /path/to/new/virtual/environment/bin/activate.csh
+
+* **Linux/macOS (fish):**
+
+.. code-block:: console
+
+   source /path/to/new/virtual/environment/bin/activate.fish
+
+* **Linux/macOS (pwsh):**
+
+.. code-block:: console
+
+   /path/to/new/virtual/environment/bin/activate.ps1
+
+* **Windows (Command Prompt):**
+
+.. code-block:: console
+
+   \path\to\new\virtual\environment\Scripts\activate.bat
+
+* **Windows (PowerShell):**
+
+.. code-block:: console
+
+   \path\to\new\virtual\environment\Scripts\Activate.ps1
+
+Clone the Catalog Builder source code
+-------------------------------------
+
+Clone the `Github repository <https://github.com/NOAA-GFDL/CatalogBuilder>`_ using ssh:
+
+.. code-block:: console
+
+   git clone git@github.com:NOAA-GFDL/CatalogBuilder.git
+
+or https:
+
+.. code-block:: console
+
+   git clone https://github.com/NOAA-GFDL/CatalogBuilder.git
+
+Install the package
+-------------------
+
+It is recommended that developers install an `editable <https://setuptools.pypa.io/en/latest/userguide/development_mode.html>`_ Catalog Builder package. This makes development simple as any local changes will immediately be testable. From the root of the repository, run:
+
+.. code-block:: console
+
+   pip install -e .

catalogbuilder/scripts/gen_intake_gfdl.py

@@ -1,90 +1,63 @@
+========================
 Generating data catalogs
 ========================
 
-There are a few ways to use the catalog builder.
+There are a few ways to use the catalog builder. This page contains instructions to help you start using the tool.
 
 Installation
-------------
+============
 
-Recommended approach: Install as a `conda package <https://anaconda.org/NOAA-GFDL/catalogbuilder>`_
+You will need to install the Catalog Builder package to begin.
 
-.. code-block:: console
+Cloning the repository
+----------------------
 
-  conda install catalogbuilder -c noaa-gfdl
+The current recommended approach for installing the catalog builder is to install the tool as a pip package. You'll need to first clone the `github repository <https://github.com/NOAA-GFDL/CatalogBuilder>`_:
 
-Alternatively, you may clone the `git repository <https://github.com/NOAA-GFDL/CatalogBuilder>`_
-and create your conda environment using the `environment.yml <https://github.com/NOAA-GFDL/CatalogBuilder/blob/main/environment.yml>`_ in the git repository. 
+**With ssh**
 
 .. code-block:: console
 
-   git clone https://github.com/NOAA-GFDL/CatalogBuilder
-
-   conda env create -f environment_intake.yml 
-
-Expected output
----------------
-
-A JSON catalog specification file and a CSV catalog in the specfied output directory with the specified name. 
-
-Using conda package
--------------------
+   git clone git@github.com:NOAA-GFDL/CatalogBuilder.git
 
-**1. Install the package using conda:** 
+**With https**
 
 .. code-block:: console
 
-  conda install catalogbuilder -c noaa-gfdl
-
-If you're trying these steps from GFDL, likely that you may need to do additional things to get it to work. See below
-  
-Add these to your ~/.condarc file 
-
-whitelist_channels:
-  - noaa-gfdl
-  - conda-forge
-  - anaconda
-channels:
-  - noaa-gfdl
-  - conda-forge
-  - anaconda
-
-(and try: conda config --add channels noaa-gfdl conda config --append channels conda-forge)
-
-If you encounter issues "ChecksumMismatchError: Conda detected a mismatch between the expected.." , do the following:
+  https://github.com/NOAA-GFDL/CatalogBuilder.git
 
-conda config --add pkgs_dirs /local2/home/conda/pkgs
-conda config --add envs_dirs /local2/home/conda/envs
+Installing the package
+----------------------
 
-**2. Add conda environment's site packages to PATH**
-
-   See example below.
+Now that you have a local copy of the source code, you are able to install the package. From the root of the repository, run:
 
 .. code-block:: console
 
-   setenv PATH ${PATH}:${CONDA_PREFIX}/lib/python3.1/site-packages/scripts/
-
-**3. Call the builder** 
+   pip install .
 
-Catalogs are generated by the following command:  *gen_intake_gfdl.py <INPUT_PATH> <OUTPUT_PATH>*
+.. note::
+   This installation method is expected to change slightly when the package is uploaded to the Python Package index.
 
-Output path argumment should end with the desired output filename WITHOUT a file ending. See example below.
+Alternative approach: Install as a `conda package <https://anaconda.org/NOAA-GFDL/catalogbuilder>`_:
 
 .. code-block:: console
 
- gen_intake_gfdl.py /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp $HOME/catalog
+  conda install catalogbuilder -c noaa-gfdl
 
-This would create a catalog.csv and catalog.json in the user's home directory.
+Configuration
+=============
 
-.. image:: _static/ezgif-4-786144c287.gif
- :width: 1000px
- :alt: Catalog generation demonstration
+A template/configuration file is used for all catalog generation.
 
-See `Flags`_ here.
+What is a catalog template?
+---------------------------
+
+A catalog template is a YAML file defining headerlist, output path template, output file template, and input/output paths.
 
-Using a configuration file
---------------------------
+Using a custom template
+-----------------------
 
-We recommend the use of a configuration file to provide input to the catalog builder. This is necessary and useful if you want to work with datasets and directories that are *not quite* GFDL post-processed directory oriented.
+A default configuration is used for catalog generation unless a custom configuration is provided. We recommend the use of a custom configuration file if you want to work with datasets and directories that are *not quite* GFDL post-processed directory oriented. Configs must be passed to the builder using the ``--config flag``. See `Flags`_ here.
 
 `Here <https://github.com/NOAA-GFDL/CatalogBuilder/blob/main/catalogbuilder/tests/config-cfname.yaml>`_ is an example configuration file.
 
@@ -106,9 +79,12 @@ with the ESM collection specification standards and the appropriate workflows.
  #Directory structure information
  output_path_template = ['NA','NA','source_id','NA','experiment_id','platform','custom_pp','realm','cell_methods','frequency','chunk_freq']
 
-For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp
-the output_path_template is set as above. We have NA in those values that do not match up with any of the expected headerlist (CSV columns), otherwise we
-simply specify the associated header name in the appropriate place. E.g. The third directory in the PP path example above is the model (source_id), so the third list value in output_path_template is set to 'source_id'. We make sure this is a valid value in headerlist as well. The fourth directory is am5f3b1r0 which does not map to an existing header value. So we simply NA in output_path_template for the fourth value. We have NA in values that do not match up with any of the expected headerlist (CSV columns), otherwise we simply specify the associated header name in the appropriate place. E.g. The third directory in the PP path example above is the model (source_id), so the third list value in output_path_template is set to 'source_id'. We make sure this is a valid value in headerlist as well. #The fourth directory is am5f3b1r0 which does not map to an existing header value. So we simply set NA in output_path_template for the fourth value.
+For a directory structure like /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp the output_path_template is set as above. 
+
+We have NA in those values that do not match up with any of the expected headerlist (CSV columns), otherwise we
+simply specify the associated header name in the appropriate place. E.g. The third directory in the PP path example above is the model (source_id), so the third list value in output_path_template is set to 'source_id'. We make sure this is a valid value in headerlist as well. The fourth directory is am5f3b1r0 which does not map to an existing header value. So we simply add NA in output_path_template for the fourth value. 
+
+We have NA in values that do not match up with any of the expected headerlist (CSV columns), otherwise we simply specify the associated header name in the appropriate place. E.g. The third directory in the PP path example above is the model (source_id), so the third list value in output_path_template is set to 'source_id'. We make sure this is a valid value in headerlist as well.
 
 .. code-block:: yaml
 
@@ -121,14 +97,32 @@ simply specify the associated header name in the appropriate place. E.g. The thi
   input_path:  "/archive/am5/am5/am5f7b10r0/c96L65_am5f7b10r0_amip/gfdl.ncrc5-deploy-prod-openmp/pp/"
   output_path: "/home/a1r/github/noaa-gfdl/catalogs/c96L65_am5f7b10r0_amip" # ENTER NAME OF THE CSV AND JSON, THE SUFFIX ALONE. This can  be an absolute or a relative path
 
-Template
---------
+Creating a data catalog
+=======================
+
+Using the installed package
+---------------------------
+
+Catalogs are generated by the following command:  *gen_intake_gfdl.py <INPUT_PATH> <OUTPUT_PATH>*
+
+Output path argumment should end with the desired output filename WITHOUT a file ending. See example below.
+
+.. code-block:: console
+
+ gen_intake_gfdl.py /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp $HOME/catalog
+
+This would create a catalog.csv and catalog.json in the user's home directory.
+
+.. image:: _static/ezgif-4-786144c287.gif
+ :width: 1000px
+ :alt: Catalog generation demonstration
 
-All data catalogs are generated using a template file. This file defines headerlist, output path template, output file template, and input/output paths.
+See `Flags`_ here.
 
 From a Python script
 ---------------------
 Do you have a python script or a notebook where you could also include steps to generate a data catalog? 
+
 See example `here <https://github.com/NOAA-GFDL/CatalogBuilder/blob/main/catalogbuilder/scripts/gen_intake_gfdl_runner_config.py>`_
 
 Here is another example *with a custom configuration*:
@@ -211,17 +205,10 @@ Refer to this `notebook <https://github.com/aradhakrishnanGFDL/canopy-cats/blob/
 .. image:: _static/catalog_generation.png
   :alt: Screenshot of a notebook showing catalog generation
 
-
 Using FRE-CLI (GFDL only)
 -------------------------
 
-**1. Activate conda environment**
-
-.. code-block:: console
-
- conda activate /nbhome/fms/conda/envs/fre-cli
-
-**2. Call the builder**
+Follow the `fre-cli setup documentation <https://noaa-gfdl.readthedocs.io/projects/fre-cli/en/latest/setup.html>`_ to gain access to fre-cli.
 
 Catalogs are generated by the following command: *fre catalog buildcatalog <INPUT_PATH> <OUTPUT_PATH>*
 
@@ -234,13 +221,18 @@ Catalogs are generated by the following command: *fre catalog buildcatalog <INPU
 
 See `Flags`_ here.
 
-See `Fre-CLI Documentation here <https://noaa-gfdl.github.io/fre-cli/>`_
+See `Fre-CLI Documentation here <https://noaa-gfdl.readthedocs.io/projects/fre-cli/en/latest/>`_
+
+Expected output
+---------------
 
+The catalog builder tool generates a JSON catalog specification file and a CSV catalog in the specfied output directory with the specified name.
 
-Arguments/Options
-_____
+Arguments and Options
+=====================
 
-**Input/Output paths can be passed directly to catalog builder tool through calling command**
+Arguments
+---------
 
 All methods of catalog builder generation support direct input/output path passing.
 
@@ -249,6 +241,8 @@ Input path must be the 1st argument. Output path must be the 2nd.
 Ex. gen_intake_gfdl.py /archive/Some.User/input-path ./output_path
 
 
+Flags
+-----
 .. Reference `Flags`_.
 
 - --config - Allows for catalogs to be generated with a custom configuration. Requires path to YAML configuration file. (Ex. "--config custom_config.yaml")