Merge pull request #17 from pedrocamargo/pedro/logging

- Uses logging rather than print statements
- Re-organizes validation module

Author: e-lo

gmnspy/__init__.py

@@ -1,5 +1,3 @@
-from .conversion import *
 from .in_out import *
 from .schema import *
-from .utils import *
-from .validate import *
+from .utils import list_to_md_table, logger

gmnspy/conversion.py

@@ -2,13 +2,10 @@
 
 import pandas as pd
 
+from gmnspy.utils import logger
+from gmnspy.validation import update_resources_based_on_existance
+from gmnspy.validation import validate_foreign_keys, check_required_files, apply_schema_to_df
 from .schema import read_config
-from .validate import (
-    apply_schema_to_df,
-    confirm_required_files,
-    update_resources_based_on_existance,
-    validate_foreign_keys,
-)
 
 spec_folder = join(dirname(realpath(__file__)), "spec")
 
@@ -31,12 +28,12 @@ def read_gmns_csv(filename: str, validate: bool = True, schema_file: str = None)
     if validate:
         apply_schema_to_df(df, schema_file=schema_file, originating_file=filename)
     else:
-        print("not validating {}".format(filename))
+        logger.info(f"not validating {filename}")
 
     return df
 
 
-def read_gmns_network(data_directory: str, config: str = join(spec_folder, "gmns.spec.json")) -> dict:
+def read_gmns_network(data_directory: str, config: str = None, raise_error=False) -> dict:
     """
     Reads each GMNS file as specified in the config and validates it to
     their specified schema including foreign keys between the tables.
@@ -47,6 +44,8 @@ def read_gmns_network(data_directory: str, config: str = join(spec_folder, "gmns
             specifying the "name", "path", and "schema" for each GMNS table as
             well as a boolean value for "required". If not specified, assumes
             it is in a subdirectory "spec/gmns.spec.json"
+        raise_error: Raises error if missing folder
+
             Example:
             ::
                 {
@@ -67,14 +66,14 @@ def read_gmns_network(data_directory: str, config: str = join(spec_folder, "gmns
     returns: a dictionary mapping the name of each GMNS table to a
         validated dataframe.
     """
+
+    config = config or join(spec_folder, "gmns.spec.json")
     gmns_net_d = {}
 
-    if config:
-        resource_df = read_config(config, data_dir=data_directory)
-    else:
-        resource_df = 1
+    resource_df = read_config(config, data_dir=data_directory) if config else 1
+
     # check required files exist,
-    confirm_required_files(resource_df)
+    check_required_files(resource_df, raise_error)
 
     # update resource dictionary based on what files are in the directory
     resource_df = update_resources_based_on_existance(resource_df)
@@ -85,6 +84,6 @@ def read_gmns_network(data_directory: str, config: str = join(spec_folder, "gmns
         gmns_net_d[row["name"]] = read_gmns_csv(row["fullpath"])
 
     # validate foreign keys
-    validate_foreign_keys(gmns_net_d, resource_df)
+    validate_foreign_keys(gmns_net_d, resource_df, raise_error)
 
     return gmns_net_d

gmnspy/in_out.py

@@ -4,7 +4,7 @@
 
 import pandas as pd
 
-from .utils import list_to_md_table
+from .utils import list_to_md_table, logger
 
 SCHEMA_TO_PANDAS_TYPES = {
     "integer": "int64",
@@ -73,13 +73,11 @@ def read_config(config_file: str, data_dir: str = "", schema_dir: str = "") -> p
     with open(config_file, "r", encoding="utf-8") as f:
         config = json.load(f)
     ## todo validate config
-    # resource_dict = {i["name"]: i for i in config["resources"]}
-    # print(config["resources"])
 
     resource_df = pd.DataFrame(config["resources"])
     resource_df["required"].fillna(False, inplace=True)
 
-    print(resource_df)
+    logger.info(str(resource_df))
 
     # Add full paths to data files
     if not data_dir:
@@ -90,27 +88,27 @@ def read_config(config_file: str, data_dir: str = "", schema_dir: str = "") -> p
     if not schema_dir:
         schema_dir = dirname(config_file)
     resource_df["fullpath_schema"] = resource_df["schema"].apply(lambda x: join(schema_dir, x))
-    print(resource_df)
+    logger.info(str(resource_df))
 
     resource_df.set_index("name", drop=False, inplace=True)
     return resource_df
 
 
 def document_schema(base_path: str = None, out_path: str = None):
     """ """
-    print("DOCUMENTING SCHEMA")
+    logger.info("DOCUMENTING SCHEMA")
 
     base_path = base_path or join(dirname(realpath(__file__)), "spec")
     out_path = out_path or join(base_path, "docs")
-    print("Looking for specs in: {}".format(base_path))
+    logger.info(f"Looking for specs in: {base_path}")
 
     # Create markdown with a table for each schema file
     file_schema_markdown = ""
     schema_files = glob.glob(join(base_path, "**/*.schema.json"), recursive=True)
-    print("files: {}".format(schema_files))
+    logger.info(f"files: {schema_files}")
 
     for s in schema_files:
-        print("Documenting Schema: {}".format(s))
+        logger.info(f"Documenting Schema: {s}")
         spec_name = s.split("/")[-1].split(".")[0]
         schema = read_schema(s)
         file_schema_markdown += "\n\n## {}\n".format(spec_name)

gmnspy/schema.py

@@ -0,0 +1,2 @@
+from .set_logger import logger
+from .list_to_md_table import list_to_md_table

gmnspy/utils/__init__.py

@@ -0,0 +1,24 @@
+import logging
+from os.path import join
+from tempfile import gettempdir
+
+
+def build_logger():
+    logger = logging.getLogger("gmnspy")
+    handler_name = "gmnspy"
+
+    if len([h for h in logger.handlers if h.name and handler_name == h.name]) > 0:
+        return logger
+
+    log_path = join(gettempdir(), "gmnspy.log")
+    FORMATTER = logging.Formatter("%(asctime)s;%(levelname)s ; %(message)s", datefmt="%H:%M:%S:")
+    ch = logging.FileHandler(log_path)
+    ch.setFormatter(FORMATTER)
+    ch.set_name(handler_name)
+    ch.setLevel(logging.INFO)
+    logger.addHandler(ch)
+
+    return logger
+
+
+logger = build_logger()