Logger & Resolver (#14)

* Added more functionality to logger

* Added functionality to get resolver

Resolvers now parse through parent classes looking for resolvers.

Added using_resolver decorator that when decorating a class will add resolvers to the global list based on keyword arguments

Author: willgebhardt

ngcsimlib/compilers/command_compiler.py

@@ -132,7 +132,8 @@ def dynamic_compile(*components, compile_key=None):
     Returns:
         compiled_command, needed_arguments
     """
-    assert compile_key is not None
+    if compile_key is None:
+        critical("Can not compile a command without a compile key")
     return _compile(compile_key, {c.name: c for c in components})
 
 

ngcsimlib/compilers/component_compiler.py

@@ -34,7 +34,7 @@ def parse(component, compile_key):
 
     """
     (pure_fn, output_compartments), (args, parameters, compartments, parse_varnames) = \
-        get_resolver(component.__class__.__name__, compile_key)
+        get_resolver(component.__class__, compile_key)
 
     if parse_varnames:
         args = []

ngcsimlib/compilers/op_compiler.py

@@ -11,6 +11,7 @@
 by the compiled command.
 """
 from ngcsimlib.operations.baseOp import BaseOp
+from ngcsimlib.logger import critical
 
 
 def parse(op):
@@ -28,7 +29,10 @@ def parse(op):
     for s in op.sources:
         if isinstance(s, BaseOp):
             needed_inputs, dest = parse(s)
-            assert dest is None
+            if dest is not None:
+                critical(
+                    "An operation with a destination compartment is being used as a source object for another "
+                    "operation")
             for inp in needed_inputs:
                 if inp not in inputs:
                     inputs.append(inp)

ngcsimlib/context.py

@@ -1,10 +1,11 @@
 from ngcsimlib.utils import make_unique_path, check_attributes, check_serializable, load_from_path
-from ngcsimlib.logger import warn, info
+from ngcsimlib.logger import warn, info, critical
 from ngcsimlib.utils import get_compartment_by_name, \
     get_context, add_context, get_current_path, get_current_context, set_new_context, load_module, is_pre_loaded
 from ngcsimlib import preload_modules
 from ngcsimlib.compilers.command_compiler import dynamic_compile, wrap_command
-import json, os
+import json, os, shutil
+
 
 
 class Context:
@@ -30,11 +31,13 @@ def __new__(cls, name, *args, **kwargs):
         Args:
              name: the name of the context
         """
-        assert len(name) > 0
+        if len(name) == 0:
+            critical
         con = get_context(str(name))
         if con is None:
             return super().__new__(cls)
         else:
+            info(f"Returning already existing context: {name}")
             return con
 
     def __init__(self, name):
@@ -189,7 +192,7 @@ def add_command(self, command, name=None):
         self.commands[name] = command
         self.__setattr__(name, command)
 
-    def save_to_json(self, directory, model_name=None, custom_save=True):
+    def save_to_json(self, directory, model_name=None, custom_save=True, overwrite=False):
         """
         Dumps all the required json files to rebuild the current controller to a specified directory. If there is a
         `save` command present on the controller and custom_save is True, it will run that command as well.
@@ -204,11 +207,26 @@ def save_to_json(self, directory, model_name=None, custom_save=True):
             custom_save: A boolean that if true will attempt to call the `save`
                 command if present on the controller (Default: True)
 
+            overwrite: A boolean for if the saved model should be in a unique folder or if it should overwrite
+            existing folders
+
         Returns:
             a tuple where the first value is the path to the model, and the
                 second is the path to the custom save folder if custom_save is
                 true and None if false
         """
+        if overwrite and os.path.isdir(directory + "/" + model_name):
+            for filename in os.listdir(directory + "/" + model_name):
+                file_path = os.path.join(directory + "/" + model_name, filename)
+                try:
+                    if os.path.isfile(file_path) or os.path.islink(file_path):
+                        os.unlink(file_path)
+                    elif os.path.isdir(file_path):
+                        shutil.rmtree(file_path)
+                except Exception as e:
+                    print('Failed to delete %s. Reason: %s' % (file_path, e))
+            shutil.rmtree(directory + "/" + model_name)
+
         path = make_unique_path(directory, model_name)
 
         with open(path + "/modules.json", "w") as fp:
@@ -359,7 +377,8 @@ def make_commands(self, path_to_commands_file):
                         name=c_name)
                 else:
                     klass = load_from_path(command['class'])
-                    klass(*command['args'], **command['kwargs'], components=self.get_components(*command['components'], unwrap=False),
+                    klass(*command['args'], **command['kwargs'],
+                          components=self.get_components(*command['components'], unwrap=False),
                           command_name=c_name)
 
     def _make_op(self, op_spec):
@@ -421,7 +440,6 @@ def compile_by_key(self, *components, compile_key, name=None):
         self.__setattr__(name, cmd)
         return cmd, args
 
-
     def wrap_and_add_command(self, command, name=None):
         """
         wraps a command and adds it to the context, if no name is provided it will use the command's internal name
@@ -461,7 +479,6 @@ def make_modules(self):
             if klass not in map(lambda x: x["name"], modules[module]["attributes"]):
                 modules[module]["attributes"].append({"name": klass})
 
-
         jCommands = self._json_objects['commands']
         for c_name, c in jCommands.items():
             if c["class"] == "dynamic_compiled":

ngcsimlib/logger.py

@@ -1,30 +1,80 @@
 from ngcsimlib.configManager import get_config
 import logging
 import sys
+from datetime import datetime
 
 
-def _concatArgs(*args, **kwargs):
+def _concatArgs(func):
     """Internal Decorator for concatenating arguments into a single string"""
+    def wrapped(*wargs, sep=" ", end="", **kwargs):
+        msg = sep.join(str(a) for a in wargs) + end
+        return func(msg, **kwargs)
 
-    def inner(func):
-        def wrapped(*wargs, sep=" ", end="", **kwargs):
-            msg = sep.join(str(a) for a in wargs) + end
-            return func(msg, **kwargs)
+    return wrapped
 
-        return wrapped
 
-    return inner
+_ngclogger = logging.getLogger("ngclogger")
 
+_mapped_calls = {}
 
-_ngclogger = logging.getLogger("ngclogger")
+def addLoggingLevel(levelName, levelNum, methodName=None):
+    """
+    Comprehensively adds a new logging level to the `logging` module and the
+    currently configured logging class.
+
+    `levelName` becomes an attribute of the `logging` module with the value
+    `levelNum`. `methodName` becomes a convenience method for both `logging`
+    itself and the class returned by `logging.getLoggerClass()` (usually just
+    `logging.Logger`). If `methodName` is not specified, `levelName.lower()` is
+    used.
+
+    Credit: https://stackoverflow.com/a/35804945
+
+    Args:
+        levelName: The custom level name
+
+        levelNum: The custom level number
+
+        methodName: The name of the method
+    """
+
+
+    if not methodName:
+        methodName = levelName.lower()
+
+    if hasattr(logging, levelName):
+       raise AttributeError('{} already defined in logging module'.format(levelName))
+    if hasattr(logging, methodName):
+       raise AttributeError('{} already defined in logging module'.format(methodName))
+    if hasattr(logging.getLoggerClass(), methodName):
+       raise AttributeError('{} already defined in logger class'.format(methodName))
+
+    def logForLevel(self, message, *args, **kwargs):
+        if self.isEnabledFor(levelNum):
+            self._log(levelNum, message, args, **kwargs)
+    def logToRoot(message, *args, **kwargs):
+        logging.log(levelNum, message, *args, **kwargs)
+
+    logging.addLevelName(levelNum, levelName)
+    setattr(logging, levelName, levelNum)
+    setattr(logging.getLoggerClass(), methodName, logForLevel)
+    setattr(logging, methodName, logToRoot)
 
+    _mapped_calls[levelNum] = getattr(_ngclogger, methodName)
+    _mapped_calls[levelName] = getattr(_ngclogger, methodName)
 
 def init_logging():
     loggingConfig = get_config("logging")
     if loggingConfig is None:
         loggingConfig = {"logging_file": None,
                          "logging_level": logging.WARNING,
-                         "hide_console": False}
+                         "hide_console": False,
+                         "custom_levels": {"ANALYSIS", 25}}
+
+    if loggingConfig.get("custom_levels", None) is not None:
+        for level_name, level_num in loggingConfig.get("custom_levels", {}).items():
+            addLoggingLevel(level_name.upper(), level_num)
+
 
     if isinstance(loggingConfig.get("logging_level", None), str):
         loggingConfig["logging_level"] = \
@@ -40,12 +90,16 @@ def init_logging():
         _ngclogger.addHandler(err_stream_handler)
 
     if loggingConfig.get("logging_file", None) is not None:
+        with open(loggingConfig.get("logging_file", None), "a+") as fp:
+            fp.write(f"~~~~~~~~~~~~~~~~~~~~~~~~~~~\n"
+                     f"New Log {f'{datetime.utcnow():%m/%d/%Y %H:%M:%S}'}"
+                     f"\n~~~~~~~~~~~~~~~~~~~~~~~~~~~\n")
         file_handler = logging.FileHandler(filename=loggingConfig.get("logging_file", None))
         file_handler.setFormatter(formatter)
         _ngclogger.addHandler(file_handler)
 
 
-@_concatArgs()
+@_concatArgs
 def warn(msg):
     """
     Logs a warning message
@@ -57,7 +111,7 @@ def warn(msg):
     _ngclogger.warning(msg)
 
 
-@_concatArgs()
+@_concatArgs
 def error(msg):
     """
     Logs an error message
@@ -70,7 +124,7 @@ def error(msg):
     raise RuntimeError(msg)
 
 
-@_concatArgs()
+@_concatArgs
 def critical(msg):
     """
     Logs a critical message
@@ -83,7 +137,7 @@ def critical(msg):
     raise RuntimeError(msg)
 
 
-@_concatArgs()
+@_concatArgs
 def info(msg):
     """
     Logs an info message
@@ -93,3 +147,29 @@ def info(msg):
         msg: message to log
     """
     _ngclogger.info(msg)
+
+
+@_concatArgs
+def debug(msg):
+    """
+    Logs a debug message
+    This is decorated to have the same functionality of python's print argument concatenation
+
+    Args:
+        msg: message to log
+    """
+    _ngclogger.debug(msg)
+
+
+@_concatArgs
+def custom_log(msg, logging_level=None):
+    if isinstance(logging_level, str):
+        logging_level = logging_level.upper()
+
+    if logging_level is None:
+        warn("No logging level passed into message")
+    elif logging_level not in _mapped_calls.keys():
+        warn("Attempted to log to undefined level", logging_level)
+    else:
+        _mapped_calls[logging_level](msg)
+

ngcsimlib/utils.py

@@ -3,7 +3,7 @@
 """
 import sys, uuid, os, json
 from importlib import import_module
-from ngcsimlib.logger import info
+from ngcsimlib.logger import info, critical, debug
 
 ## Globally tracking all the modules, and attributes have been dynamically loaded
 _Loaded_Attributes = {}
@@ -301,10 +301,30 @@ def get_compartment_by_name(context, name):
 __resolver_meta_data = {}
 
 
-def get_resolver(class_name, resolver_key):
+def get_resolver(klass, resolver_key, root=None):
     """
     A helper method for searching through the resolver list
     """
+    class_name = klass.__name__
+
+    if class_name + "/" + resolver_key not in __component_resolvers.keys():
+        parent_classes = klass.__bases__
+        if len(parent_classes) == 0:
+            return None, None
+
+        resolver = None
+        for parent in parent_classes:
+            resolver, meta = get_resolver(parent, resolver_key, root=klass if root is None else root)
+            if resolver is not None:
+                return resolver, meta
+
+        if resolver is None and root is None:
+            critical(class_name, "has no resolver for", resolver_key)
+        if resolver is None:
+            return None, None
+
+    if root is not None:
+        debug(f"{root.__name__} is using the resolver from {class_name} for resolving key \"{resolver_key}\"")
     return __component_resolvers[class_name + "/" + resolver_key], __resolver_meta_data[class_name + "/" + resolver_key]
 
 
@@ -321,6 +341,24 @@ def add_resolver_meta(class_name, resolver_key, data):
     """
     __resolver_meta_data[class_name + "/" + resolver_key] = data
 
+def using_resolver(**kwargs):
+    """
+    A decorator for linking resolvers defined in other classes to this class.
+    the keyword arguments are compile_key=class_to_inherit_resolver_from. This
+    will add the resolver directly to the class and thus will get used before
+    any resolvers in parent classes.
+
+    Args:
+        **kwargs:  any number or compile_key=class_to_inherit_resolver_from
+    """
+    def _klass_wrapper(cls):
+        klass_name = cls.__name__
+        for key, value in kwargs.items():
+            resolver, data = get_resolver(value, key)
+            add_component_resolver(klass_name, key, resolver)
+            add_resolver_meta(klass_name, key, data)
+        return cls
+    return _klass_wrapper
 
 ## Contexts
 __current_context = ""