Json work (#12)

* Move operations into their own directory

* Fix kwarg problem in wrapping commands

* preload_updates

Now saves a modules.json file with saved contexts
Will automatically use the modules file in the save model if one is not provided

* Added null option

If module_path is null, will skip loading modules.json

Author: willgebhardt

ngcsimlib/__init__.py

@@ -6,25 +6,31 @@
 from types import SimpleNamespace
 from importlib import import_module
 from ngcsimlib.configManager import init_config, get_config
-
+from ngcsimlib.logger import warn
 from pkg_resources import get_distribution
 
 __version__ = get_distribution('ngcsimlib').version  ## set software version
 
 
 ###### Preload Modules
-def preload_modules():
-    module_config = get_config("modules")
-    if module_config is None:
-        module_path = "json_files/modules.json"
+def preload_modules(path=None):
+    if path is None:
+        module_config = get_config("modules")
+        if module_config is None:
+            module_path = "json_files/modules.json"
+        else:
+            module_path = module_config.get("module_path", "json_files/modules.json")
+
+        if module_path is None:
+            return
+
+        if not os.path.isfile(module_path):
+            warn("Missing file to preload modules from. Attempted to locate file at \"" + str(module_path) +
+                          "\". No modules will be preloaded. "
+                          "\nSee https://ngc-learn.readthedocs.io/en/latest/tutorials/model_basics/json_modules.html for additional information")
+            return
     else:
-        module_path = module_config.get("module_path", "json_files/modules.json")
-
-    if not os.path.isfile(module_path):
-        warnings.warn("\nMissing file to preload modules from. Attempted to locate file at \"" + str(module_path) +
-                      "\". No modules will be preloaded. "
-                      "\nSee https://ngc-learn.readthedocs.io/en/latest/tutorials/model_basics/json_modules.html for additional information")
-        return
+        module_path = path
 
     with open(module_path, 'r') as file:
         modules = json.load(file, object_hook=lambda d: SimpleNamespace(**d))
@@ -42,6 +48,7 @@ def preload_modules():
                 for keyword in attribute.keywords:
                     utils._Loaded_Attributes[keyword] = atr
 
+    utils.set_loaded(True)
 
 ###### Initialize Config
 def configure():
@@ -59,7 +66,7 @@ def configure():
         config_path = "json_files/config.json"
 
     if not os.path.isfile(config_path):
-        warnings.warn("\nMissing configuration file. Attempted to locate file at \"" + str(config_path) +
+        warn("Missing configuration file. Attempted to locate file at \"" + str(config_path) +
                       "\". Default Config will be used. "
                       "\nSee https://ngc-learn.readthedocs.io/en/latest/tutorials/model_basics/configuration.html for "
                       "additional information")

ngcsimlib/commands/clamp.py

@@ -41,7 +41,7 @@ def __init__(self, components=None, compartment=None, clamp_name=None,
         for name, component in self.components.items():
             mapped = hasattr(component, self.compartment)
             if mapped:
-                if Compartment.is_compartment(getattr(compartment, self.compartment)):
+                if Compartment.is_compartment(getattr(component, self.compartment)):
                     continue
 
             error(self.name, " is attempting to initialize clamp to non-existent compartment \"",

ngcsimlib/context.py

@@ -1,7 +1,8 @@
 from ngcsimlib.utils import make_unique_path, check_attributes, check_serializable, load_from_path
 from ngcsimlib.logger import warn, info
 from ngcsimlib.utils import get_compartment_by_name, \
-    get_context, add_context, get_current_path, get_current_context, set_new_context
+    get_context, add_context, get_current_path, get_current_context, set_new_context, load_module, is_pre_loaded
+from ngcsimlib import preload_modules
 from ngcsimlib.compilers.command_compiler import dynamic_compile, wrap_command
 import json, os
 
@@ -75,13 +76,15 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         """
         set_new_context(self._last_context)
 
-    def get_components(self, *component_names):
+    def get_components(self, *component_names, unwrap=True):
         """
         Gets all the components by name in a context
 
         Args:
             component_names: an arbitrary list of component names to get
 
+            unwrap: return just the component not a list of length 1 if only a single component is retrieved
+
         Returns:
              either a list of components or a single component depending on the number of components being retrieved
         """
@@ -93,7 +96,7 @@ def get_components(self, *component_names):
                 _components.append(self.components[a])
             else:
                 warn(f"Could not fine a component with the name \"{a}\" in the context")
-        return _components if len(component_names) > 1 else _components[0]
+        return _components if len(component_names) > 1 or not unwrap else _components[0]
 
     def register_op(self, op):
         """
@@ -119,7 +122,7 @@ def register_command(self, klass, *args, components=None, command_name=None, **k
 
             kwargs: the keyword arguments passed into the command
         """
-        _components = [components.name for components in components]
+        _components = [component.name for component in components]
         self._json_objects['commands'][command_name] = {"class": klass, "components": _components, "args": args,
                                                         "kwargs": kwargs}
 
@@ -208,6 +211,9 @@ def save_to_json(self, directory, model_name=None, custom_save=True):
         """
         path = make_unique_path(directory, model_name)
 
+        with open(path + "/modules.json", "w") as fp:
+            json.dump(self.make_modules(), fp, indent=4)
+
         with open(path + "/ops.json", 'w') as fp:
             json.dump(self._json_objects['ops'], fp, indent=4)
 
@@ -278,6 +284,11 @@ def load_from_dir(self, directory, custom_folder="/custom"):
             custom_folder: The name of the custom data folder for building
                 components. (Default: `/custom`)
         """
+
+        if os.path.isfile(directory + "/modules.json") and not is_pre_loaded():
+            info("No modules file loaded, loading from model directory")
+            preload_modules(path=directory + "/modules.json")
+
         self.make_components(directory + "/components.json", directory + custom_folder)
         self.make_ops(directory + "/ops.json")
         self.make_commands(directory + "/commands.json")
@@ -345,14 +356,15 @@ def make_commands(self, path_to_commands_file):
                 if command['class'] == "dynamic_compiled":
                     if len(command['components']) > 1:
                         self.compile_by_key(
-                            *self.get_components(*command['components']), compile_key=command['compile_key'], name=c_name)
+                            *self.get_components(*command['components'], unwrap=False), compile_key=command['compile_key'],
+                            name=c_name)
                     else:
                         self.compile_by_key(
-                            self.get_components(*command['components']), compile_key=command['compile_key'],
+                            self.get_components(*command['components'], unwrap=False), compile_key=command['compile_key'],
                             name=c_name)
                 else:
                     klass = load_from_path(command['class'])
-                    klass(*command['args'], **command['kwargs'], components=self.get_components(*command['components']),
+                    klass(*command['args'], **command['kwargs'], components=self.get_components(*command['components'], unwrap=False),
                           command_name=c_name)
 
     def _make_op(self, op_spec):
@@ -425,3 +437,54 @@ def wrap_and_add_command(self, command, name=None):
             name: The name of the command (default: None)
         """
         self.add_command(wrap_command(command), name=name)
+
+    def make_modules(self):
+        modules = {}
+        jComponents = self._json_objects['components']
+        for c_path, c in jComponents.items():
+            mod = load_module(c["class"]).__name__
+
+            module = ".".join(mod.split(".")[:-1])
+            klass = c["class"]
+
+            if module not in modules.keys():
+                modules[module] = {"attributes": []}
+
+            if klass not in map(lambda x: x["name"], modules[module]["attributes"]):
+                modules[module]["attributes"].append({"name": klass})
+
+        jOperators = self._json_objects['ops']
+        for o in jOperators:
+            mod = load_module(o["class"]).__name__
+
+            module = ".".join(mod.split(".")[:-1])
+            klass = o["class"]
+
+            if module not in modules.keys():
+                modules[module] = {"attributes": []}
+
+            if klass not in map(lambda x: x["name"], modules[module]["attributes"]):
+                modules[module]["attributes"].append({"name": klass})
+
+
+        jCommands = self._json_objects['commands']
+        for c_name, c in jCommands.items():
+            if c["class"] == "dynamic_compiled":
+                continue
+
+            mod = load_module(c["class"]).__name__
+
+            module = ".".join(mod.split(".")[:-1])
+            klass = c["class"]
+
+            if module not in modules.keys():
+                modules[module] = {"attributes": []}
+
+            if klass not in map(lambda x: x["name"], modules[module]["attributes"]):
+                modules[module]["attributes"].append({"name": klass})
+
+        _modules = []
+        for key, value in modules.items():
+            _modules.append({"absolute_path": key, "attributes": value["attributes"]})
+
+        return _modules

ngcsimlib/utils.py

@@ -3,11 +3,18 @@
 """
 import sys, uuid, os, json
 from importlib import import_module
+from ngcsimlib.logger import info
 
 ## Globally tracking all the modules, and attributes have been dynamically loaded
 _Loaded_Attributes = {}
 _Loaded_Modules = {}
+_Loaded = False
+def is_pre_loaded():
+    return _Loaded
 
+def set_loaded(val):
+    global _Loaded
+    _Loaded = val
 
 def check_attributes(obj, required, fatal=False):
     """
@@ -68,7 +75,7 @@ def load_module(module_path, match_case=False, absolute_path=False):
             last_mod = module.split('.')[-1]
             last_mod = last_mod if match_case else last_mod.lower()
             if final_mod == last_mod:
-                print("Loading module from " + module)
+                info("Loading module from " + module)
                 module_name = module
                 break