Compiled (#8)

* Start of rewrite

* fix argument bugs for resolver

* address Will's comment on line 26

* remove loop and params generator functionalities in command compilation.

* add representation debugging strings

* add more debugging strings

* Fixed artifact bug in connection compiling

* remove printing

* add component to context functionality

* add repr functionality

* Compile changes

- Pattern matches argument input for pure function remove order constraint
- Pattern matches argument input from wrapped function to remove output_compartment specification
- Added functionality to account for nested operations

* Cleaned up aspects of the code

* Made compilers

Moved all the compilers to their own directory.

* Missed parse call

* Added save and load functionality

* Actually loads custom folder

* fixed small ordering error in command-compiler

* Fixed formatting

---------

Co-authored-by: Viet Nguyen <[email protected]>
Co-authored-by: Alex Ororbia <[email protected]>

Author: 3 people

ngcsimlib/__init__.py

@@ -1,12 +1,9 @@
 from . import utils
 from . import controller
 from . import commands
-from . import logger
 
 import argparse, os, warnings, json
 from types import SimpleNamespace
-from pathlib import Path
-from sys import argv
 from importlib import import_module
 from ngcsimlib.configManager import init_config, get_config
 
@@ -68,12 +65,4 @@ def configure():
                       "additional information")
         return
 
-
     init_config(config_path)
-
-
-if not Path(argv[0]).name == "sphinx-build" or Path(argv[0]).name == "build.py":
-    if "readthedocs" not in argv[0]:  ## prevent readthedocs execution of preload
-        configure()
-        logger.init_logging()
-        preload_modules()

ngcsimlib/bundle_rules.py

@@ -1,5 +1,6 @@
 from abc import ABC, abstractmethod
-from ngcsimlib.utils import check_attributes
+from ngcsimlib.utils import check_attributes, get_current_context
+
 
 
 class Command(ABC):
@@ -11,6 +12,18 @@ class Command(ABC):
     attributes are present on each component. Note that this step does not
     ensure types or values, just that they do or do not exist.
     """
+
+    """
+    The Compile key is the name of the resolver the compile function will look for
+    when compiling this command. If unset the compile will fail.
+    """
+    compile_key = None
+
+    def __new__(cls, components=None, command_name=None, *args, **kwargs):
+        if get_current_context() is not None:
+            get_current_context().register_command(cls.__name__, *args, components=components, command_name=command_name, **kwargs)
+        return super().__new__(cls)
+
     def __init__(self, components=None, command_name=None, required_calls=None):
         """
         Required calls on Components: ['name']
@@ -22,13 +35,20 @@ def __init__(self, components=None, command_name=None, required_calls=None):
 
             command_name: the name of the command on the controller
         """
+
         self.name = str(command_name)
+
         self.components = {}
         required_calls = ['name'] if required_calls is None else required_calls + ['name']
         for comp in components:
             if check_attributes(comp, required_calls, fatal=True):
                 self.components[comp.name] = comp
 
+
+        get_current_context().add_command(self)
+
+
     @abstractmethod
     def __call__(self, *args, **kwargs):
         pass
+

ngcsimlib/commands/command.py

@@ -0,0 +1,49 @@
+from ngcsimlib.operations import BaseOp, overwrite
+from ngcsimlib.utils import Set_Compartment_Batch, get_current_path
+import uuid
+
+
+class Compartment:
+    @classmethod
+    def is_compartment(cls, obj):
+        return hasattr(obj, "_is_compartment")
+
+    def __init__(self, initial_value=None, static=False):
+
+        self._is_compartment = True
+        self.__add_connection = None
+        self._static = static
+        self.value = initial_value
+        self._uid = uuid.uuid4()
+        self.name = None
+        self.path = None
+
+    def _setup(self, current_component, key):
+        self.__add_connection = current_component.add_connection
+        self.name = current_component.name + "/" + key
+        self.path = get_current_path() + "/" + self.name
+        Set_Compartment_Batch({str(self.path): self})
+
+    def set(self, value):
+        if not self._static:
+            self.value = value
+        else:
+            raise RuntimeError("Can not assign value to static compartment")
+
+    def clamp(self, value):
+        self.set(value)
+
+    def __repr__(self):
+        return f"[{self.name}]"
+
+    def __str__(self):
+        return str(self.value)
+
+    def __lshift__(self, other) -> None:
+        if isinstance(other, BaseOp):
+            other.set_destination(self)
+            self.__add_connection(other)
+        else:
+            op = overwrite(other)
+            op.set_destination(self)
+            self.__add_connection(op)

ngcsimlib/compartment.py

@@ -0,0 +1,3 @@
+from .command_compiler import compile as compile_command, dynamic_compile, wrap_command
+from .component_compiler import compile as compile_component
+from .op_compiler import compile as compile_op

ngcsimlib/compilers/__init__.py

@@ -0,0 +1,67 @@
+from ngcsimlib.compilers.component_compiler import parse as parse_component, compile as compile_component
+from ngcsimlib.compilers.op_compiler import parse as parse_connection
+from ngcsimlib.utils import Get_Compartment_Batch, Set_Compartment_Batch
+
+
+def _compile(compile_key, components):
+    assert compile_key is not None
+    ## for each component, get compartments, get output compartments
+    resolvers = {}
+    for c_name, component in components.items():
+        resolvers[c_name] = parse_component(component, compile_key)
+
+    needed_args = []
+    needed_comps = []
+
+    for c_name, component in components.items():
+        _, outs, args, params, comps = resolvers[c_name]
+        for _, a in args:
+            if a not in needed_args:
+                needed_args.append(a)
+
+        for connection in component.connections:
+            inputs, outputs = parse_connection(connection)
+            ncs = [str(i) for i in inputs]
+            for nc in ncs:
+                if nc not in needed_comps:
+                    needed_comps.append(nc)
+
+        for _, comp in comps:
+            path = str(component.__dict__[comp].path)
+            if path not in needed_comps:
+                needed_comps.append(path)
+    arg_order = needed_args + needed_comps
+    exc_order = []
+    for c_name, component in components.items():
+        exc_order.extend(compile_component(component, resolvers[c_name], arg_order))
+
+    def compiled(compartment_values, *cargs):
+        for exc, outs, name in exc_order:
+            _comps = [compartment_values[key] for key in needed_comps]
+            vals = exc(*cargs, *_comps)
+            if len(outs) == 1:
+                compartment_values[outs[0]] = vals
+            elif len(outs) > 1:
+                for v, t in zip(vals, outs):
+                    compartment_values[t] = v
+        return compartment_values
+
+    return compiled, needed_args
+
+
+def compile(command):
+    return _compile(command.compile_key, command.components)
+
+
+def dynamic_compile(*components, compile_key=None):
+    assert compile_key is not None
+    return _compile(compile_key, {c.name: c for c in components})
+
+
+def wrap_command(command):
+    def _wrapped(*args):
+        vals = command(Get_Compartment_Batch(), *args)
+        Set_Compartment_Batch(vals)
+        return vals
+
+    return _wrapped

ngcsimlib/compilers/command_compiler.py

@@ -0,0 +1,72 @@
+from ngcsimlib.compilers.op_compiler import compile as op_compile
+
+from ngcsimlib.utils import get_resolver
+from ngcsimlib.compartment import Compartment
+
+
+def parse(component, compile_key):
+    (pure_fn, output_compartments), (args, parameters, compartments, parse_varnames) = \
+        get_resolver(component.__class__.__name__, compile_key)
+
+    if parse_varnames:
+        args = []
+        parameters = []
+        compartments = []
+        varnames = pure_fn.__func__.__code__.co_varnames[:pure_fn.__func__.__code__.co_argcount]
+
+        for idx, n in enumerate(varnames):
+            if n not in component.__dict__.keys():
+                args.append((idx, n))
+            elif Compartment.is_compartment(component.__dict__[n]):
+                compartments.append((idx, n))
+            else:
+                parameters.append((idx, n))
+
+        if output_compartments is None:
+            output_compartments = compartments[:]
+
+    return (pure_fn, output_compartments, args, parameters, compartments)
+
+
+def compile(component, resolver, arg_order):
+    exc_order = []
+    pure_fn, outs, _args, params, comps = resolver
+
+    ### Op resolve
+    for connection in component.connections:
+        exc_order.append(op_compile(connection, arg_order))
+
+    ### Component resolve
+    comp_ids = [str(component.__dict__[comp].path) for _, comp in comps]
+    out_ids = [str(component.__dict__[comp].path) for comp in outs]
+
+    funParams = [component.__dict__[narg] for _, narg in (list(params))]
+
+    arg_locs = [loc for loc, _ in _args]
+    param_locs = [loc for loc, _ in params]
+    comp_locs = [loc for loc, _ in comps]
+
+    def compiled(*args):
+        funArgs = [args[arg_order.index(narg)] for _, narg in (list(_args))]
+        funComps = [args[arg_order.index(narg)] for narg in comp_ids]
+
+        _arg_loc = 0
+        _param_loc = 0
+        _comps_loc = 0
+
+        fargs = []
+        for i in range(len(arg_locs) + len(param_locs) + len(comp_locs)):
+            if i in arg_locs:
+                fargs.append(funArgs[_arg_loc])
+                _arg_loc += 1
+            elif i in param_locs:
+                fargs.append(funParams[_param_loc])
+                _param_loc += 1
+            else:
+                fargs.append(funComps[_comps_loc])
+                _comps_loc += 1
+
+        return pure_fn(*fargs)
+
+    exc_order.append((compiled, out_ids, component.name))
+    return exc_order

ngcsimlib/compilers/component_compiler.py

@@ -0,0 +1,56 @@
+from ngcsimlib.operations import BaseOp
+
+
+def parse(op):
+    assert op.is_compilable
+    inputs = []
+    for s in op.sources:
+        if isinstance(s, BaseOp):
+            needed_inputs, dest = parse(s)
+            assert dest is None
+            for inp in needed_inputs:
+                if inp not in inputs:
+                    inputs.append(inp)
+        else:
+            inputs.append(s.path)
+
+    return inputs, op.destination.name if op.destination is not None else None
+
+
+def compile(op, arg_order):
+    exc_order = []
+    ops = []
+
+    for idx, s in enumerate(op.sources):
+        if isinstance(s, BaseOp):
+            exc_order.append(compile(s, arg_order))
+            ops.append(idx)
+
+    output = op.destination.path if op.destination is not None else None
+
+    iids = []
+    for s in op.sources:
+        if isinstance(s, BaseOp):
+            pass
+        else:
+            iids.append(str(s.path))
+
+    def _op_compiled(*args):
+        computed_values = [cmd(*args) for cmd, _, _ in exc_order]
+        compartment_args = [args[arg_order.index(narg)] for narg in iids]
+
+        _val_loc = 0
+        _arg_loc = 0
+
+        _args = []
+        for i in range(len(op.sources)):
+            if i in ops:
+                _args.append(computed_values[_val_loc])
+                _val_loc += 1
+            else:
+                _args.append(compartment_args[_arg_loc])
+                _arg_loc += 1
+
+        return op.operation(*_args)
+
+    return (_op_compiled, [str(output)], "op")