Merge pull request #12 from ModECI/development

Latest from development with bson support

Author: pgleeson

.github/workflows/ci.yml

@@ -7,13 +7,24 @@ on:
     branches: [ main, development, experimental ]
 
 jobs:
+
+  pre-commit:
+    name: Format
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v1
+    - uses: actions/setup-python@v2
+    - uses: pre-commit/[email protected]
+      with:
+        extra_args: --hook-stage manual --all-files
+
   build:
 
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ 3.7 , 3.9 ]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
     - uses: actions/checkout@v2
@@ -60,7 +71,7 @@ jobs:
 
 
     - name: Install MDF
-      if: ${{ matrix.python-version != '2.7' }}
+      if: ${{ matrix.python-version != '3.10' }}
       run: |
 
           git clone --branch development https://github.com/ModECI/MDF.git

.pre-commit-config.yaml

@@ -46,6 +46,6 @@ repos:
 #    files: src
 
 - repo: https://github.com/psf/black
-  rev: 20.8b1
+  rev: 22.3.0
   hooks:
   - id: black

examples/document.bson

@@ -68,6 +68,7 @@ class Document(Base):
 
 doc.to_json_file("document.json")
 doc.to_yaml_file("document.yaml")
+doc.to_bson_file("document.bson")
 
 doc_md = doc.generate_documentation(format="markdown")
 
@@ -85,10 +86,14 @@ class Document(Base):
 
 import json
 import yaml
+import bson
 
 with open("document.specification.json", "w") as d:
     d.write(json.dumps(doc_dict, indent=4))
 
 print("Generating specification in YAML")
 with open("document.specification.yaml", "w") as d:
     d.write(yaml.dump(doc_dict, indent=4, sort_keys=False))
+
+with open("document.specification.bson", "wb") as d:
+    d.write(bson.encode(doc_dict))

examples/document.py

@@ -16,16 +16,16 @@ classifiers =
     License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
     Natural Language :: English
     Operating System :: OS Independent
-    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
     Topic :: Scientific/Engineering
     Intended Audience :: Science/Research
     Programming Language :: Python
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3 :: Only
-    Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
     Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Topic :: Scientific/Engineering
     Topic :: Software Development
     Typing :: Typed
@@ -34,6 +34,7 @@ classifiers =
 packages = find:
 install_requires =
    pyyaml
+   pymongo
    numpy
    tabulate
    attrs

setup.cfg

@@ -1,4 +1,4 @@
-__version__ = "0.2.3"
+__version__ = "0.2.4"
 
 from .base_types import Base, define, has, field, fields, optional, instance_of, in_
 

src/modelspec/__init__.py

@@ -1,5 +1,6 @@
 import json
 import yaml
+import bson
 import sys
 
 import numpy as np
@@ -23,6 +24,7 @@
 
 from cattr.gen import make_dict_unstructure_fn, override
 from cattr.preconf.pyyaml import make_converter as make_yaml_converter
+from cattr.preconf.bson import make_converter as make_bson_converter
 
 from tabulate import tabulate
 
@@ -72,7 +74,7 @@ def print_v(text: str):
 # A setup an additional converter to apply when we are serializing using PyYAML.
 # This handles things like tuples as lists.
 yaml_converter = make_yaml_converter()
-
+bson_converter = make_bson_converter()
 
 # A simple converter that handles only value expressions.
 value_expr_converter = cattr.Converter()
@@ -104,6 +106,12 @@ def to_json(self) -> str:
         """
         return json.dumps(self.to_dict(), indent=4)
 
+    def to_bson(self) -> str:
+        """
+        Convert the Base object to a BSON string representation.
+        """
+        return bson.encode(self.to_dict())
+
     @classmethod
     def from_dict(cls, d: Dict[str, Any]) -> "Base":
         """Instantiate an Base object from a dictionary"""
@@ -130,6 +138,11 @@ def from_json(cls, json_str: str) -> "Base":
         """Instantiate an modelspec object from a JSON string"""
         return cls.from_dict(json.loads(json_str))
 
+    @classmethod
+    def from_bson(cls, bson_str: str) -> "Base":
+        """Instantiate an modelspec object from a BSON string"""
+        return cls.from_dict(bson.decode(bson_str))
+
     def to_json_file(
         self, filename: Optional[str] = None, include_metadata: bool = True
     ) -> str:
@@ -155,6 +168,28 @@ def to_json_file(
 
         return filename
 
+    def to_bson_file(self, filename: str, include_metadata: bool = True) -> str:
+        """Convert modelspec format to bson format
+
+        Args:
+            filename: File in modelspec format (Filename extension: .bson )
+            include_metadata: Contains contact information, citations, acknowledgements, pointers to sample data,
+                              benchmark results, and environments in which the specified model was originally implemented
+        Returns:
+            The name of the generated bson file
+        """
+
+        if filename is None:
+            filename = f"{self.id}.bson"
+
+        with open(filename, "wb") as outfile:
+            bson_data = bson.encode(
+                bson_converter.unstructure(self.to_dict()),
+            )
+            outfile.write(bson_data)
+
+        return filename
+
     def to_yaml(self, include_metadata: bool = True) -> str:
         """
         Convert the Base object to a YAML dictionary representation.
@@ -211,6 +246,8 @@ def from_file(cls, filename: str) -> "Base":
             return cls.from_yaml_file(filename)
         elif filename.endswith(".json"):
             return cls.from_json_file(filename)
+        elif filename.endswith(".bson"):
+            return cls.from_bson_file(filename)
         else:
             raise ValueError(
                 f"Cannot auto-detect modelspec serialization format from filename ({filename}). The filename "
@@ -232,6 +269,22 @@ def from_json_file(cls, filename: str) -> "Base":
             d = json.load(infile)
             return cls.from_dict(d)
 
+    @classmethod
+    def from_bson_file(cls, filename: str) -> "Base":
+        """
+        Create a :class:`.Base` from its BSON representation stored in a file.
+
+        Args:
+            filename: The file from which to load the BSON data.
+
+        Returns:
+            An modelspec :class:`.Base` for this BSON
+        """
+        with open(filename, "rb") as infile:
+            data_encoded = infile.read()
+            d = bson.decode(data_encoded)
+            return cls.from_dict(d)
+
     @classmethod
     def from_yaml_file(cls, filename: str) -> "Base":
         """

src/modelspec/base_types.py

@@ -1,5 +1,6 @@
 import sys
 import json
+import bson
 import yaml
 import os
 import math
@@ -32,6 +33,17 @@ def load_yaml(filename):
     return data
 
 
+def load_bson(filename):
+    """
+    Load a generic BSON file
+    """
+    with open(filename, "rb") as infile:
+        data_encoded = infile.read()
+        data = bson.decode(data_encoded)
+
+    return data
+
+
 def save_to_json_file(info_dict, filename, indent=4):
 
     strj = json.dumps(info_dict, indent=indent)

src/modelspec/utils.py

@@ -312,3 +312,30 @@ class Node(Base):
 
     model = Node(id="a", metadata={"b": np.array([0])})
     model.to_json()
+
+
+def test_bson_array(tmp_path):
+    import numpy as np
+
+    test_filename = str(tmp_path / "test_array.bson")
+
+    @modelspec.define(eq=False)
+    class ArrayTestModel(Base):
+        array: Optional[ValueExprType] = field()
+        list_of_lists: List[List[int]] = field(default=None)
+        ragged_list: List[List[int]] = field(default=None)
+
+    model = ArrayTestModel(
+        array=np.arange(27).reshape((3, 3, 3)),
+        list_of_lists=[[1, 2], [3, 4]],
+        ragged_list=[[1, 2], [1]],
+    )
+
+    model.to_bson_file(test_filename)
+
+    # Load it back in
+    m2 = model.from_bson_file(test_filename)
+    # Check we get the same values back
+    np.testing.assert_array_equal(model.array, m2.array)
+    assert model.list_of_lists == m2.list_of_lists
+    assert model.ragged_list == m2.ragged_list