add test for incremental updates

Author: jreadey

src/h5json/hdf5db.py

@@ -82,9 +82,19 @@ def reader(self):
     @reader.setter
     def reader(self, value: H5Reader):
         """ set the reader """
+        if self._writer:
+            self.flush()
         if self._reader:
             self._reader.close()
+        root_id = value.get_root_id()
+        if not root_id:
+            raise ValueError(f"reader {type(value)} unable to return root_id")
+        group_json = value.getObjectById(root_id)
+        if not group_json:
+            raise ValueError(f"reader {type(value)} unable to return group json")
         self._reader = value
+        self._db[root_id] = group_json
+        self._root_id = root_id
 
     @property
     def writer(self):
@@ -411,15 +421,10 @@ def createAttribute(self, obj_id, name, value, shape=None, dtype=None):
 
         obj_json = self.getObjectById(obj_id)
         attrs_json = obj_json["attributes"]
-        if name in attrs_json:
-            # replace, keep, created timestamp
-            created = attrs_json["created"]
-        else:
-            created = time.time()
         type_json = getTypeItem(dtype)
         # finally put it all together...
         attr_json = {"shape": shape_json, "type": type_json, "value": value_json}
-        attr_json["created"] = created
+        attr_json["created"] = time.time()
 
         # slot into the obj_json["attrs"]
         attrs_json[name] = attr_json

src/h5json/writer/h5py_writer.py

@@ -11,6 +11,7 @@
 ##############################################################################
 import h5py
 import numpy as np
+import time
 
 from ..objid import getCollectionForId, isValidUuid, getUuidFromId, isObjId
 from ..hdf5dtype import createDataType
@@ -39,6 +40,7 @@ def __init__(
             self._init = False
         else:
             self._init = True
+        self._flush_time = 0.0
 
     def _copy_element(self, val, src_dt, tgt_dt, fout=None):
         """ convert the given dataset or attribute element to h5py equivalent """
@@ -379,10 +381,14 @@ def updateAttributes(self, obj_id, obj):
         attrs = obj_json["attributes"]
         for name in attrs:
             attr_json = attrs[name]
+            if "created" in attr_json and attr_json["created"] < self._flush_time:
+                # ttribute should be saved already
+                continue
             self.createAttribute(obj, name, attr_json)
 
     def flush(self):
         """ Write dirty items """
+
         if not self.db:
             # no db set yet
             return False
@@ -393,6 +399,7 @@ def flush(self):
         with h5py.File(self._filepath, mode=mode) as f:
             if self.db.new_objects or self._init:
                 root_json = self.db.getObjectById(root_id)
+
                 if "links" in root_json:
                     root_links = root_json["links"]
                     self._createObjects(f, root_links, visited=set((root_id,)))
@@ -408,6 +415,10 @@ def flush(self):
                             self.initializeDatasetValues(obj_id, obj)
                         else:
                             self.updateDatasetValues(obj_id, obj)
+            # mark time write is complete
+            # updates before this time will not need to be written
+            # TBD: possible race condition with multithreading
+            self._flush_time = time.time()
 
         self._init = False  # done with init after first flush
         return True  # all objects written successfully

test/unit/h5json_reader_test.py

@@ -39,7 +39,9 @@ def __init__(self, *args, **kwargs):
     def testSimple(self):
         filepath = "data/json/tall.json"
         kwargs = {"app_logger": self.log}
-        with Hdf5db(h5_reader=H5JsonReader(filepath, **kwargs), **kwargs) as db:
+        with Hdf5db(**kwargs) as db:
+            h5_reader = H5JsonReader(filepath, **kwargs)
+            db.reader = h5_reader
             root_id = db.getObjectIdByPath("/")
             root_json = db.getObjectById(root_id)
 

test/unit/h5py_writer_test.py

@@ -15,6 +15,7 @@
 import h5py
 import numpy as np
 from h5json import Hdf5db
+from h5json.reader.h5json_reader import H5JsonReader
 from h5json.writer.h5py_writer import H5pyWriter
 from h5json.hdf5dtype import special_dtype, Reference
 from h5json import selections
@@ -473,6 +474,53 @@ def testCommittedCompoundType(self):
             sub_dt = t1.dtype["field_4"]
             self.assertEqual(sub_dt, h5py.special_dtype(vlen=str))
 
+    def testReaderWithUpdate(self):
+
+        file_in = "data/json/tall.json"
+        file_out = "test/unit/out/h5py_writer_test_testReaderWithUpdate.h5"
+
+        with Hdf5db(app_logger=self.log) as db:
+            db.reader = H5JsonReader(file_in)
+            db.writer = H5pyWriter(file_out, no_data=False)
+            dset111_id = db.getObjectIdByPath("/g1/g1.1/dset1.1.1")
+            db.flush()
+
+            with h5py.File(file_out) as f:
+                self.assertTrue("/g1/g1.1/dset1.1.1" in f)
+                dset111 = f["/g1/g1.1/dset1.1.1"]
+                self.assertEqual(len(dset111.attrs), 2)
+                
+            db.createAttribute(dset111_id, "attr3", "hello")
+            dset_json = db.getObjectById(dset111_id)
+            db.flush()
+
+            with h5py.File(file_out) as f:
+                self.assertTrue("/g1/g1.1/dset1.1.1" in f)
+                dset111 = f["/g1/g1.1/dset1.1.1"]
+                self.assertEqual(len(dset111.attrs), 3)
+                self.assertEqual(dset111.attrs["attr3"], b"hello")
+
+            db.createAttribute(dset111_id, "attr3", "bye-bye")
+            db.flush()
+
+            with h5py.File(file_out) as f:
+                self.assertTrue("/g1/g1.1/dset1.1.1" in f)
+                dset111 = f["/g1/g1.1/dset1.1.1"]
+                self.assertEqual(len(dset111.attrs), 3)
+                self.assertEqual(dset111.attrs["attr3"], b"bye-bye")
+                g1 = f["g1"]
+                
+            # create a new link
+            g13_id = db.createGroup()
+            g1_id = db.getObjectIdByPath("/g1")
+            db.createHardLink(g1_id, "g1.3", g13_id)
+            db.flush()
+
+            with h5py.File(file_out) as f:
+                g1 = f["g1"]
+                self.assertEqual(len(g1), 3)
+                self.assertTrue("g1.3" in g1)
+
 
 if __name__ == "__main__":
     # setup test files