revert to using members for dtype enums

Author: jreadey

src/h5json/hdf5db.py

@@ -99,6 +99,14 @@ def is_dirty(self, obj_id):
             return True
         return obj_id in self._dirty_objects
 
+    @property
+    def new_objects(self):
+        return self._new_objects
+    
+    @property
+    def dirty_objects(self):
+        return self._dirty_objects
+    
     def make_dirty(self, obj_id):
         """ Mark the object as dirty and update the lastModified timestamp """
         if self.is_new(obj_id):

src/h5json/hdf5dtype.py

@@ -424,9 +424,11 @@ def getTypeItem(dt, metadata=None):
         if dt.base.byteorder == ">":
             byteorder = "BE"
         # this mapping is an h5py convention for boolean support
-        mapping = {"FALSE": 0, "TRUE": 1}
+        bool_false = {"name": "FALSE", "value": 0}
+        bool_true = {"name": "TRUE", "value": 1}
+        members = [bool_false, bool_true]
         type_info["class"] = "H5T_ENUM"
-        type_info["mapping"] = mapping
+        type_info["members"] = members
         base_info = {"class": "H5T_INTEGER"}
         base_info["base"] = "H5T_STD_I8" + byteorder
         type_info["base"] = base_info
@@ -456,7 +458,13 @@ def getTypeItem(dt, metadata=None):
             # yes, this is an enum!
             mapping = metadata["enum"]
             type_info["class"] = "H5T_ENUM"
-            type_info["mapping"] = mapping
+            members = []
+            for name in mapping:
+                value = mapping[name]
+                item = {"name": name, "value": value}
+                members.append(item)
+            type_info["members"] = members
+            #type_info["mapping"] = mapping
             if dt.name not in predefined_int_types:
                 raise TypeError("Unexpected integer type: " + dt.name)
             # maps to one of the HDF5 predefined types

src/h5json/reader/h5py_reader.py

@@ -96,7 +96,7 @@ def getAttribute(self, obj_id, name, include_data=True):
             addr = h5py.h5o.get_info(typeid).addr
             type_uuid = self.getObjIdByAddress(addr)
             committedType = self._id_map[type_uuid]
-            type_item = committedType["type"].copy()
+            type_item = getTypeItem(committedType.dtype)
             type_item["id"] = type_uuid
         else:
             type_item = getTypeItem(attrObj.dtype)
@@ -353,7 +353,7 @@ def getObjectById(self, obj_id, include_attrs=True, include_links=True):
         elif isinstance(h5obj, h5py.Dataset):
             obj_json = self._getDataset(h5obj)
         elif isinstance(h5obj, h5py.Datatype):
-            obj_json = self._getDataType(h5obj)
+            obj_json = self._getDatatype(h5obj)
         else:
             raise TypeError(f"unexpected object type: {type(h5obj)}")
 

src/h5json/writer/h5py_writer.py

@@ -45,9 +45,8 @@ def __init__(
     def _createGroup(self, parent, grp_json, name=None):
         """ create the group and any links it contains """
         grp = parent.create_group(name)
-        if "links" in grp_json:
-            grp_links = grp_json["links"]
-            self._createObjects(grp, grp_links)
+        return grp
+
 
     def _createDataset(self, parent, dset_json, name=None):
         """ create a dataset object """
@@ -156,58 +155,85 @@ def _createDataset(self, parent, dset_json, name=None):
                         else:
                             self.log.info(f"Unexpected filter name: {filter_alias}, ignoring")
 
-        parent.create_dataset(name, **kwargs)
+        dset = parent.create_dataset(name, **kwargs)
+        return dset
 
     def _createDatatype(self, parent, ctype_json, name=None):
         """ create a datatype object """
 
         type_item = ctype_json["type"]
         dtype = createDataType(type_item)
         parent[name] = dtype
+        return parent[name]
 
 
-    def _createObjects(self, parent, links_json):
+    def _createObjects(self, parent, links_json, visited=set()):
         """ create child object in the given group, recurse for any sub-groups """
+
         for title in links_json:
-            if title in parent:
-                # TBD: this will do the wrong thing if the link tgt has changed
-                continue
+            #if title in parent:
+            #    # TBD: this will do the wrong thing if the link tgt has changed
+            #    continue
             link_json = links_json[title]
             link_class = link_json["class"]
-            if link_class == "H5L_TYPE_SOFT":
+            if link_class == "H5L_TYPE_SOFT" and title not in parent:
                 h5path = link_json["h5path"]
                 parent[title] = h5py.SoftLink(h5path)
-            elif link_class == "H5L_TYPE_EXTERNAL":
+            elif link_class == "H5L_TYPE_EXTERNAL" and title not in parent:
                 h5path = link_json["h5path"]
                 filename = link_json["file"]
                 parent[title] = h5py.ExternalLink(filename, h5path)
-            elif link_class == "H5L_TYPE_USER_DEFINED":
+            elif link_class == "H5L_TYPE_USER_DEFINED" and title not in parent:
                 self.log.warning("unable to create user-defined link: {title}")
             elif link_class == "H5L_TYPE_HARD":
                 tgt_id = link_json["id"]
+                """
+                if tgt_id in visited:
+                    # we've already processed this object
+                    if title not in parent:
+                        if tgt_id in self._id_map:
+                            tgt_obj = self._id_map[tgt_id]
+                            parent[title] = tgt_obj
+                    else:
+                        self.log.warning("h5py_writer - expected to find {tgt_id} in id_map")
+                    continue
+                """
+                
+                collection = getCollectionForId(tgt_id)
+
+                obj_json = self.db.getObjectById(tgt_id)
+            
                 if tgt_id in self._id_map:
+                    # object has already been created
                     tgt_path = self._id_map[tgt_id]
                     tgt_obj = parent[tgt_path]
-                    parent[title] = tgt_obj
+                    if title not in parent:
+                        parent[title] = tgt_obj
+                    if collection == "groups" and tgt_id not in visited:
+                        # recurse over sub-objects to pick up any new links
+                        grp_links = obj_json["links"]
+                        visited.add(tgt_id)
+                        self._createObjects(tgt_obj, grp_links, visited=visited)
                 else:
-                    obj_json = self.db.getObjectById(tgt_id)
                     parent_path = parent.name
                     if parent_path[-1] != '/':
                         parent_path += '/'
                     self._id_map[tgt_id] = parent_path + title
-                    collection = getCollectionForId(tgt_id)
                     kwds = {"name": title}
                     if collection == "groups":
-                        tgt_obj = self._createGroup(parent, obj_json, **kwds)
+                        tgt_grp = self._createGroup(parent, obj_json, **kwds)
+                        if "links" in obj_json:
+                            grp_links = obj_json["links"]
+                            visited.add(tgt_id)
+                            self._createObjects(tgt_grp, grp_links, visited=visited)
                     elif collection == "datasets":
-                        tgt_obj = self._createDataset(parent, obj_json, **kwds)
+                        self._createDataset(parent, obj_json, **kwds)
                     elif collection == "datatypes":
-                        tgt_obj = self._createDatatype(parent, obj_json, **kwds)
+                        self._createDatatype(parent, obj_json, **kwds)
                     else:
                         self.log.warning(f"unexpected collection: {collection}")
-                        tgt_obj = None
-                    if tgt_obj:
-                        parent[title] = tgt_obj
+                visited.add(tgt_id)
+
             else:
                 self.log.warning(f"unexpected link class: {link_class}")
 
@@ -231,7 +257,6 @@ def updateDatasetValues(self, dset_id, dset):
 
     def createAttribute(self, obj, name, attr_json):
         """ add the given attribute to obj """
-        print(f"h5py_writer.createAttribute {obj.name}: {name}")
 
         dtype = createDataType(attr_json["type"])
         shape_json = attr_json["shape"]
@@ -276,10 +301,11 @@ def flush(self):
         root_id = self.db.root_id
         self._id_map[root_id] = "/"
         with h5py.File(self._filepath, mode=self._mode) as f:
-            root_json = self.db.getObjectById(root_id)
-            if "links" in root_json:
-                root_links = root_json["links"]
-                self._createObjects(f, root_links)
+            if self.db.new_objects:
+                root_json = self.db.getObjectById(root_id)
+                if "links" in root_json:
+                    root_links = root_json["links"]
+                    self._createObjects(f, root_links, visited=set(root_id))
             # update attributes, dataset values
             for obj_id in self._id_map:
                 if self.db.is_dirty(obj_id):

test/unit/h5py_writer_test.py

@@ -90,6 +90,44 @@ def testGroup(self):
                 g2 = f["g2"]
                 self.assertTrue("extlink" in g2)
                 self.assertTrue("slink" in g2)
+
+            db.createAttribute(g1_id, "a2", "bye-bye")
+            db.flush()
+
+            with h5py.File(filepath) as f:
+                g1 = f["g1"]
+                self.assertEqual(len(g1.attrs), 2)
+                self.assertTrue("a1" in g1.attrs)
+                self.assertTrue("a2" in g1.attrs)
+
+            print("create group /g2/g2.1")
+            g21 = db.createGroup()
+            db.createHardLink(g2_id, "g2.1", g21)
+            db.flush()
+
+            with h5py.File(filepath) as f:
+                g2 = f["g2"]
+                self.assertTrue("g2.1" in g2)
+            
+            sel = selections.select((10, 10), (slice(4, 5), slice(4, 5)))
+            arr = np.zeros((), dtype=np.int32)
+            arr[()] = 42
+            db.setDatasetValues(dset_111_id, sel, arr)
+            db.flush()
+
+            with h5py.File(filepath) as f:
+                dset = f["/g1/g1.1/dset1.1.1"]
+                for i in range(10):
+                    for j in range(10):
+                        if i == 4 and j == 4:
+                            # this is the one element that was updated
+                            expected = 42
+                        else:
+                            expected = i * j
+                        self.assertEqual(dset[i, j], expected)
+
+
+            
 
 
 

test/unit/hdf5dtype_test.py

@@ -125,8 +125,10 @@ def testBaseEnumTypeItem(self):
         baseItem = typeItem["base"]
         self.assertEqual(baseItem["class"], "H5T_INTEGER")
         self.assertEqual(baseItem["base"], "H5T_STD_I8LE")
-        self.assertTrue("mapping" in typeItem)
-        self.assertEqual(typeItem["mapping"]["GREEN"], 1)
+        self.assertTrue("members" in typeItem)
+        members = typeItem["members"]
+        expected = [{'name': 'RED', 'value': 0}, {'name': 'GREEN', 'value': 1}, {'name': 'BLUE', 'value': 2}]
+        self.assertEqual(members, expected)
         self.assertEqual(typeSize, 1)
 
     def testBaseBoolTypeItem(self):
@@ -136,11 +138,11 @@ def testBaseBoolTypeItem(self):
         baseItem = typeItem["base"]
         self.assertEqual(baseItem["class"], "H5T_INTEGER")
         self.assertEqual(baseItem["base"], "H5T_STD_I8LE")
-        self.assertTrue("mapping" in typeItem)
-        mapping = typeItem["mapping"]
-        self.assertEqual(len(mapping), 2)
-        self.assertEqual(mapping["FALSE"], 0)
-        self.assertEqual(mapping["TRUE"], 1)
+        self.assertTrue("members" in typeItem)
+        members = typeItem["members"]
+        self.assertEqual(len(members), 2)
+        self.assertEqual(members[0], {"name": "FALSE", "value": 0})
+        self.assertEqual(members[1], {"name": "TRUE", "value": 1})
         self.assertEqual(typeSize, 1)
 
     def testBaseArrayTypeItem(self):
@@ -205,8 +207,12 @@ def testEnumArrayTypeItem(self):
         self.assertEqual(typeItem["dims"], (2, 3))
         baseItem = typeItem["base"]
         self.assertEqual(baseItem["class"], "H5T_ENUM")
-        self.assertTrue("mapping" in baseItem)
-        self.assertEqual(baseItem["mapping"]["GREEN"], 1)
+        self.assertTrue("members" in baseItem)
+        members = baseItem["members"]
+        self.assertEqual(len(members), 3)
+        self.assertEqual(members[0], {"name": "RED", "value": 0})
+        self.assertEqual(members[1], {"name": "GREEN", "value": 1})
+        self.assertEqual(members[2], {"name": "BLUE", "value": 2})
         self.assertTrue("base" in baseItem)
         basePrim = baseItem["base"]
         self.assertEqual(basePrim["class"], "H5T_INTEGER")