added reader, writer stat method

jreadey · jreadey · commit 4413e9b23179 · 2025-07-31T12:53:23.000+01:00
diff --git a/src/h5json/h5pystore/h5py_reader.py b/src/h5json/h5pystore/h5py_reader.py
@@ -12,6 +12,7 @@
 import h5py
 import numpy as np
 import logging
+from os import stat as os_stat
 import time
 
 from ..objid import createObjId, getCollectionForId
@@ -557,3 +558,16 @@ def getDatasetValues(self, dset_id, sel=None, dtype=None):
         # convert any h5py references to h5json references
         arr = self._copy_array(arr, fin=dset.file)
         return arr
+
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stat_info = os_stat(self.filepath)
+        stats = {}
+        stats['created'] = stat_info.st_ctime
+        stats["lastModified"] = stat_info.st_mtime
+        stats['owner'] = stat_info.st_uid  # TBD: convert to username?
+        return stats
diff --git a/src/h5json/h5pystore/h5py_writer.py b/src/h5json/h5pystore/h5py_writer.py
@@ -11,6 +11,7 @@
 ##############################################################################
 import h5py
 import numpy as np
+from os import stat as os_stat
 import time
 
 from ..objid import getCollectionForId, isValidUuid, createObjId
@@ -460,3 +461,16 @@ def close(self):
     def isClosed(self):
         """ return closed status """
         return False if self._f else True
+
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stat_info = os_stat(self.filepath)
+        stats = {}
+        stats['created'] = stat_info.st_ctime
+        stats["lastModified"] = stat_info.st_mtime
+        stats['owner'] = stat_info.st_uid  # TBD: convert to username?
+        return stats
diff --git a/src/h5json/h5reader.py b/src/h5json/h5reader.py
@@ -92,3 +92,12 @@ def close(self):
     def isClosed(self):
         """ return True if handle is closed """
         pass
+
+    @abstractmethod
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        pass
diff --git a/src/h5json/h5writer.py b/src/h5json/h5writer.py
@@ -32,6 +32,7 @@ def __init__(
         self._no_data = no_data
         self._filepath = filepath
         self._db_ref = None
+        self._lastModified = None
         if app_logger:
             self.log = app_logger
         else:
@@ -49,6 +50,10 @@ def filepath(self):
     def closed(self):
         return self.isClosed()
 
+    @property
+    def lastModified(self):
+        return self._lastModified
+
     @property
     def db(self):
         if not self._db_ref:
@@ -83,3 +88,12 @@ def close(self):
     def isClosed(self):
         """ return True if handle is closed """
         pass
+
+    @abstractmethod
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        pass
diff --git a/src/h5json/hdf5db.py b/src/h5json/hdf5db.py
@@ -105,6 +105,18 @@ def isClosed(self):
         """ return True if handle is closed """
         return self._is_closed
 
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stats = {}
+        stats['created'] = 0
+        stats["lastModified"] = 0
+        stats['owner'] = ""
+        return stats
+
 
 class H5NullWriter(H5Writer):
     """
@@ -165,6 +177,18 @@ def isClosed(self):
         """ return True if handle is closed """
         return self._is_closed
 
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stats = {}
+        stats['created'] = 0
+        stats["lastModified"] = 0
+        stats['owner'] = ""
+        return stats
+
 
 class Hdf5db:
     """
@@ -417,11 +441,11 @@ def _checkWriter(self):
         if self.writer.closed:
             raise IOError("writer is closed")
 
-    def getObjectById(self, obj_id):
+    def getObjectById(self, obj_id, refresh=False):
         """ return object with given id """
         self.log.debug(f"getObjectById {obj_id}")
         self._checkReader()
-        if obj_id not in self.db:
+        if obj_id not in self.db or refresh:
             # load the obj from the reader
             obj_json = self.reader.getObjectById(obj_id)
             self.db[obj_id] = obj_json
diff --git a/src/h5json/hsdsstore/hsds_reader.py b/src/h5json/hsdsstore/hsds_reader.py
@@ -310,3 +310,15 @@ def getDatasetValues(self, dset_id, sel=None, dtype=None):
             self.log.debug(f"jsonToArray returned: {arr}")
 
         return arr
+
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stats = {}
+        stats['created'] = 0
+        stats["lastModified"] = 0
+        stats['owner'] = ""
+        return stats
diff --git a/src/h5json/hsdsstore/hsds_writer.py b/src/h5json/hsdsstore/hsds_writer.py
@@ -385,6 +385,7 @@ def updateLinks(self, grp_ids):
                 raise IOError("hsds_writer unable to update links")
             else:
                 self.log.debug(f"hsds_writer> {grp_id} {count} links updated")
+                self._lastModified = time.time()
 
     def updateAttributes(self, obj_ids):
         """ update any modified links of the given objects """
@@ -418,6 +419,7 @@ def updateAttributes(self, obj_ids):
                 self.log.error(f"hsds_writer> put {req} failed, status: {put_rsp.status_code}")
             else:
                 self.log.debug(f"hsds_writer> {count} attributes updated")
+                self._lastModified = time.time()
 
     def updateValue(self, dset_id, sel, arr):
         """ update the given dataset using selection and array """
@@ -437,6 +439,7 @@ def updateValue(self, dset_id, sel, arr):
             self.log.error(f"PUT {req} returned error: {rsp.status_code}")
         else:
             self.log.debug(f"PUT {len(data)} bytes successful")
+            self._lastModified = time.time()
 
     def updateValues(self, dset_ids):
         """ write any pending dataset values """
@@ -476,7 +479,6 @@ def flush(self):
         if not self._http_conn:
             self.log.warning("hsds_writer no http connection")
             raise IOError("no http connection")
-
         self.log.info("hsds_writer.flush()")
         self.log.debug(f"    new object count: {len(self.db.new_objects)}")
         self.log.debug(f"    dirty object count: {len(self.db.dirty_objects)}")
@@ -535,3 +537,15 @@ def isClosed(self):
     def get_root_id(self):
         """ Return root id """
         return self._root_id
+
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stats = {}
+        stats['created'] = 0
+        stats["lastModified"] = 0
+        stats['owner'] = ""
+        return stats
diff --git a/src/h5json/jsonstore/h5json_reader.py b/src/h5json/jsonstore/h5json_reader.py
@@ -11,6 +11,7 @@
 ##############################################################################
 import json
 import logging
+from os import stat as os_stat
 
 from ..objid import getCollectionForId, getUuidFromId
 
@@ -215,3 +216,16 @@ def getDatasetValues(self, obj_id, sel=None, dtype=None):
             raise NotImplementedError("selection type not supported")
 
         return arr
+
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stat_info = os_stat(self.filepath)
+        stats = {}
+        stats['created'] = stat_info.st_ctime
+        stats["lastModified"] = stat_info.st_mtime
+        stats['owner'] = stat_info.st_uid  # TBD: convert to username?
+        return stats
diff --git a/src/h5json/jsonstore/h5json_writer.py b/src/h5json/jsonstore/h5json_writer.py
@@ -11,6 +11,8 @@
 ##############################################################################
 
 import json
+from os import stat as os_stat
+import time
 
 from ..h5writer import H5Writer
 from ..objid import getUuidFromId, getCollectionForId, createObjId
@@ -292,3 +294,17 @@ def dumpFile(self):
                 json.dump(self.json, f, ensure_ascii=ensure_ascii, indent=indent)
         else:
             print(json.dumps(self.json, sort_keys=True, ensure_ascii=ensure_ascii, indent=indent))
+        self._lastModified = time.time()  # update timestamp
+
+    def getStats(self):
+        """ return a dictionary object with at minimum the following keys:
+            'created': creation time
+            'lastModified': modificationTime
+            'owner': owner name
+        """
+        stat_info = os_stat(self.filepath)
+        stats = {}
+        stats['created'] = stat_info.st_ctime
+        stats["lastModified"] = stat_info.st_mtime
+        stats['owner'] = stat_info.st_uid  # TBD: convert to username?
+        return stats
diff --git a/test/unit/h5json_writer_test.py b/test/unit/h5json_writer_test.py
@@ -70,7 +70,9 @@ def testSimple(self):
         db.createSoftLink(g2_id, "slink", "somewhere")
         db.createExternalLink(g2_id, "extlink", "somewhere", "someplace")
         db.createCustomLink(g2_id, "cust", {"foo": "bar"})
+        self.assertTrue(db.writer.lastModified is None)  # no update yet
         db.flush()
+        self.assertTrue(db.writer.lastModified > 0)  # timestamp should be updated
 
     def testNullSpaceAttribute(self):
 
diff --git a/test/unit/hdf5db_test.py b/test/unit/hdf5db_test.py
@@ -53,6 +53,8 @@ def testOpen(self):
         self.assertTrue(db.closed)
         obj_id = db.open()
         self.assertEqual(obj_id, root_id)
+        root_json = db.getObjectById(root_id)
+        self.assertFalse("id" in root_json)
         db.close()
 
     def testWith(self):
diff --git a/test/unit/hsds_reader_test.py b/test/unit/hsds_reader_test.py
@@ -11,6 +11,8 @@
 ##############################################################################
 import unittest
 import logging
+import random
+import string
 import numpy as np
 from h5json import Hdf5db
 from h5json.hsdsstore.hsds_reader import HSDSReader
@@ -106,6 +108,20 @@ def testSimple(self):
 
         db.close()
 
+    def testNoFile(self):
+        # create a random string so we don't try to open an existing file
+        filename = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
+        filepath = "/home/test_user1/test/" + filename
+        kwargs = {"app_logger": self.log}
+        db = Hdf5db(**kwargs)
+        hsds_reader = HSDSReader(filepath, **kwargs)
+        db.reader = hsds_reader
+        try:
+            db.open()
+            self.assertTrue(False)
+        except IOError as ioe:
+            self.assertEqual(ioe.errno, 404)
+
 
 if __name__ == "__main__":
     # setup test files
diff --git a/test/unit/hsds_writer_test.py b/test/unit/hsds_writer_test.py
@@ -11,12 +11,14 @@
 ##############################################################################
 import unittest
 import logging
+import random
+import string
 import requests
-import os
 import numpy as np
 from h5json import Hdf5db
 from h5json.hsdsstore.httpconn import HttpConn
 from h5json.hsdsstore.hsds_writer import HSDSWriter
+from h5json.hsdsstore.hsds_reader import HSDSReader
 from h5json.h5pystore.h5py_reader import H5pyReader
 from h5json.hdf5dtype import special_dtype, Reference
 from h5json import selections
@@ -42,6 +44,10 @@ def testSimple(self):
         db = Hdf5db(app_logger=self.log)
         db.writer = HSDSWriter(domain_path, app_logger=self.log)
         root_id = db.open()
+
+        stats = db.writer.getStats()
+        for k in ("created", "lastModified", "owner"):
+            self.assertTrue(k in stats)
         http_conn = HttpConn(domain_path, mode='r', retries=1)
 
         db.createAttribute(root_id, "attr1", value=[1, 2, 3, 4])
@@ -61,8 +67,9 @@ def testSimple(self):
         self.assertEqual(root_json["attributeCount"], 0)
         # same for link count
         self.assertEqual(root_json["linkCount"], 0)
-
+        self.assertTrue(db.writer.lastModified is None)  # no write yet
         db.flush()
+        self.assertTrue(db.writer.lastModified > 0)  # timestamp should be updated
 
         # validate - get the root group again and see if counts are updated
         http_rsp = http_conn.GET(f"/groups/{root_id}")
@@ -159,7 +166,33 @@ def testSimple(self):
         rsp_value = rsp_json["value"]
         self.assertEqual(rsp_value, 42)
 
+        # create a dataset and try to read from it
+        dset_222_id = db.createDataset(shape=(10, 10), dtype=np.int32)
+        sel_all = selections.select((10, 10), ...)
+        arr = db.getDatasetValues(dset_222_id, sel_all)
+        self.assertTrue((arr == 0).all())
+
+        db.close()
+
+    def testReaderWriter(self):
+        # try reading and writer to an HSDS domain
+        # create a random string so we don't try to open an existing file
+        filename = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
+        domain_path = "/home/test_user1/test/" + filename + ".h5"
+        db = Hdf5db(app_logger=self.log)
+        db.writer = HSDSWriter(domain_path, app_logger=self.log)
+        self.assertEqual(db.writer.filepath, domain_path)
+        root_id = db.open()
+        self.assertTrue(root_id)
+        db.reader = HSDSReader(domain_path, app_logger=self.log)
         db.close()
+        root_id2 = db.open()
+        self.assertEqual(root_id, root_id2)
+        root_json = db.getObjectById(root_id)
+        self.assertTrue("id" not in root_json)
+        self.assertTrue("created" in root_json)
+        self.assertTrue(root_json["created"] > 0)
+        self.assertTrue(db.writer.lastModified is None)  # no flush yet
 
     def testH5PyToHS(self):
         # test reading from HDF5 file and writing to HSDS
