Abstract reader/writer design

.github/workflows/ci.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -25,23 +25,67 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         shell: bash
         run: |
           python -m pip install --upgrade pip
           python -m pip install flake8 pytest
+
       - name: Lint with flake8
         shell: bash
         run: |
           # stop the build if there are Python syntax errors or undefined names
           flake8 . --count  --select=E9,F63,F7,F82 --show-source --statistics
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --ignore=F401,W503,E203 --max-complexity=99 --max-line-length=127 --statistics
+
       - name: Install h5json
         shell: bash
         run: |
           pip install -e .
+
+      - name: Checkout HSDS 
+        uses: actions/checkout@v4
+        with:
+          repository: HDFGroup/hsds
+          path: ${{github.workspace}}/hsds
+
+      - name: Install HSDS
+        working-directory: ${{github.workspace}}/hsds
+        shell: bash
+        run: |
+          pip install -e .
+
+      - name: Start HSDS
+        shell: bash
+        working-directory: ${{github.workspace}}/hsds
+        run: |
+          mkdir hsds_root
+          mkdir hsds_root/hsds_bucket
+          cp admin/config/groups.default admin/config/groups.txt
+          cp admin/config/passwd.default admin/config/passwd.txt
+          hsds --root_dir hsds_root --host localhost --port 5101 --password_file admin/config/passwd.txt --logfile hs.log --loglevel DEBUG --config_dir=admin/config --count=4 &
+
+      - name: Wait for node startup
+        shell: bash
+        run: |
+          sleep 30
+
+      - name: HSDS Setup
+        shell: bash
+        env:
+          ADMIN_PASSWORD: admin
+          ADMIN_USERNAME: admin
+        working-directory: ${{github.workspace}}/hsds
+        run: |
+          python tests/integ/setup_test.py
+
       - name: Run tests
         shell: bash
+        env:
+          HS_ENDPOINT: http://localhost:5101
+          HS_USERNAME: test_user1
+          HS_PASSWORD: test
         run: |
           python testall.py

pyproject.toml

@@ -9,36 +9,35 @@ classifiers = [
     "Topic :: Software Development :: Build Tools",
     "License :: OSI Approved :: BSD License",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
 authors = [{ "name" = "The HDF Group", "email" = "[email protected]" }]
 keywords = ["json", "hdf5", "multidimensional array", "data", "datacube"]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
-    "h5py >=3.10",
+    "h5py >= 3.10",
     "numpy >= 2.0; python_version>='3.9'",
     "jsonschema >=4.4.0",
     "tomli; python_version<'3.11'",
-    "numpy >=1.20,<2.0.0; python_version=='3.8'",
 ]
+
 dynamic = ["version"]
 
 [project.urls]
-Homepage = "https://hdf5-json.readthedocs.io"
-Documentation = "https://hdf5-json.readthedocs.io"
+Homepage = "https://support.hdfgroup.org/documentation/hdf5-json/latest/"
+Documentation = "https://support.hdfgroup.org/documentation/hdf5-json/latest/"
 Source = "https://github.com/HDFGroup/hdf5-json"
 "Bug Reports" = "https://github.com/HDFGroup/hdf5-json/issues"
 Social = "https://twitter.com/hdf5"
 Discussion = "https://forum.hdfgroup.org"
 
 [project.scripts]
-h5tojson = "h5json.h5tojson.h5tojson:main"
-jsontoh5 = "h5json.jsontoh5.jsontoh5:main"
-h5jvalidate = "h5json.validator.validator:main"
+h5tojson = "h5json.apps.h5tojson:main"
+jsontoh5 = "h5json.apps.jsontoh5:main"
+h5jvalidate = "h5json.apps.validator:main"
 
 [project.optional-dependencies]
 dev = ["check-manifest"]
@@ -52,10 +51,10 @@ build-backend = "setuptools.build_meta"
 package-dir = { "" = "src" }
 packages = [
     "h5json",
-    "h5json.h5tojson",
-    "h5json.jsontoh5",
+    "h5json.jsonstore",
+    "h5json.h5pystore",
     "h5json.schema",
-    "h5json.validator",
+    "h5json.apps",
 ]
 package-data = { "h5json.schema" = ["*.schema.json"] }
 platforms = ["any"]

setup.cfg

@@ -0,0 +1,6 @@
+[flake8]
+max-line-length = 120
+# E402: module level import not at top of file
+# C901: too complex
+# F401: unused exports are necessary in __init__.py
+ignore = E402, C901, F401

src/h5json/__init__.py

@@ -21,6 +21,14 @@
 from .hdf5dtype import getTypeResponse
 from .hdf5dtype import getItemSize
 from .hdf5dtype import createDataType
+from .objid import createObjId
+from .objid import getCollectionForId
+from .objid import isObjId
+from .objid import isS3ObjKey
+from .objid import getS3Key
+from .objid import getObjId
+from .objid import isSchema2Id
+from .objid import isRootObjId
 from .hdf5db import Hdf5db
 from . import _version
 

src/h5json/apps/h5tohs.py

@@ -0,0 +1,66 @@
+##############################################################################
+# Copyright by The HDF Group.                                                #
+# All rights reserved.                                                       #
+#                                                                            #
+# This file is part of H5Serv (HDF5 REST Server) Service, Libraries and      #
+# Utilities.  The full HDF5 REST Server copyright notice, including          #
+# terms governing use, modification, and redistribution, is contained in     #
+# the file COPYING, which can be found at the root of the source code        #
+# distribution tree.  If you do not have access to this file, you may        #
+# request a copy from [email protected].                                     #
+##############################################################################
+import sys
+import os.path as op
+import logging
+
+from h5json import Hdf5db
+from h5json.hsdsstore.hsds_writer import HSDSWriter
+from h5json.h5pystore.h5py_reader import H5pyReader
+
+
+def usage():
+    print(f"usage: {sys.argv[0]} [-h] [--nodata] <hdf5_file> <hsds_domain>")
+    sys.exit(0)
+
+
+def main():
+    no_data = False
+    filename = None
+    domain = None
+    for i in range(1, len(sys.argv)):
+        if sys.argv[i] in ("-h", "--help"):
+            usage()
+        elif sys.argv[i] == "--nodata":
+            no_data = True
+        elif filename is None:
+            filename = sys.argv[i]
+        elif domain is None:
+            domain = sys.argv[i]
+        else:
+            usage()
+
+    if domain is None:
+        usage()
+
+    # create logger
+    logfname = "h5tohs.log"
+    loglevel = logging.DEBUG
+    logging.basicConfig(filename=logfname, format='%(levelname)s %(asctime)s %(message)s', level=loglevel)
+    log = logging.getLogger()
+
+    # check that the input file exists
+    if not op.isfile(filename):
+        sys.exit(f"Cannot find file: {filename}")
+
+    log.info(f"h5tohs {filename}")
+
+    db = Hdf5db(app_logger=log)
+    db.writer = HSDSWriter(domain, no_data=no_data, app_logger=log)
+    db.reader = H5pyReader(filename, app_logger=log)
+    db.open()  # read HDF5 data into db
+
+    db.close()  # close will trigger write to HSDS
+
+
+if __name__ == "__main__":
+    main()

src/h5json/apps/h5tojson.py

@@ -0,0 +1,54 @@
+##############################################################################
+# Copyright by The HDF Group.                                                #
+# All rights reserved.                                                       #
+#                                                                            #
+# This file is part of H5Serv (HDF5 REST Server) Service, Libraries and      #
+# Utilities.  The full HDF5 REST Server copyright notice, including          #
+# terms governing use, modification, and redistribution, is contained in     #
+# the file COPYING, which can be found at the root of the source code        #
+# distribution tree.  If you do not have access to this file, you may        #
+# request a copy from [email protected].                                     #
+##############################################################################
+import sys
+import os.path as op
+import logging
+
+from h5json import Hdf5db
+from h5json.jsonstore.h5json_writer import H5JsonWriter
+from h5json.h5pystore.h5py_reader import H5pyReader
+
+
+def main():
+    if len(sys.argv) < 2 or sys.argv[1] in ("-h", "--help"):
+        print(f"usage: {sys.argv[0]} [-h] [--nodata] <hdf5_file>")
+        sys.exit(0)
+
+    no_data = False
+    filename = None
+    for i in range(1, len(sys.argv)):
+        if sys.argv[i] == "--nodata":
+            no_data = True
+        else:
+            filename = sys.argv[i]
+
+    # create logger
+    logfname = "h5tojson.log"
+    loglevel = logging.DEBUG
+    logging.basicConfig(filename=logfname, format='%(levelname)s %(asctime)s %(message)s', level=loglevel)
+    log = logging.getLogger()
+
+    # check that the input file exists
+    if not op.isfile(filename):
+        sys.exit(f"Cannot find file: {filename}")
+
+    log.info(f"h5tojson {filename}")
+
+    db = Hdf5db(app_logger=log)
+    db.reader = H5pyReader(filename, app_logger=log)
+    db.writer = H5JsonWriter(None, no_data=no_data, app_logger=log)
+    db.open()  # read HDF5 data into db
+    db.close()  # close will trigger write to json file
+
+
+if __name__ == "__main__":
+    main()

src/h5json/apps/jsontoh5.py

@@ -0,0 +1,58 @@
+##############################################################################
+# Copyright by The HDF Group.                                                #
+# All rights reserved.                                                       #
+#                                                                            #
+# This file is part of H5Serv (HDF5 REST Server) Service, Libraries and      #
+# Utilities.  The full HDF5 REST Server copyright notice, including          #
+# terms governing use, modification, and redistribution, is contained in     #
+# the file COPYING, which can be found at the root of the source code        #
+# distribution tree.  If you do not have access to this file, you may        #
+# request a copy from [email protected].                                     #
+##############################################################################
+import sys
+import os.path as op
+import logging
+
+from h5json import Hdf5db
+from h5json.h5pystore.h5py_writer import H5pyWriter
+from h5json.jsonstore.h5json_reader import H5JsonReader
+
+
+def main():
+    if len(sys.argv) < 3 or sys.argv[1] in ("-h", "--help"):
+        print(f"usage: {sys.argv[0]} [-h] [--nodata] <json_file> <h5_file>")
+        sys.exit(0)
+
+    no_data = False
+    json_filename = None
+    hdf5_filename = None
+    for i in range(1, len(sys.argv)):
+        if sys.argv[i] == "--nodata":
+            no_data = True
+        elif not json_filename:
+            json_filename = sys.argv[i]
+        else:
+            hdf5_filename = sys.argv[i]
+
+    # create logger
+    logfname = "jsontoh5.log"
+    loglevel = logging.DEBUG
+    logging.basicConfig(filename=logfname, format='%(levelname)s %(asctime)s %(message)s', level=loglevel)
+    log = logging.getLogger()
+
+    # check that the input file exists
+    if not op.isfile(json_filename):
+        sys.exit(f"Cannot find file: {json_filename}")
+
+    log.info(f"jsontoh5 {json_filename} to {hdf5_filename}")
+
+    db = Hdf5db(app_logger=log)
+    db.reader = H5JsonReader(json_filename, app_logger=log)
+    db.writer = H5pyWriter(hdf5_filename, no_data=no_data, app_logger=log)
+    db.open()  # read json data
+    # close should create everything the json reader read to the output file
+    db.close()
+
+
+if __name__ == "__main__":
+    main()