H5json

hsds/async_lib.py

@@ -15,11 +15,13 @@
 from aiohttp.client_exceptions import ClientError
 from aiohttp.web_exceptions import HTTPNotFound, HTTPInternalServerError
 from aiohttp.web_exceptions import HTTPForbidden
-from .util.idUtil import isValidUuid, isSchema2Id, getS3Key, isS3ObjKey
-from .util.idUtil import getObjId, isValidChunkId, getCollectionForId
+from h5json.hdf5dtype import getItemSize
+from h5json.hdf5dtype import createDataType
+from h5json.array_util import getNumElements, bytesToArray
+from h5json.objid import isValidUuid, isSchema2Id, getS3Key, isS3ObjKey
+from h5json.objid import getObjId, isValidChunkId, getCollectionForId
+
 from .util.chunkUtil import getDatasetId, getNumChunks, ChunkIterator
-from .util.hdf5dtype import getItemSize, createDataType
-from .util.arrayUtil import getNumElements, bytesToArray
 from .util.dsetUtil import getHyperslabSelection, getFilterOps, getChunkDims, getFilters
 from .util.dsetUtil import getDatasetLayoutClass, getDatasetLayout, getShapeDims
 from .util.storUtil import getStorKeys, putStorJSONObj, getStorJSONObj

hsds/attr_dn.py

@@ -19,12 +19,13 @@
 from aiohttp.web_exceptions import HTTPInternalServerError
 from aiohttp.web import json_response
 
+from h5json.hdf5dtype import getItemSize, createDataType
+from h5json.array_util import arrayToBytes, jsonToArray, decodeData
+from h5json.array_util import bytesToArray, bytesArrayToList, getNumElements
+
 from .util.attrUtil import validateAttributeName, isEqualAttr
-from .util.hdf5dtype import getItemSize, createDataType
 from .util.globparser import globmatch
 from .util.dsetUtil import getShapeDims
-from .util.arrayUtil import arrayToBytes, jsonToArray, decodeData
-from .util.arrayUtil import bytesToArray, bytesArrayToList, getNumElements
 from .util.domainUtil import isValidBucketName
 from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj
 from . import hsds_logger as log

hsds/attr_sn.py

@@ -18,17 +18,18 @@
 from aiohttp.web import StreamResponse
 from json import JSONDecodeError
 
+from h5json.hdf5dtype import validateTypeItem, getBaseTypeJson
+from h5json.hdf5dtype import createDataType, getItemSize
+from h5json.array_util import jsonToArray, getNumElements, bytesArrayToList
+from h5json.array_util import bytesToArray, arrayToBytes, decodeData, encodeData
+from h5json.objid import isValidUuid, getRootObjId
+
 from .util.httpUtil import getAcceptType, jsonResponse, getHref, getBooleanParam
 from .util.globparser import globmatch
-from .util.idUtil import isValidUuid, getRootObjId
 from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
 from .util.domainUtil import getDomainFromRequest, isValidDomain
 from .util.domainUtil import getBucketForDomain, verifyRoot
 from .util.attrUtil import validateAttributeName, getRequestCollectionName
-from .util.hdf5dtype import validateTypeItem, getBaseTypeJson
-from .util.hdf5dtype import createDataType, getItemSize
-from .util.arrayUtil import jsonToArray, getNumElements, bytesArrayToList
-from .util.arrayUtil import bytesToArray, arrayToBytes, decodeData, encodeData
 from .util.dsetUtil import getShapeDims
 
 from .servicenode_lib import getDomainJson, getObjectJson, validateAction

hsds/basenode.py

@@ -25,15 +25,17 @@
 from aiohttp.web_exceptions import HTTPInternalServerError
 from aiohttp.web_exceptions import HTTPServiceUnavailable
 
+
 from . import config
 from .util.httpUtil import http_get, http_post, jsonResponse
-from .util.idUtil import createNodeId, getNodeNumber, getNodeCount
 from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
 from .util.authUtil import isAdminUser
 from .util.k8sClient import getDnLabelSelector, getPodIps
+from .util.nodeUtil import createNodeId, getNodeNumber, getNodeCount
+
 from . import hsds_logger as log
 
-HSDS_VERSION = "0.9.2"
+HSDS_VERSION = "1.0.0"
 
 
 def getVersion():

hsds/chunk_crawl.py

@@ -24,16 +24,17 @@
 from aiohttp.web_exceptions import HTTPInternalServerError
 from aiohttp.client_exceptions import ClientError
 
+from h5json.hdf5dtype import createDataType
+from h5json.array_util import jsonToArray, getNumpyValue
+from h5json.array_util import getNumElements, arrayToBytes, bytesToArray
+
+from .util.nodeUtil import getDataNodeUrl, getNodeCount
 from .util.httpUtil import http_get, http_put, http_post, get_http_client
 from .util.httpUtil import isUnixDomainUrl
-from .util.idUtil import getDataNodeUrl, getNodeCount
-from .util.hdf5dtype import createDataType
 from .util.dsetUtil import getSliceQueryParam, getShapeDims
 from .util.dsetUtil import getSelectionShape, getChunkLayout
 from .util.chunkUtil import getChunkCoverage, getDataCoverage
 from .util.chunkUtil import getChunkIdForPartition, getQueryDtype
-from .util.arrayUtil import jsonToArray, getNumpyValue
-from .util.arrayUtil import getNumElements, arrayToBytes, bytesToArray
 
 from . import config
 from . import hsds_logger as log

hsds/chunk_dn.py

@@ -20,18 +20,20 @@
 from aiohttp.web_exceptions import HTTPNotFound, HTTPServiceUnavailable
 from aiohttp.web import json_response, StreamResponse
 
+from h5json.hdf5dtype import createDataType, getSubType
+from h5json.array_util import bytesToArray, arrayToBytes, getBroadcastShape
+from h5json.objid import getS3Key, isValidUuid
+
 from .util.httpUtil import request_read, getContentType
-from .util.arrayUtil import bytesToArray, arrayToBytes, getBroadcastShape
-from .util.idUtil import getS3Key, validateInPartition, isValidUuid
 from .util.storUtil import isStorObj, deleteStorObj
-from .util.hdf5dtype import createDataType, getSubType
 from .util.dsetUtil import getSelectionList, getChunkLayout, getShapeDims
 from .util.dsetUtil import getSelectionShape, getChunkInitializer
 from .util.chunkUtil import getChunkIndex, getDatasetId, chunkQuery
 from .util.chunkUtil import chunkWriteSelection, chunkReadSelection
 from .util.chunkUtil import chunkWritePoints, chunkReadPoints
 from .util.domainUtil import isValidBucketName
 from .util.boolparser import BooleanParser
+from .util.nodeUtil import validateInPartition
 from .datanode_lib import get_metadata_obj, get_chunk, save_chunk
 
 from . import hsds_logger as log

hsds/chunk_sn.py

@@ -25,19 +25,19 @@
 from aiohttp.web_exceptions import HTTPConflict, HTTPInternalServerError
 from aiohttp.web import StreamResponse
 
+from h5json.hdf5dtype import getItemSize, getDtypeItemSize, getSubType, createDataType
+from h5json.array_util import bytesArrayToList, jsonToArray, getNumElements, arrayToBytes
+from h5json.array_util import bytesToArray, squeezeArray, getBroadcastShape
+from h5json.objid import isValidUuid
+
 from .util.httpUtil import getHref, getAcceptType, getContentType
 from .util.httpUtil import request_read, jsonResponse, isAWSLambda
-from .util.idUtil import isValidUuid
 from .util.domainUtil import getDomainFromRequest, isValidDomain
 from .util.domainUtil import getBucketForDomain
-from .util.hdf5dtype import getItemSize, getDtypeItemSize, getSubType, createDataType
 from .util.dsetUtil import isNullSpace, isScalarSpace, get_slices, getShapeDims
 from .util.dsetUtil import isExtensible, getSelectionPagination
 from .util.dsetUtil import getSelectionShape, getDsetMaxDims, getChunkLayout
 from .util.chunkUtil import getNumChunks, getChunkIds, getChunkId
-from .util.arrayUtil import bytesArrayToList, jsonToArray
-from .util.arrayUtil import getNumElements, arrayToBytes, bytesToArray
-from .util.arrayUtil import squeezeArray, getBroadcastShape
 from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
 from .servicenode_lib import getDsetJson, validateAction
 from .dset_lib import getSelectionData, getParser, extendShape

hsds/chunklocator.py

@@ -5,7 +5,9 @@
 import numpy as np
 from . import config
 from . import hsds_logger as log
-from .util.arrayUtil import bytesArrayToList, getNumElements
+
+from h5json.array_util import bytesArrayToList, getNumElements
+
 from .util.dsetUtil import getSelectionList, getSelectionShape
 
 

hsds/ctype_dn.py

@@ -18,7 +18,8 @@
 from aiohttp.web_exceptions import HTTPInternalServerError
 from aiohttp.web import json_response
 
-from .util.idUtil import isValidUuid, validateUuid
+from h5json.objid import isValidUuid, validateUuid
+
 from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj
 from .datanode_lib import delete_metadata_obj, check_metadata_obj
 from .util.domainUtil import isValidBucketName
@@ -33,7 +34,7 @@ async def GET_Datatype(request):
     params = request.rel_url.query
     ctype_id = get_obj_id(request)
 
-    if not isValidUuid(ctype_id, obj_class="type"):
+    if not isValidUuid(ctype_id, obj_class="datatypes"):
         log.error(f"Unexpected type_id: {ctype_id}")
         raise HTTPInternalServerError()
 
@@ -90,7 +91,7 @@ async def POST_Datatype(request):
         raise HTTPBadRequest(reason=msg)
 
     ctype_id = get_obj_id(request, body=body)
-    if not isValidUuid(ctype_id, obj_class="datatype"):
+    if not isValidUuid(ctype_id, obj_class="datatypes"):
         log.error("Unexpected type_id: {ctype_id}")
         raise HTTPInternalServerError()
 

hsds/ctype_sn.py

@@ -16,15 +16,17 @@
 
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPGone
 from json import JSONDecodeError
+
+from h5json.hdf5dtype import validateTypeItem, getBaseTypeJson
+from h5json.objid import isValidUuid
+
 from .util.httpUtil import getHref, respJsonAssemble, getBooleanParam
 from .util.httpUtil import jsonResponse
-from .util.idUtil import isValidUuid
 from .util.linkUtil import validateLinkName
 from .util.authUtil import getUserPasswordFromRequest, aclCheck
 from .util.authUtil import validateUserPassword
 from .util.domainUtil import getDomainFromRequest, getPathForDomain, isValidDomain
 from .util.domainUtil import getBucketForDomain, verifyRoot
-from .util.hdf5dtype import validateTypeItem, getBaseTypeJson
 from .servicenode_lib import getDomainJson, getObjectJson, validateAction
 from .servicenode_lib import getObjectIdByPath, getPathForObjectId
 from .servicenode_lib import createObject, createObjectByPath, deleteObject
@@ -49,7 +51,7 @@ async def GET_Datatype(request):
         include_attrs = True
 
     if ctype_id:
-        if not isValidUuid(ctype_id, "Type"):
+        if not isValidUuid(ctype_id, "datatypes"):
             msg = f"Invalid type id: {ctype_id}"
             log.warn(msg)
             raise HTTPBadRequest(reason=msg)
@@ -60,7 +62,7 @@ async def GET_Datatype(request):
         group_id = None
         if "grpid" in params:
             group_id = params["grpid"]
-            if not isValidUuid(group_id, "Group"):
+            if not isValidUuid(group_id, "groups"):
                 msg = f"Invalid parent group id: {group_id}"
                 log.warn(msg)
                 raise HTTPBadRequest(reason=msg)
@@ -101,7 +103,7 @@ async def GET_Datatype(request):
         # throws 404 if not found
         kwargs = {"bucket": bucket, "domain": domain}
         ctype_id, domain, _ = await getObjectIdByPath(app, group_id, h5path, **kwargs)
-        if not isValidUuid(ctype_id, "Datatype"):
+        if not isValidUuid(ctype_id, "datatypes"):
             msg = f"No datatype exist with the path: {h5path}"
             log.warn(msg)
             raise HTTPGone()
@@ -271,7 +273,7 @@ async def DELETE_Datatype(request):
         msg = "Missing committed type id"
         log.warn(msg)
         raise HTTPBadRequest(reason=msg)
-    if not isValidUuid(ctype_id, "Type"):
+    if not isValidUuid(ctype_id, "datatypes"):
         msg = f"Invalid committed type id: {ctype_id}"
         log.warn(msg)
         raise HTTPBadRequest(reason=msg)

hsds/datanode.py

@@ -17,10 +17,11 @@
 import traceback
 from aiohttp.web import run_app
 
+from h5json.objid import isValidUuid, isSchema2Id, getCollectionForId
+from h5json.objid import isRootObjId
+
 from . import config
 from .util.lruCache import LruCache
-from .util.idUtil import isValidUuid, isSchema2Id, getCollectionForId
-from .util.idUtil import isRootObjId
 from .util.httpUtil import isUnixDomainUrl, bindToSocket, getPortFromUrl
 from .util.httpUtil import jsonResponse, release_http_client
 from .util.storUtil import setBloscThreads, getBloscThreads

hsds/datanode_lib.py

@@ -19,9 +19,14 @@
 from aiohttp.web_exceptions import HTTPGone, HTTPInternalServerError
 from aiohttp.web_exceptions import HTTPNotFound, HTTPForbidden
 from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPBadRequest
-from .util.idUtil import validateInPartition, getS3Key, isValidUuid
-from .util.idUtil import isValidChunkId, getDataNodeUrl, isSchema2Id
-from .util.idUtil import getRootObjId, isRootObjId
+
+from h5json.hdf5dtype import createDataType
+from h5json.array_util import arrayToBytes, bytesToArray, jsonToArray
+from h5json.objid import getS3Key, isValidUuid
+from h5json.objid import isValidChunkId, isSchema2Id
+from h5json.objid import getRootObjId, isRootObjId
+
+from .util.nodeUtil import getDataNodeUrl
 from .util.storUtil import getStorJSONObj, putStorJSONObj, putStorBytes
 from .util.storUtil import getStorBytes, isStorObj, deleteStorObj, getHyperChunks
 from .util.storUtil import getBucketFromStorURI, getKeyFromStorURI, getURIFromKey
@@ -31,8 +36,7 @@
 from .util.dsetUtil import getChunkLayout, getFilterOps, getShapeDims
 from .util.dsetUtil import getChunkInitializer, getSliceQueryParam, getFilters
 from .util.chunkUtil import getDatasetId, getChunkSelection, getChunkIndex
-from .util.arrayUtil import arrayToBytes, bytesToArray, jsonToArray
-from .util.hdf5dtype import createDataType
+from .util.nodeUtil import validateInPartition
 from .util.rangegetUtil import ChunkLocation, chunkMunge, getHyperChunkIndex, getHyperChunkFactors
 from .util.timeUtil import getNow
 from . import config

hsds/domain_crawl.py

@@ -18,8 +18,10 @@
 from aiohttp.web_exceptions import HTTPServiceUnavailable, HTTPConflict, HTTPBadRequest
 from aiohttp.web_exceptions import HTTPInternalServerError, HTTPNotFound, HTTPGone
 
+from h5json.objid import getCollectionForId
+
+from .util.nodeUtil import getDataNodeUrl
 from .util.httpUtil import isOK
-from .util.idUtil import getCollectionForId, getDataNodeUrl
 from .util.globparser import globmatch
 from .servicenode_lib import getObjectJson, getAttributes, putAttributes, getLinks, putLinks
 from . import hsds_logger as log

hsds/domain_dn.py

@@ -18,7 +18,7 @@
 
 from .util.authUtil import getAclKeys
 from .util.domainUtil import isValidDomain, getBucketForDomain
-from .util.idUtil import validateInPartition
+from .util.nodeUtil import validateInPartition
 from .util.timeUtil import getNow
 from .datanode_lib import get_metadata_obj, save_metadata_obj
 from .datanode_lib import delete_metadata_obj, check_metadata_obj

hsds/domain_sn.py

@@ -22,11 +22,13 @@
 from aiohttp.web_exceptions import HTTPConflict, HTTPServiceUnavailable
 from aiohttp.web import json_response
 
+from h5json.objid import createObjId, getCollectionForId
+from h5json.objid import isValidUuid, isSchema2Id
+
+from .util.nodeUtil import getNodeCount, getDataNodeUrl
 from .util.httpUtil import getObjectClass, http_post, http_put, http_delete
 from .util.httpUtil import getHref, respJsonAssemble
 from .util.httpUtil import jsonResponse
-from .util.idUtil import getDataNodeUrl, createObjId, getCollectionForId
-from .util.idUtil import isValidUuid, isSchema2Id, getNodeCount
 from .util.authUtil import getUserPasswordFromRequest, aclCheck, isAdminUser
 from .util.authUtil import validateUserPassword, getAclKeys
 from .util.domainUtil import getParentDomain, getDomainFromRequest
@@ -983,7 +985,7 @@ async def PUT_Domain(request):
 
     if not is_folder and not linked_json:
         # create a root group for the new domain
-        root_id = createObjId("roots")
+        root_id = createObjId("groups")
         log.debug(f"new root group id: {root_id}")
         group_json = {"id": root_id, "root": root_id, "domain": domain}
         log.debug(f"create group for domain, body: {group_json}")

hsds/dset_dn.py

@@ -17,8 +17,8 @@
 from aiohttp.web_exceptions import HTTPInternalServerError
 from aiohttp.web import json_response
 
+from h5json.objid import isValidUuid, validateUuid
 
-from .util.idUtil import isValidUuid, validateUuid
 from .util.domainUtil import isValidBucketName
 from .util.timeUtil import getNow
 from .datanode_lib import get_obj_id, check_metadata_obj, get_metadata_obj
@@ -33,7 +33,7 @@ async def GET_Dataset(request):
     params = request.rel_url.query
     dset_id = get_obj_id(request)
 
-    if not isValidUuid(dset_id, obj_class="dataset"):
+    if not isValidUuid(dset_id, obj_class="datasets"):
         log.error(f"Unexpected dataset_id: {dset_id}")
         raise HTTPInternalServerError()
     if "bucket" in params:
@@ -94,7 +94,7 @@ async def POST_Dataset(request):
         raise HTTPBadRequest(reason=msg)
 
     dset_id = get_obj_id(request, body=body)
-    if not isValidUuid(dset_id, obj_class="dataset"):
+    if not isValidUuid(dset_id, obj_class="datasets"):
         log.error(f"Unexpected dataset_id: {dset_id}")
         raise HTTPInternalServerError()
 
@@ -176,7 +176,7 @@ async def DELETE_Dataset(request):
     dset_id = request.match_info.get("id")
     log.info(f"DELETE dataset: {dset_id}")
 
-    if not isValidUuid(dset_id, obj_class="dataset"):
+    if not isValidUuid(dset_id, obj_class="datasets"):
         log.error(f"Unexpected dataset id: {dset_id}")
         raise HTTPInternalServerError()
 
@@ -220,7 +220,7 @@ async def PUT_DatasetShape(request):
     params = request.rel_url.query
     dset_id = request.match_info.get("id")
 
-    if not isValidUuid(dset_id, obj_class="dataset"):
+    if not isValidUuid(dset_id, obj_class="datasets"):
         log.error(f"Unexpected dset_id: {dset_id}")
         raise HTTPInternalServerError()