enable links to be initialized in post groups

Author: jreadey

hsds/group_dn.py

@@ -132,12 +132,19 @@ async def POST_Group(request):
     else:
         attrs = {}
 
+    if "links" in body:
+        # initialize links
+        links = body["links"]
+        log.debug(f"POST Group with links: {links}")
+    else:
+        links = {}
+
     group_json = {
         "id": group_id,
         "root": root_id,
         "created": now,
         "lastModified": now,
-        "links": {},
+        "links": links,
         "attributes": attrs,
     }
 
@@ -153,7 +160,7 @@ async def POST_Group(request):
     resp_json["root"] = root_id
     resp_json["created"] = group_json["created"]
     resp_json["lastModified"] = group_json["lastModified"]
-    resp_json["linkCount"] = 0
+    resp_json["linkCount"] = len(links)
     resp_json["attributeCount"] = len(attrs)
 
     resp = json_response(resp_json, status=201)

hsds/group_sn.py

@@ -13,6 +13,8 @@
 # group handler for service node of hsds cluster
 #
 
+import time
+
 from aiohttp.web_exceptions import HTTPBadRequest, HTTPForbidden, HTTPNotFound
 from json import JSONDecodeError
 
@@ -23,11 +25,12 @@
 from .util.authUtil import validateUserPassword
 from .util.domainUtil import getDomainFromRequest, isValidDomain
 from .util.domainUtil import getBucketForDomain, getPathForDomain, verifyRoot
-from .util.linkUtil import validateLinkName
+from .util.linkUtil import validateLinkName, getLinkClass
 from .servicenode_lib import getDomainJson, getObjectJson, validateAction
 from .servicenode_lib import getObjectIdByPath, getPathForObjectId
 from .servicenode_lib import createObject, createObjectByPath, deleteObject
 from . import hsds_logger as log
+from . import config
 
 
 async def GET_Group(request):
@@ -189,6 +192,7 @@ async def POST_Group(request):
     h5path = None
     creation_props = None
     attrs = None
+    links = None
 
     if request.has_body:
         try:
@@ -236,28 +240,66 @@ async def POST_Group(request):
                 creation_props = body["creationProperties"]
             if "attributes" in body:
                 attrs = body["attributes"]
+                if not isinstance(attrs, dict):
+                    msg = f"POST_Groups expected dict for for links, but got: {type(links)}"
+                    log.warn(msg)
+                    raise HTTPBadRequest(reason=msg)
                 log.debug(f"POST Group attributes: {attrs}")
+            if "links" in body:
+                links = body["links"]
+                if not isinstance(links, dict):
+                    msg = f"POST_Groups expected dict for for links, but got: {type(links)}"
+                    log.warn(msg)
+                    raise HTTPBadRequest(reason=msg)
+                # validate the links
+                now = time.time()
+
+                for title in links:
+                    try:
+                        validateLinkName(title)
+                        link_item = links[title]
+                        link_class = getLinkClass(link_item)
+                        if "class" in link_item:
+                            if link_class != link_item["class"]:
+                                msg = f"expected link class of: {link_class} but got {link_item}"
+                                log.warn(msg)
+                                raise HTTPBadRequest(reason=msg)
+                        else:
+                            link_item["class"] = link_class
+                        getLinkClass(link_item)
+                        if "created" in link_item:
+                            created = link_item["created"]
+                            # allow "pre-dated" attributes if recent enough
+                            predate_max_time = config.get("predate_max_time", default=10.0)
+                            if now - created > predate_max_time:
+                                link_item["created"] = created
+                            else:
+                                log.warn("stale created timestamp for link, ignoring")
+                            if "created" not in link_item:
+                                link_item["created"] = now
+
+                    except ValueError:
+                        raise HTTPBadRequest(reason="invalid link item")
+
+    kwargs = {"bucket": bucket}
+    if obj_id:
+        kwargs["obj_id"] = obj_id
+    if creation_props:
+        kwargs["creation_props"] = creation_props
+    if attrs:
+        kwargs["attrs"] = attrs
+    if links:
+        kwargs["links"] = links
 
     if parent_id:
-        kwargs = {"bucket": bucket, "parent_id": parent_id, "h5path": h5path}
-        if obj_id:
-            kwargs["obj_id"] = obj_id
-        if creation_props:
-            kwargs["creation_props"] = creation_props
-        if attrs:
-            kwargs["attrs"] = attrs
+        kwargs["parent_id"] = parent_id
+        kwargs["h5path"] = h5path
         if implicit:
             kwargs["implicit"] = True
         group_json = await createObjectByPath(app, **kwargs)
     else:
         # create an anonymous group
-        kwargs = {"bucket": bucket, "root_id": root_id}
-        if obj_id:
-            kwargs["obj_id"] = obj_id
-        if creation_props:
-            kwargs["creation_props"] = creation_props
-        if attrs:
-            kwargs["attrs"] = attrs
+        kwargs["root_id"] = root_id
         group_json = await createObject(app, **kwargs)
 
     log.debug(f"returning resp: {group_json}")

hsds/link_sn.py

@@ -13,7 +13,7 @@
 # service node of hsds cluster
 #
 
-from aiohttp.web_exceptions import HTTPBadRequest
+from aiohttp.web_exceptions import HTTPBadRequest, HTTPInternalServerError
 from json import JSONDecodeError
 
 from h5json.objid import isValidUuid, getCollectionForId
@@ -142,6 +142,10 @@ async def GET_Links(request):
 
         # mix in collection key, target and hrefs
         for link in links:
+            if "class" not in link:
+                log.error("expected to find class key in link")
+                raise HTTPInternalServerError()
+
             if link["class"] == "H5L_TYPE_HARD":
                 collection_name = getCollectionForId(link["id"])
                 link["collection"] = collection_name

hsds/servicenode_lib.py

@@ -1294,6 +1294,7 @@ async def createObject(app,
                        layout=None,
                        creation_props=None,
                        attrs=None,
+                       links=None,
                        bucket=None):
     """ create a group, ctype, or dataset object and return object json
         Determination on whether a group, ctype, or dataset is created is based on:
@@ -1319,6 +1320,8 @@ async def createObject(app,
         log.debug(f"    cprops: {creation_props}")
     if attrs:
         log.debug(f"    attrs: {attrs}")
+    if links:
+        log.debug(f"    links: {links}")
 
     if obj_id:
         log.debug(f"using client supplied id: {obj_id}")
@@ -1347,8 +1350,13 @@ async def createObject(app,
         attrs_json = {"attributes": attrs}
         attr_items = await getAttributesFromRequest(app, attrs_json, **kwargs)
         log.debug(f"got attr_items: {attr_items}")
-
         obj_json["attributes"] = attr_items
+    if links:
+        if collection != "groups":
+            msg = "links can only be used with groups"
+            log.warn(msg)
+            raise HTTPBadRequest(reason=msg)
+        obj_json["links"] = links
     log.debug(f"create {collection} obj, body: {obj_json}")
     dn_url = getDataNodeUrl(app, obj_id)
     req = f"{dn_url}/{collection}"
@@ -1368,6 +1376,7 @@ async def createObjectByPath(app,
                              layout=None,
                              creation_props=None,
                              attrs=None,
+                             links=None,
                              bucket=None):
 
     """ create an object at the designated path relative to the parent.
@@ -1394,6 +1403,12 @@ async def createObjectByPath(app,
         log.debug(f"    cprops: {creation_props}")
     if attrs:
         log.debug(f"    attrs: {attrs}")
+    if links:
+        log.debug(f"   links: {links}")
+        if obj_type:
+            msg = "only group objects can have links"
+            log.warn(msg)
+            raise HTTPBadRequest(reason=msg)
 
     root_id = getRootObjId(parent_id)
 
@@ -1474,6 +1489,8 @@ async def createObjectByPath(app,
                     kwargs["creation_props"] = creation_props
                 if attrs:
                     kwargs["attrs"] = attrs
+                if links:
+                    kwargs["links"] = links
                 if obj_id:
                     kwargs["obj_id"] = obj_id
             obj_json = await createObject(app, **kwargs)

tests/integ/group_test.py

@@ -316,7 +316,7 @@ def testPostWithLink(self):
         self.assertEqual(rspJson["linkCount"], 0)
         self.assertEqual(rspJson["attributeCount"], 0)
         new_group_id = rspJson["id"]
-        self.assertTrue(helper.validateId(rspJson["id"]))
+        self.assertTrue(helper.validateId(new_group_id))
         self.assertTrue(new_group_id != root_uuid)
 
         # get root group and verify link count is 1
@@ -418,6 +418,64 @@ def testPostWithAttributes(self):
         self.assertTrue("attributes") in rspJson
         self.assertEqual(len(rspJson["attributes"]), attr_count)
 
+    def testPostWithLinks(self):
+        # test POST with attribute initialization
+        print("testPostWithLinks", self.base_domain)
+        headers = helper.getRequestHeaders(domain=self.base_domain)
+
+        # get root id
+        req = helper.getEndpoint() + "/"
+        rsp = self.session.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+        root_uuid = rspJson["root"]
+        helper.validateId(root_uuid)
+
+        # some objects to link
+        link_count = 4
+        links = {}
+        req = helper.getEndpoint() + "/groups"
+
+        for i in range(link_count):
+            rsp = self.session.post(req, headers=headers)
+            self.assertEqual(rsp.status_code, 201)
+            rspJson = json.loads(rsp.text)
+            group_id = rspJson["id"]
+            self.assertTrue(helper.validateId(group_id))
+            links[f"obj_{i}"] = {"id": group_id}
+
+        # create new group
+        payload = {"links": links, "link": {"id": root_uuid, "name": "g1"}}
+        req = helper.getEndpoint() + "/groups"
+        rsp = self.session.post(req, data=json.dumps(payload), headers=headers)
+        self.assertEqual(rsp.status_code, 201)
+        rspJson = json.loads(rsp.text)
+        self.assertEqual(rspJson["linkCount"], link_count)
+        self.assertEqual(rspJson["attributeCount"], 0)
+        grp_id = rspJson["id"]
+        helper.validateId(grp_id)
+
+        # fetch all the links
+        req = helper.getEndpoint() + "/groups/" + grp_id + "/links"
+        rsp = self.session.get(req, headers=headers)
+        self.assertEqual(rsp.status_code, 200)
+        rspJson = json.loads(rsp.text)
+
+        self.assertTrue("links" in rspJson)
+        links_rsp = rspJson["links"]
+        self.assertEqual(len(links_rsp), link_count)
+        for i in range(link_count):
+            link_rsp = links_rsp[i]
+            self.assertTrue("class" in link_rsp)
+            self.assertEqual(link_rsp["class"], "H5L_TYPE_HARD")
+            self.assertTrue("id" in link_rsp)
+            self.assertTrue("title" in link_rsp)
+            self.assertEqual(link_rsp["title"], f"obj_{i}")
+            self.assertTrue("collection" in link_rsp)
+            self.assertEqual(link_rsp["collection"], "groups")
+            self.assertTrue("target" in link_rsp)
+            self.assertTrue("href" in link_rsp)
+
     def testPostWithPath(self):
         # test POST with implicit parent group creation
         print("testPostWithPath", self.base_domain)