Merge pull request #434 from aperture-data/release-0.4.26

Release 0.4.26

Author: gsaluja9

aperturedb/Connector.py

@@ -182,6 +182,11 @@ def __del__(self):
             self.connected = False
 
     def _send_msg(self, data):
+        # aperturedb's param ADB_MAX_CONNECTION_MESSAGE_SIZE_MB = 256 by default
+        if len(data) > (256 * 2**20):
+            logger.warning(
+                "Message sent is larger than default for ApertureDB Server. Server may disconnect.")
+
         sent_len = struct.pack('@I', len(data))  # send size first
         x = self.conn.send(sent_len + data)
         return x == len(data) + 4

aperturedb/Images.py

@@ -13,6 +13,7 @@
 from aperturedb import Utils
 from aperturedb.Entities import Entities
 from aperturedb.Constraints import Constraints
+from aperturedb.ParallelQuery import execute_batch
 from ipywidgets import widgets
 from IPython.display import display, HTML
 import base64
@@ -222,7 +223,7 @@ def __init__(self, db, batch_size=100, response=None, **kwargs):
 
         # Blobs can be passed in addition the response.
         # This would mean that the images are already retrieved.
-        # This is useful for usage of the class dor it's utility methods.
+        # This is useful for usage of the class for it's utility methods.
         if "blobs" in kwargs:
             blobs = kwargs["blobs"]
             for i, id in enumerate(self.images_ids):
@@ -279,6 +280,9 @@ def __retrieve_batch(self, index):
             uniqueid = self.images_ids[idx]
             self.images[str(uniqueid)] = imgs[i]
 
+    def retrieve_polygons(self, index):
+        return self.__retrieve_polygons(index, constraints=None, tag_key="_label", tag_format="{}")
+
     def __retrieve_polygons(self, index, constraints, tag_key, tag_format):
 
         if index > len(self.images_ids):
@@ -327,37 +331,39 @@ def __retrieve_polygons(self, index, constraints, tag_key, tag_format):
                 fpq_res["list"].append(key)
 
         try:
-            res, _ = self.db_connector.query(query)
+            result, res, _ = execute_batch(
+                db=self.db_connector, q=query, blobs=[])
 
             polygons = []
             bounds = []
             tags = []
             meta = []
-            polys = res[1]["FindPolygon"]["entities"]
-            operations = self.query["operations"] if self.query and "operations" in self.query else [
-            ]
-            for poly in polys:
-                if tag_key and tag_format:
-                    tag = tag_format.format(poly[tag_key])
-                    tags.append(tag)
-                    meta.append(res[0]["FindImage"]["entities"][0])
-
-                bounds.append(poly["_bounds"])
-                converted = []
-                for vert in poly["_vertices"]:
-                    v = resolve(
-                        np.array(vert),
-                        res[0]["FindImage"]["entities"][0],
-                        operations)
-                    converted.append(v)
-                polygons.append(converted)
-
-            self.images_polygons[str(uniqueid)] = {
-                "bounds": bounds,
-                "polygons": polygons,
-                "tags": tags,
-                "meta": meta
-            }
+            if "entities" in res[1]["FindPolygon"]:
+                polys = res[1]["FindPolygon"]["entities"]
+                operations = self.query["operations"] if self.query and "operations" in self.query else [
+                ]
+                for poly in polys:
+                    if tag_key and tag_format:
+                        tag = tag_format.format(poly[tag_key])
+                        tags.append(tag)
+                        meta.append(res[0]["FindImage"]["entities"][0])
+
+                    bounds.append(poly["_bounds"])
+                    converted = []
+                    for vert in poly["_vertices"]:
+                        v = resolve(
+                            np.array(vert),
+                            res[0]["FindImage"]["entities"][0],
+                            operations)
+                        converted.append(v)
+                    polygons.append(converted)
+
+                self.images_polygons[str(uniqueid)] = {
+                    "bounds": bounds,
+                    "polygons": polygons,
+                    "tags": tags,
+                    "meta": meta
+                }
 
         except Exception as e:
             self.images_polygons[str(uniqueid)] = {
@@ -418,7 +424,8 @@ def __retrieve_bounding_boxes(self, index, constraints):
         uniqueid_str = str(uniqueid)
         self.images_bboxes[uniqueid_str] = {}
         try:
-            res, images = self.db_connector.query(query)
+            result, res, images = execute_batch(
+                db=self.db_connector, q=query, blobs=[])
             bboxes = []
             tags = []
             meta = []

aperturedb/ParallelQuery.py

@@ -58,26 +58,27 @@ def execute_batch(q: Commands, blobs: Blobs, db: Connector,
                 blobs_end = blobs_start + blobs_per_query
 
                 b_count = 0
-                for req, resp in zip(q[start:end], r[start:end]):
-                    for k in req:
-                        # Ref to https://docs.aperturedata.io/query_language/Reference/shared_command_parameters/blobs
-                        blobs_where_default_true = \
-                            k in ["FindImage", "FindBlob", "FindVideo"] and (
-                                "blobs" not in req[k] or req[k]["blobs"])
-                        blobs_where_default_false = \
-                            k in [
-                                "FindDescriptor", "FindBoundingBox"] and "blobs" in req[k] and req[k]["blobs"]
-                        if blobs_where_default_true or blobs_where_default_false:
-                            count = resp[k]["returned"]
-                            b_count += count
+                if issubclass(type(r), list):
+                    for req, resp in zip(q[start:end], r[start:end]):
+                        for k in req:
+                            # Ref to https://docs.aperturedata.io/query_language/Reference/shared_command_parameters/blobs
+                            blobs_where_default_true = \
+                                k in ["FindImage", "FindBlob", "FindVideo"] and (
+                                    "blobs" not in req[k] or req[k]["blobs"])
+                            blobs_where_default_false = \
+                                k in [
+                                    "FindDescriptor", "FindBoundingBox"] and "blobs" in req[k] and req[k]["blobs"]
+                            if blobs_where_default_true or blobs_where_default_false:
+                                count = resp[k]["returned"]
+                                b_count += count
 
                 try:
                     # The returned blobs need to be sliced to match the
                     # returned entities per command in query.
                     response_handler(
                         q[start:end],
                         blobs[blobs_start:blobs_end],
-                        r[start:end],
+                        r[start:end] if issubclass(type(r), list) else r,
                         b[blobs_returned:blobs_returned + b_count] if len(b) >= blobs_returned + b_count else None)
                 except BaseException as e:
                     logger.exception(e)
@@ -184,14 +185,23 @@ def update_refs(batched_commands):
                     values["image_ref"] = updates[values["image_ref"]]
                 if "video_ref" in values:
                     values["video_ref"] = updates[values["video_ref"]]
-                if "is_connected_to" in values and "_ref" in values["is_connected_to"]:
-                    values["is_connected_to"]["_ref"] = updates[values["is_connected_to"]["_ref"]]
+                if "is_connected_to" in values:
+                    if "ref" in values["is_connected_to"]:
+                        values["is_connected_to"]["ref"] = updates[values["is_connected_to"]["ref"]]
+                    for op in ["any", "all"]:
+                        if op in values["is_connected_to"]:
+                            for idx in range(len(values["is_connected_to"][op])):
+                                if "ref" in values["is_connected_to"][op][idx]:
+                                    values["is_connected_to"][op][idx]["ref"] = updates[values["is_connected_to"][op][idx]["ref"]]
                 if "connect" in values and "ref" in values["connect"]:
                     values["connect"]["ref"] = updates[values["connect"]["ref"]]
                 if "src" in values:
                     values["src"] = updates[values["src"]]
                 if "dst" in values:
                     values["dst"] = updates[values["dst"]]
+                if "ref" in values:
+                    values["ref"] = updates[values["ref"]]
+
             return batched_commands
 
         q = update_refs([cmd for query in data for cmd in query[0]])
@@ -274,8 +284,11 @@ def filter_per_group(group):
                     [v['status'] == 2 for i in r for k, v in filter_per_group(i)])
                 sq = 0
                 for i in range(0, len(r), self.commands_per_query):
-                    if all([v['status'] == 0 for j in r[i:i + self.commands_per_query] for k, v in filter_per_group(j)]):
-                        sq += 1
+                    # Some errors stop the whole query from being executed
+                    # https://docs.aperturedata.io/query_language/Overview/Responses#return-status
+                    if issubclass(type(r), list):
+                        if all([v['status'] == 0 for j in r[i:i + self.commands_per_query] for k, v in filter_per_group(j)]):
+                            sq += 1
                 worker_stats["succeeded_queries"] = sq
         else:
             query_time = 1

aperturedb/__init__.py

@@ -7,7 +7,7 @@
 
 logger = logging.getLogger(__name__)
 
-__version__ = "0.4.25"
+__version__ = "0.4.26"
 
 # set log level
 logger.setLevel(logging.DEBUG)

aperturedb/cli/ingest.py

@@ -258,10 +258,15 @@ def generate_embedding_csv_from_image_csv(
     metadata = []
     connection = []
     embeddings = []
+    errored = []
 
     for i in tqdm(range(data.sample_count)):
         d = data[i]
-        embeddings.append(np.frombuffer(d[1][1], dtype=np.float32))
+        nparr = np.frombuffer(d[1][1], dtype=np.float32)
+        if not nparr.any():
+            errored.append(d[0][0]["AddImage"]["properties"])
+
+        embeddings.append(nparr)
         descriptor_id = f"{set_name}_{os.path.basename(input_file)}_{i}"
         metadata.append({
             "filename": f"{filename}.npy",
@@ -282,3 +287,6 @@ def generate_embedding_csv_from_image_csv(
         f"{filename}_metadata.adb.csv", index=False)
     pd.json_normalize(connection).to_csv(
         f"{filename}_connection.adb.csv", index=False)
+    if(len(errored) > 0):
+        pd.json_normalize(errored).to_csv(
+            f"{filename}_errored.csv", index=False)