Merge pull request #134 from aperture-data/release-0.2.9

gsaluja9 · web-flow · commit e7a13ec5c64d · 2022-09-12T12:30:14.000-04:00
Release 0.2.9
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -34,6 +34,13 @@ jobs:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
+      - name: Login to Google Cloud
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+          project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+          export_default_credentials: true
+
       - name: Run Tests
         env:
           AWS_ACCESS_KEY_ID:     ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
@@ -32,6 +32,13 @@ jobs:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
+      - name: Login to Google Cloud
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+          project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+          export_default_credentials: true
+
       - name: Run Tests
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -34,6 +34,13 @@ jobs:
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
 
+      - name: Login to Google Cloud
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          service_account_key: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+          project_id:          ${{ secrets.GCP_SERVICE_ACCOUNT_PROJECT_ID }}
+          export_default_credentials: true
+
       - name: Run Tests
         env:
           AWS_ACCESS_KEY_ID:     ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/aperturedb/ImageDataCSV.py b/aperturedb/ImageDataCSV.py
@@ -13,6 +13,7 @@
 HEADER_PATH   = "filename"
 HEADER_URL    = "url"
 HEADER_S3_URL = "s3_url"
+HEADER_GS_URL = "gs_url"
 PROPERTIES    = "properties"
 CONSTRAINTS   = "constraints"
 IMG_FORMAT    = "format"
@@ -37,6 +38,10 @@ class ImageDataCSV(CSVParser.CSVParser):
             OR
 
             ``s3_url``, ``PROP_NAME_1``, ... ``PROP_NAME_N``, ``constraint_PROP1``, ``format``
+
+            OR
+
+            ``gs_url``, ``PROP_NAME_1``, ... ``PROP_NAME_N``, ``constraint_PROP1``, ``format``
             ...
 
     Example csv file::
@@ -63,6 +68,10 @@ class ImageDataCSV(CSVParser.CSVParser):
     """
 
     def __init__(self, filename, check_image=True, n_download_retries=3):
+        self.loaders = [self.load_image, self.load_url,
+                        self.load_s3_url, self.load_gs_url]
+        self.source_types = [HEADER_PATH,
+                             HEADER_URL, HEADER_S3_URL, HEADER_GS_URL]
 
         super().__init__(filename)
 
@@ -76,13 +85,12 @@ def __init__(self, filename, check_image=True, n_download_retries=3):
                                  if x.startswith(CSVParser.CONTRAINTS_PREFIX)]
 
         self.source_type      = self.header[0]
-        loaders = [self.load_image, self.load_url, self.load_s3_url]
-        source_types = [HEADER_PATH, HEADER_URL, HEADER_S3_URL]
-        if self.source_type not in source_types:
+
+        if self.source_type not in self.source_types:
             logger.error("Source not recognized: " + self.source_type)
             raise Exception("Error loading image: " + filename)
         self.source_loader    = {
-            st: sl for st, sl in zip(source_types, loaders)
+            st: sl for st, sl in zip(self.source_types, self.loaders)
         }
 
         self.n_download_retries = n_download_retries
@@ -189,10 +197,36 @@ def load_s3_url(self, s3_url):
         logger.error("S3 ERROR:", s3_url)
         return False, None
 
+    def load_gs_url(self, gs_url):
+        retries = 0
+        from google.cloud import storage
+        client = storage.Client()
+        while True:
+            try:
+                bucket_name = gs_url.split("/")[2]
+                object_name = gs_url.split("gs://" + bucket_name + "/")[-1]
+
+                blob = client.bucket(bucket_name).blob(
+                    object_name).download_as_bytes()
+                imgbuffer = np.frombuffer(blob, dtype='uint8')
+                if self.check_image and not self.check_image_buffer(imgbuffer):
+                    logger.error("IMAGE ERROR: ", gs_url)
+                    return False, None
+                return True, blob
+            except:
+                if retries >= self.n_download_retries:
+                    break
+                logger.warning("WARNING: Retrying object:", gs_url)
+                retries += 1
+                time.sleep(2)
+
+        logger.error("GS ERROR:", gs_url)
+        return False, None
+
     def validate(self):
 
         self.header = list(self.df.columns.values)
 
-        if self.header[0] not in [HEADER_PATH, HEADER_URL, HEADER_S3_URL]:
+        if self.header[0] not in self.source_types:
             raise Exception(
-                "Error with CSV file field: filename. Must be first field")
+                f"Error with CSV file field: {self.header[0]}. Must be first field")
diff --git a/aperturedb/Images.py b/aperturedb/Images.py
@@ -149,7 +149,7 @@ def __retrieve_batch(self, index):
             find = {
                 "FindImage": {
                     "constraints": {
-                        "_uniqueid": ["==", self.images_ids[idx]]
+                        self.img_id_prop: ["==", self.images_ids[idx]]
                     },
                 }
             }
@@ -184,7 +184,7 @@ def __retrieve_polygons(self, index, constraints, tag_key, tag_format):
             "FindImage": {
                 "_ref": 1,
                 "constraints": {
-                    "_uniqueid": ["==", uniqueid]
+                    self.img_id_prop: ["==", uniqueid]
                 },
                 "blobs": False,
             }
@@ -255,7 +255,7 @@ def __retrieve_bounding_boxes(self, index):
             "FindImage": {
                 "_ref": 1,
                 "constraints": {
-                    "_uniqueid": ["==", uniqueid]
+                    self.img_id_prop: ["==", uniqueid]
                 },
                 "blobs": False,
             }
@@ -381,12 +381,12 @@ def search(self, constraints=None, operations=None, format=None, limit=None, sor
 
         self.search_result = response
 
-    def search_by_id(self, ids, id_key="id"):
+    def search_by_property(self, prop_key, prop_values):
         const = Constraints()
-        const.is_in(id_key, ids)
+        const.is_in(prop_key, prop_values)
         img_sort = {
-            "key": id_key,
-            "sequence": ids,
+            "key": prop_key,
+            "sequence": prop_values,
         }
         self.search(constraints=const, sort=img_sort)
 
@@ -429,7 +429,7 @@ def get_similar_images(self, set_name, n_neighbors):
                 "FindImage": {
                     "_ref": 1,
                     "constraints": {
-                        "_uniqueid":  ["==", uniqueid]
+                        self.img_id_prop:  ["==", uniqueid]
                     },
                     "blobs": False,
                 }
@@ -460,7 +460,7 @@ def get_similar_images(self, set_name, n_neighbors):
                         "ref": 1,
                     },
                     "results": {
-                        "list": ["_uniqueid"]
+                        "list": [self.img_id_prop]
                     }
                 }
             }]
@@ -581,9 +581,6 @@ def display(self, show_bboxes=False, show_polygons=False, limit=None, polygon_co
         if polygon_constraints:
             show_polygons = True
             self.images_polygons = {}
-            if "_uniqueid" in polygon_constraints.constraints.keys():
-                print("WARNING: don't use '_uniqueid' in polygon_constraints")
-                print("see https://github.com/aperture-data/athena/issues/532")
 
         for i in range(len(self.images_ids)):
 
@@ -616,9 +613,12 @@ def display(self, show_bboxes=False, show_polygons=False, limit=None, polygon_co
                     self.__retrieve_polygons(
                         i, polygon_constraints, polygon_tag_key, polygon_tag_format)
 
-                bounds = self.images_polygons[uniqueid]["bounds"]
-                polygons = self.images_polygons[uniqueid]["polygons"]
-                tags = self.images_polygons[uniqueid]["tags"]
+                bounds = self.images_polygons[uniqueid]["bounds"] if uniqueid in self.images_polygons else [
+                ]
+                polygons = self.images_polygons[uniqueid]["polygons"] if uniqueid in self.images_polygons else [
+                ]
+                tags = self.images_polygons[uniqueid]["tags"] if uniqueid in self.images_polygons else [
+                ]
 
                 for pi in range(len(polygons)):
                     self.__draw_polygon_and_tag(image, polygons[pi], tags[pi] if pi < len(
@@ -670,7 +670,7 @@ def get_properties(self, prop_list=[]):
                     "FindImage": {
                         "_ref": 1,
                         "constraints": {
-                            "_uniqueid": ["==", uniqueid]
+                            self.img_id_prop: ["==", uniqueid]
                         },
                         "blobs": False,
                         "results": {
diff --git a/aperturedb/Utils.py b/aperturedb/Utils.py
@@ -533,3 +533,52 @@ def count_descriptors_in_set(self, set_name):
             raise e
 
         return total
+
+    def remove_all_objects(self):
+
+        cmd = {"constraints": {"_uniqueid": ["!=", "0.0.0"]}}
+
+        # There is no DeleteDescriptor, but when the sets are removed
+        # all the descriptors are also removed.
+        queries = [
+            [{"DeleteDescriptorSet": cmd}],
+            # [{"DeleteDescriptor": cmd}],
+            [{"DeleteBoundingBox": cmd}],
+            [{"DeleteVideo": cmd}],
+            [{"DeleteImage": cmd}],
+            [{"DeleteBlob": cmd}],
+            [{"DeleteEntity": cmd}],
+        ]
+
+        try:
+            for q in queries:
+                response, _ = self.connector.query(q)
+                if not self.connector.last_query_ok():
+                    logger.error(self.connector.get_last_response_str())
+                    return False
+
+            response, _ = self.connector.query(
+                [{"GetSchema": {"refresh": True}}])
+
+            if not self.connector.last_query_ok():
+                logger.error(self.connector.get_last_response_str())
+                return False
+
+            entities    = response[0]["GetSchema"]["entities"]
+            connections = response[0]["GetSchema"]["connections"]
+
+            if entities is not None:
+                logger.error("Entities not removed completely")
+                logger.error(self.connector.get_last_response_str())
+                return False
+
+            if connections is not None:
+                logger.error("Connections not removed completely")
+                logger.error(self.connector.get_last_response_str())
+                return False
+
+        except BaseException as e:
+            logger.error(self.connector.get_last_response_str())
+            raise e
+
+        return True
diff --git a/aperturedb/__init__.py b/aperturedb/__init__.py
@@ -5,7 +5,7 @@
 
 logger = logging.getLogger(__name__)
 
-__version__ = "0.2.8"
+__version__ = "0.2.9"
 
 # set log level
 logger.setLevel(logging.DEBUG)
diff --git a/docs/requirements-documentation.txt b/docs/requirements-documentation.txt
@@ -3,5 +3,5 @@ pre-commit==2.17.0
 sphinx-autoapi==1.8.4
 sphinx-rtd-theme==1.0.0
 torch==1.10.2
-nbconvert==6.5.0
+nbconvert==7.0.0
 ipython==8.0.1
diff --git a/setup.py b/setup.py
@@ -5,10 +5,10 @@
 
 setuptools.setup(
     name="aperturedb",
-    version="0.2.8",
+    version="0.2.9",
     description="ApertureDB Client Module",
     install_requires=['protobuf>=3.20.0', 'scikit-image', 'image', 'requests', 'boto3',
-                      'opencv-python', 'numpy', 'matplotlib', 'pandas', 'kaggle'],
+                      'opencv-python', 'numpy', 'matplotlib', 'pandas', 'kaggle', 'google-cloud-storage'],
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/aperture-data/aperturedb-python",
diff --git a/test/conftest.py b/test/conftest.py
@@ -35,7 +35,8 @@ def insert_data_from_csv(in_csv_file, rec_count=-1):
             "./input/setB.adb.csv": DescriptorDataCSV,
             "./input/s3_images.adb.csv": ImageDataCSV,
             "./input/http_images.adb.csv": ImageDataCSV,
-            './input/bboxes-constraints.adb.csv': BBoxDataCSV
+            "./input/bboxes-constraints.adb.csv": BBoxDataCSV,
+            "./input/gs_images.adb.csv": ImageDataCSV
         }
 
         data = file_data_pair[in_csv_file](in_csv_file)
@@ -55,3 +56,8 @@ def insert_data_from_csv(in_csv_file, rec_count=-1):
 @pytest.fixture(scope="module")
 def utils(db):
     return Utils(db)
+
+
+@pytest.fixture()
+def images(insert_data_from_csv):
+    return insert_data_from_csv("./input/images.adb.csv")
diff --git a/test/generateInput.py b/test/generateInput.py
@@ -153,6 +153,30 @@ def generate_s3_images_csv(ip_file_csv):
     return df
 
 
+def generate_gs_images_csv(ip_file_csv):
+
+    images    = pd.read_csv(ip_file_csv, sep=",", header=None)
+
+    ids      = random.sample(range(1000000000), len(images))
+    age      = [int(100 * random.random()) for i in range(len(images))]
+    height   = [float(200 * random.random()) for i in range(len(images))]
+    license  = [x for x in range(len(images))]
+
+    df = pd.DataFrame()
+    df['gs_url']   = images
+    df["id"]       = ids
+    df['license']  = license
+    df["age"]      = age
+    df["height"]   = height
+    df["constraint_id"] = ids
+
+    df = df.sort_values("id")
+
+    df.to_csv("input/gs_images.adb.csv", index=False)
+
+    return df
+
+
 def generate_connections_csv(persons, images):
 
     connections  = list(product(images["id"][::100], persons["id"][::100]))
@@ -281,6 +305,7 @@ def main(params):
     images  = generate_images_csv(int(params.multiplier / 2))
     s3_imgs = generate_http_images_csv("input/sample_http_urls.csv")
     s3_imgs = generate_s3_images_csv("input/sample_s3_urls.csv")
+    generate_gs_images_csv("input/sample_gs_urls")
     connect = generate_connections_csv(persons, images)
     bboxes  = generate_bboxes_csv(images)
     bboxes_constraints = generate_bboxes_constraints_csv(images)
diff --git a/test/input/sample_gs_urls b/test/input/sample_gs_urls
@@ -0,0 +1,10 @@
+gs://aperturedb-testing/sample_images/1002318269_97db6e0975.jpg
+gs://aperturedb-testing/sample_images/10201275523_3e6ea67c7f.jpg
+gs://aperturedb-testing/sample_images/2297552664_1ee0e8855d.jpg
+gs://aperturedb-testing/sample_images/4140939180_07aeded917.jpg
+gs://aperturedb-testing/sample_images/4436463882_b96a3d9df9.jpg
+gs://aperturedb-testing/sample_images/4572998878_658b45226f.jpg
+gs://aperturedb-testing/sample_images/6985418911_df7747990d.jpg
+gs://aperturedb-testing/sample_images/7289030198_1f1ba44113.jpg
+gs://aperturedb-testing/sample_images/9329902958_0bc80ce58a.jpg
+gs://aperturedb-testing/sample_images/9506922316_c19019e38f.jpg
diff --git a/test/test_Data.py b/test/test_Data.py
diff --git a/test/test_Loaders.py b/test/test_Loaders.py
diff --git a/test/test_Utils.py b/test/test_Utils.py
diff --git a/test/test_torch_connector.py b/test/test_torch_connector.py
