Merge pull request #24 from aperture-data/release-0.0.12

Release 0.0.12

Author: luisremis

aperturedb/BlobLoader.py

@@ -43,7 +43,9 @@ def __getitem__(self, idx):
         filename      = self.df.loc[idx, BLOB_PATH]
         blob_ok, blob = self.load_blob(filename)
         if not blob_ok:
-            Exception("Error loading blob: " + filename )
+            print("Error loading blob: " + filename )
+            raise Exception("Error loading blob: " + filename )
+
         data["blob"] = blob
 
         properties  = self.parse_properties (self.df, idx)

aperturedb/DescriptorLoader.py

@@ -51,7 +51,8 @@ def __getitem__(self, idx):
         descriptor, desc_ok = self.load_descriptor(filename, index)
 
         if not desc_ok:
-            Exception("Error loading descriptor: " + filename + ":" + index)
+            print("Error loading descriptor: " + filename + ":" + index)
+            raise Exception("Error loading descriptor: " + filename + ":" + index)
 
         data = {
             "descriptor_blob": descriptor,

aperturedb/ImageLoader.py

@@ -40,14 +40,16 @@ def __init__(self, filename, check_image=True):
         self.props_keys       = [x for x in self.props_keys if x != IMG_FORMAT]
         self.constraints_keys = [x for x in self.header[1:] if x.startswith(CSVParser.CONTRAINTS_PREFIX) ]
 
+    # TODO: we can add support for slicing here.
     def __getitem__(self, idx):
 
         filename   = self.df.loc[idx, HEADER_PATH]
         data = {}
 
         img_ok, img = self.load_image(filename)
         if not img_ok:
-            Exception("Error loading image: " + filename )
+            print("Error loading image: " + filename )
+            raise Exception("Error loading image: " + filename )
 
         data["img_blob"] = img
         if self.format_given:

aperturedb/ParallelLoader.py

@@ -50,27 +50,30 @@ def worker(self, thid, generator, start, end):
 
         db = self.db.create_new_connection()
 
-        data_for_query = []
-
         if thid == 0 and self.stats:
             pb = ProgressBar.ProgressBar(self.loader_filename)
 
-        for i in range(start, end):
+        total_batches = (end - start) // self.batchsize
 
-            data_for_query.append(generator[i])
+        if (end - start) % self.batchsize > 0:
+            total_batches += 1
 
-            if len(data_for_query) >= self.batchsize:
-                self.do_batch(db, data_for_query)
-                data_for_query = []
-                if thid == 0 and self.stats:
-                    pb.update((i - start) / (end - start))
+        for i in range(total_batches):
 
-        if len(data_for_query) > 0:
-            self.do_batch(db, data_for_query)
-            data_for_query = []
+            batch_start = start + i * self.batchsize
+            batch_end   = min(batch_start + self.batchsize, end)
 
-        if thid == 0 and self.stats:
-            pb.update(1)
+            try:
+                # This can be done with slices instead of list comprehension.
+                # but required support from generator.
+                data_for_query = [ generator[idx]
+                                   for idx in range(batch_start, batch_end) ]
+                self.do_batch(db, data_for_query)
+            except:
+                self.error_counter += 1
+
+            if thid == 0 and self.stats:
+                pb.update((i + 1) / total_batches)
 
     def ingest(self, generator, batchsize=1, numthreads=1, stats=False):
 

aperturedb/VideoLoader.py

@@ -45,7 +45,7 @@ def __getitem__(self, idx):
         video_ok, video = self.load_video(filename)
         if not video_ok:
             print("Error loading video: " + filename )
-            return data
+            raise Exception("Error loading video: " + filename )
 
         data["video_blob"] = video
 

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name="aperturedb",
-    version="0.0.11",
+    version="0.0.12",
     description="ApertureDB Client Module",
     install_requires=['vdms', 'scikit-image', 'image',
                       'opencv-python', 'numpy', 'matplotlib', 'pandas'],