Merge pull request #108 from aperture-data/video_downloader_class_change

video downloader doesn't load to db, doesn't need loader.

Author: drewaogle

aperturedb/VideoDownloader.py

@@ -6,7 +6,7 @@
 import cv2
 import numpy as np
 
-from aperturedb import ParallelLoader
+from aperturedb import Parallelizer
 from aperturedb import CSVParser
 from aperturedb import ProgressBar
 
@@ -61,11 +61,11 @@ def validate(self):
             self.has_filename = True
 
 
-class VideoDownloader(ParallelLoader.ParallelLoader):
+class VideoDownloader(Parallelizer.Parallelizer):
 
-    def __init__(self, db, dry_run=False):
+    def __init__(self, ):
 
-        super().__init__(db, dry_run=dry_run)
+        super().__init__()
 
         self.type = "video"
 
@@ -81,7 +81,7 @@ def check_if_video_is_ok(self, filename, url):
             if a.isOpened() == False:
                 print("Video present but error reading it:", url)
                 return False
-        except:
+        except BaseException:
             print("Video present but error decoding:", url)
             return False
 
@@ -110,7 +110,7 @@ def download_video(self, url, filename):
                     print("Downloaded Video size error:", url)
                     os.remove(filename)
                     self.error_counter += 1
-            except:
+            except BaseException:
                 print("Downloaded Video cannot be decoded:", url)
                 os.remove(filename)
                 self.error_counter += 1
@@ -122,20 +122,14 @@ def download_video(self, url, filename):
 
     def worker(self, thid, generator, start, end):
 
-        if thid == 0 and self.stats:
-            pb = ProgressBar.ProgressBar("download_progress.txt")
-
         for i in range(start, end):
 
             url, filename = generator[i]
 
             self.download_video(url, filename)
 
             if thid == 0 and self.stats:
-                pb.update((i - start) / (end - start))
-
-        if thid == 0 and self.stats:
-            pb.update(1)
+                self.pb.update((i - start) / (end - start))
 
     def print_stats(self):
 
@@ -147,8 +141,8 @@ def print_stats(self):
         print("Avg download throughput (videos/s)):",
               1 / np.mean(times) * self.numthreads)
 
-        print("Total time(s):", self.ingestion_time)
+        print("Total time(s):", self.total_actions_time)
         print("Overall throughput (videos/s):",
-              self.total_elements / self.ingestion_time)
+              self.total_actions / self.total_actions_time)
         print("Total errors encountered:", self.error_counter)
         print("=============================================")