Merge branch 'new_timing'

Author: RJT1990

sotabencheval/image_classification/imagenet.py

@@ -328,8 +328,13 @@ def save(self):
         self.get_results()
 
         if not self.cached_results:
-            self.speed_mem_metrics['Evaluation Time'] = len(self.outputs) / (time.time() - self.init_time)
+            exec_speed = (time.time() - self.init_time)
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = len(self.outputs) / exec_speed
+            self.speed_mem_metrics['Tasks'] = len(self.outputs)
+            self.speed_mem_metrics['Evaluation Time'] = exec_speed
         else:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = None
 
         return BenchmarkResult(

sotabencheval/machine_translation/wmt.py

@@ -104,8 +104,13 @@ def save(self):
         dataset = self._get_dataset_name()
 
         if not self.cached_results:
-            self.speed_mem_metrics['Evaluation Time'] = len(self.metrics.answers) / (time.time() - self.init_time)
+            exec_speed = (time.time() - self.init_time)
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = len(self.metrics.answers) / exec_speed
+            self.speed_mem_metrics['Tasks'] = len(self.metrics.answers)
+            self.speed_mem_metrics['Evaluation Time'] = exec_speed
         else:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = None
 
         return super().save(dataset=dataset)

sotabencheval/object_detection/coco.py

@@ -289,8 +289,13 @@ def save(self):
 
         if not self.cached_results:
             unique_image_ids = set([d['image_id'] for d in self.detections])
-            self.speed_mem_metrics['Evaluation Time'] = len(unique_image_ids) / (time.time() - self.init_time)
+            exec_speed = (time.time() - self.init_time)
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = len(unique_image_ids) / exec_speed
+            self.speed_mem_metrics['Tasks'] = len(unique_image_ids)
+            self.speed_mem_metrics['Evaluation Time'] = exec_speed
         else:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = None
 
         return BenchmarkResult(

sotabencheval/question_answering/squad.py

@@ -59,8 +59,13 @@ def save(self):
         dataset = "SQuAD{} dev".format(self.metrics.version.value[1:])
 
         if not self.cached_results:
-            self.speed_mem_metrics['Evaluation Time'] = len(self.metrics.answers) / (time.time() - self.init_time)
+            exec_speed = (time.time() - self.init_time)
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = len(self.metrics.answers) / exec_speed
+            self.speed_mem_metrics['Tasks'] = len(self.metrics.answers)
+            self.speed_mem_metrics['Evaluation Time'] = exec_speed
         else:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = None
 
         return super().save(dataset=dataset)

sotabencheval/semantic_segmentation/ade20k.py

@@ -266,8 +266,12 @@ def save(self):
         self.get_results()
 
         if not self.cached_results:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = (time.time() - self.init_time)
         else:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = None
 
         return BenchmarkResult(

sotabencheval/semantic_segmentation/pascalvoc.py

@@ -240,8 +240,12 @@ def save(self):
         self.get_results()
 
         if not self.cached_results:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = (time.time() - self.init_time)
         else:
+            self.speed_mem_metrics['Tasks / Evaluation Time'] = None
+            self.speed_mem_metrics['Tasks'] = None
             self.speed_mem_metrics['Evaluation Time'] = None
 
         return BenchmarkResult(

sotabencheval/version.py

@@ -15,6 +15,6 @@ def __repr__(self):
             f"build={self.build})"
         )
 
-version = Version(0, 0, 32)
+version = Version(0, 0, 33)
 
 __version__ = str(version)