Merge pull request #6 from paperswithcode/new_timing

new timing

Author: RJT1990

sotabencheval/__pycache__/__init__.cpython-36.pyc

@@ -2,7 +2,7 @@
 
 from sotabenchapi.client import Client
 from sotabenchapi.core import BenchmarkResult
-from sotabencheval.utils import is_server, AverageMeter
+from sotabencheval.utils import is_server
 from sotabencheval.core.cache import cache_value
 
 
@@ -25,14 +25,8 @@ def __init__(self,
         self.results = None
         self._cache_exists = None
 
-        self.inference_time = AverageMeter()
-        self.start_time = time.time()
-        self.speed_mem_metrics = {
-            'Tasks Per Second (Partial)': None,
-            'Tasks Per Second (Total)': None,
-            'Memory Allocated (Partial)': None,
-            'Memory Allocated (Total)': None
-        }
+        self.init_time = time.time()
+        self.speed_mem_metrics = {}
 
     @property
     def cache_exists(self):
@@ -92,6 +86,9 @@ def cache_exists(self):
     def cache_values(self, **kwargs):
         return cache_value(kwargs)
 
+    def reset_time(self):
+        self.init_time = time.time()
+
     def save(self, **kwargs):
         """
         Calculate results and then put into a BenchmarkResult object

sotabencheval/__pycache__/version.cpython-36.pyc

@@ -157,14 +157,8 @@ def __init__(self,
         self.batch_hash = None
         self.cached_results = False
 
-        self.inference_time = AverageMeter()
-        self.start_time = time.time()
-        self.speed_mem_metrics = {
-            'Tasks Per Second (Partial)': None,
-            'Tasks Per Second (Total)': None,
-            'Memory Allocated (Partial)': None,
-            'Memory Allocated (Total)': None
-        }
+        self.speed_mem_metrics = {}
+        self.init_time = time.time()
 
     @property
     def cache_exists(self):
@@ -207,8 +201,6 @@ def cache_exists(self):
         if not is_server():  # we only check the cache on the server
             return None
 
-        self.speed_mem_metrics['Tasks Per Second (Partial)'] = len(self.outputs)/self.inference_time.sum
-
         client = Client.public()
         cached_res = client.get_results_by_run_hash(self.batch_hash)
         if cached_res:
@@ -237,18 +229,6 @@ def load_targets(self):
         with open(os.path.join(self.root, 'imagenet_val_targets.pkl'), 'rb') as handle:
             self.targets = pickle.load(handle)
 
-    def update_inference_time(self):
-
-        if not self.outputs and self.inference_time.count < 1:
-            # assuming this is the first time the evaluator is called
-            self.inference_time.update(time.time() - self.start_time)
-        elif not self.outputs and self.inference_time.count > 0:
-            # assuming the user has cleared outputs, and is then readding (evaluation post batching)
-            pass
-        else:
-            # if there are outputs and the inference time count is > 0
-            self.inference_time.update(time.time() - self.start_time)
-
     def add(self, output_dict: dict):
         """
         Updates the evaluator with new results
@@ -270,7 +250,6 @@ def add(self, output_dict: dict):
             print('Empty output_dict; will not process')
             return
 
-        self.update_inference_time()
         self.outputs = dict(list(self.outputs.items()) + list(output_dict.items()))
 
         for i, dict_key in enumerate(output_dict.keys()):
@@ -288,8 +267,6 @@ def add(self, output_dict: dict):
             self.batch_hash = calculate_batch_hash(hash_dict)
             self.first_batch_processed = True
 
-        self.start_time = time.time()
-
     def get_results(self):
         """
         Gets the results for the evaluator. This method only runs if predictions for all 5,000 ImageNet validation
@@ -330,11 +307,13 @@ def get_results(self):
             self.top5.update(prec5, 1)
 
         self.results = {'Top 1 Accuracy': self.top1.avg, 'Top 5 Accuracy': self.top5.avg}
-        self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.outputs) / self.inference_time.sum
         self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 
+    def reset_time(self):
+        self.init_time = time.time()
+
     def save(self):
         """
         Calculate results and then puts into a BenchmarkResult object
@@ -348,6 +327,11 @@ def save(self):
         # recalculate to ensure no mistakes made during batch-by-batch metric calculation
         self.get_results()
 
+        if not self.cached_results:
+            self.speed_mem_metrics['Evaluation Time'] = len(self.outputs) / (time.time() - self.init_time)
+        else:
+            self.speed_mem_metrics['Evaluation Time'] = None
+
         return BenchmarkResult(
             task=self.task,
             config={},

sotabencheval/core/evaluator.py

@@ -8,6 +8,7 @@
 from enum import Enum
 import time
 
+
 class WMTDataset(Enum):
     News2014 = "newstest2014"
     News2019 = "newstest2019"
@@ -52,8 +53,6 @@ def __init__(self,
 
         self.metrics = TranslationMetrics(self.source_dataset_path, self.target_dataset_path)
 
-        self.start_time = time.time()
-
     def _get_source_dataset_filename(self):
         if self.dataset == WMTDataset.News2014:
             other_lang = self.source_lang.value if self.target_lang == Language.English else self.target_lang.value
@@ -78,47 +77,36 @@ def _get_dataset_name(self):
         ds_names = {WMTDataset.News2014: "WMT2014", WMTDataset.News2019: "WMT2019"}
         return "{0} {1}-{2}".format(ds_names.get(self.dataset), self.source_lang.fullname, self.target_lang.fullname)
 
-    def update_inference_time(self):
-
-        if not self.metrics._results and self.inference_time.count < 1:
-            # assuming this is the first time the evaluator is called
-            self.inference_time.update(time.time() - self.start_time)
-        elif not self.metrics._results and self.inference_time.count > 0:
-            # assuming the user has reset outputs, and is then readding (evaluation post batching)
-            pass
-        else:
-            # if there are outputs and the inference time count is > 0
-            self.inference_time.update(time.time() - self.start_time)
 
     def add(self, answers: Dict[str, str]):
 
-        self.update_inference_time()
-
         self.metrics.add(answers)
 
         if not self.first_batch_processed and self.metrics.has_data:
-            self.speed_mem_metrics['Tasks Per Second (Partial)'] = len(self.metrics.answers) / self.inference_time.sum
             self.batch_hash = calculate_batch_hash(
                 self.cache_values(answers=self.metrics.answers,
                                   metrics=self.metrics.get_results(ignore_missing=True))
             )
             self.first_batch_processed = True
 
-        self.start_time = time.time()
-
     def reset(self):
         self.metrics.reset()
 
     def get_results(self):
         if self.cached_results:
             return self.results
         self.results = self.metrics.get_results()
-        self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.metrics.answers) / self.inference_time.sum
         self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 
     def save(self):
         dataset = self._get_dataset_name()
+
+        if not self.cached_results:
+            self.speed_mem_metrics['Evaluation Time'] = len(self.metrics.answers) / (time.time() - self.init_time)
+        else:
+            self.speed_mem_metrics['Evaluation Time'] = None
+
         return super().save(dataset=dataset)
 

sotabencheval/image_classification/imagenet.py

@@ -9,7 +9,7 @@
 import time
 
 from sotabencheval.utils import calculate_batch_hash, extract_archive, change_root_if_server, is_server
-from sotabencheval.utils import AverageMeter, get_max_memory_allocated
+from sotabencheval.utils import get_max_memory_allocated
 from sotabencheval.object_detection.coco_eval import CocoEvaluator
 from sotabencheval.object_detection.utils import get_coco_metrics
 
@@ -107,15 +107,9 @@ def __init__(self,
         self.batch_hash = None
         self.cached_results = False
 
-        self.inference_time = AverageMeter()
-        self.start_time = time.time()
-        self.speed_mem_metrics = {
-            'Tasks Per Second (Partial)': None,
-            'Tasks Per Second (Total)': None,
-            'Memory Allocated (Partial)': None,
-            'Memory Allocated (PartTotalial)': None
-        }
+        self.speed_mem_metrics = {}
 
+        self.init_time = time.time()
 
     def _download(self, annFile):
         if not os.path.isdir(annFile):
@@ -172,9 +166,6 @@ def cache_exists(self):
         if not is_server():  # we only check the cache on the server
             return None
 
-        unique_image_ids = set([d['image_id'] for d in self.detections])
-        self.speed_mem_metrics['Tasks Per Second (Partial)'] = len(unique_image_ids)/self.inference_time.sum
-
         client = Client.public()
         cached_res = client.get_results_by_run_hash(self.batch_hash)
         if cached_res:
@@ -223,18 +214,6 @@ def cache_values(self, annotations, metrics):
 
         return new_annotations + [metrics]
 
-    def update_inference_time(self):
-
-        if not self.detections and self.inference_time.count < 1:
-            # assuming this is the first time the evaluator is called
-            self.inference_time.update(time.time() - self.start_time)
-        elif not self.detections and self.inference_time.count > 0:
-            # assuming the user has reset outputs, and is then readding (evaluation post batching)
-            pass
-        else:
-            # if there are outputs and the inference time count is > 0
-            self.inference_time.update(time.time() - self.start_time)
-
     def add(self, detections: list):
         """
         Update the evaluator with new detections
@@ -258,8 +237,6 @@ def add(self, detections: list):
                 110.14895629882812, 278.2847595214844], 'score': 0.999152421951294, 'category_id': 1}])
         """
 
-        self.update_inference_time()
-
         self.detections.extend(detections)
 
         self.coco_evaluator.update(detections)
@@ -273,8 +250,6 @@ def add(self, detections: list):
                     self.cache_values(annotations=detections, metrics=get_coco_metrics(self.coco_evaluator)))
                 self.first_batch_processed = True
 
-        self.start_time = time.time()
-
     def get_results(self):
         """
         Reruns the evaluation using the accumulated detections, returns COCO results with AP metrics
@@ -292,12 +267,13 @@ def get_results(self):
         self.coco_evaluator.summarize()
 
         self.results = get_coco_metrics(self.coco_evaluator)
-        unique_image_ids = set([d['image_id'] for d in self.detections])
-        self.speed_mem_metrics['Tasks Per Second (Total)'] = len(unique_image_ids) / self.inference_time.sum
         self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 
+    def reset_time(self):
+        self.init_time = time.time()
+
     def save(self):
         """
         Calculate results and then put into a BenchmarkResult object
@@ -311,6 +287,12 @@ def save(self):
         # recalculate to ensure no mistakes made during batch-by-batch metric calculation
         self.get_results()
 
+        if not self.cached_results:
+            unique_image_ids = set([d['image_id'] for d in self.detections])
+            self.speed_mem_metrics['Evaluation Time'] = len(unique_image_ids) / (time.time() - self.init_time)
+        else:
+            self.speed_mem_metrics['Evaluation Time'] = None
+
         return BenchmarkResult(
             task=self.task,
             config={},

sotabencheval/machine_translation/wmt.py

@@ -7,7 +7,6 @@
 import json
 import time
 
-
 class SQuADVersion(Enum):
     V11 = 'v1.1'
     V20 = 'v2.0'
@@ -35,48 +34,35 @@ def __init__(self,
 
         self.metrics = SQuADMetrics(self.dataset_path, version)
 
-        self.start_time = time.time()
-
-    def update_inference_time(self):
-
-        if not self.metrics._results and self.inference_time.count < 1:
-            # assuming this is the first time the evaluator is called
-            self.inference_time.update(time.time() - self.start_time)
-        elif not self.metrics._results and self.inference_time.count > 0:
-            # assuming the user has reset outputs, and is then readding (evaluation post batching)
-            pass
-        else:
-            # if there are outputs and the inference time count is > 0
-            self.inference_time.update(time.time() - self.start_time)
-
     def add(self, answers: Dict[str, str]):
-        self.update_inference_time()
         self.metrics.add(answers)
 
         if not self.first_batch_processed and self.metrics.has_data:
-            self.speed_mem_metrics['Tasks Per Second (Partial)'] = len(self.metrics.answers) / self.inference_time.sum
             self.batch_hash = calculate_batch_hash(
                 self.cache_values(answers=self.metrics.answers,
                                   metrics=self.metrics.get_results(ignore_missing=True))
             )
             self.first_batch_processed = True
 
-        self.start_time = time.time()
-
     def reset(self):
         self.metrics.reset()
 
     def get_results(self):
         if self.cached_results:
             return self.results
         self.results = self.metrics.get_results()
-        self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.metrics.answers) / self.inference_time.sum
         self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 
     def save(self):
         dataset = "SQuAD{} dev".format(self.metrics.version.value[1:])
+
+        if not self.cached_results:
+            self.speed_mem_metrics['Evaluation Time'] = len(self.metrics.answers) / (time.time() - self.init_time)
+        else:
+            self.speed_mem_metrics['Evaluation Time'] = None
+
         return super().save(dataset=dataset)