test memory calculation

Author: RJT1990

sotabencheval/__pycache__/__init__.cpython-36.pyc

@@ -9,6 +9,7 @@
 import time
 
 from sotabencheval.utils import AverageMeter, calculate_batch_hash, download_url, change_root_if_server, is_server
+from sotabencheval.utils import get_max_memory_allocated
 from sotabencheval.image_classification.utils import top_k_accuracy_score
 
 ARCHIVE_DICT = {
@@ -330,6 +331,7 @@ def get_results(self):
 
         self.results = {'Top 1 Accuracy': self.top1.avg, 'Top 5 Accuracy': self.top5.avg}
         self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.outputs) / self.inference_time.sum
+        self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 

sotabencheval/__pycache__/version.cpython-36.pyc

@@ -2,6 +2,7 @@
 from sotabencheval.utils import calculate_batch_hash, change_root_if_server, is_server
 from sotabencheval.machine_translation.languages import Language
 from sotabencheval.machine_translation.metrics import TranslationMetrics
+from sotabencheval.utils import get_max_memory_allocated
 from typing import Dict
 from pathlib import Path
 from enum import Enum
@@ -113,6 +114,7 @@ def get_results(self):
             return self.results
         self.results = self.metrics.get_results()
         self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.metrics.answers) / self.inference_time.sum
+        self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 

sotabencheval/image_classification/imagenet.py

@@ -9,7 +9,7 @@
 import time
 
 from sotabencheval.utils import calculate_batch_hash, extract_archive, change_root_if_server, is_server
-from sotabencheval.utils import AverageMeter
+from sotabencheval.utils import AverageMeter, get_max_memory_allocated
 from sotabencheval.object_detection.coco_eval import CocoEvaluator
 from sotabencheval.object_detection.utils import get_coco_metrics
 
@@ -294,6 +294,7 @@ def get_results(self):
         self.results = get_coco_metrics(self.coco_evaluator)
         unique_image_ids = set([d['image_id'] for d in self.detections])
         self.speed_mem_metrics['Tasks Per Second (Total)'] = len(unique_image_ids) / self.inference_time.sum
+        self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 

sotabencheval/machine_translation/wmt.py

@@ -1,5 +1,5 @@
 from sotabencheval.core import BaseEvaluator
-from sotabencheval.utils import calculate_batch_hash, change_root_if_server, is_server
+from sotabencheval.utils import calculate_batch_hash, change_root_if_server, is_server, get_max_memory_allocated
 from sotabencheval.question_answering.utils import *
 from typing import Dict
 from enum import Enum
@@ -71,6 +71,8 @@ def get_results(self):
             return self.results
         self.results = self.metrics.get_results()
         self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.metrics.answers) / self.inference_time.sum
+        self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
+
         return self.results
 
     def save(self):

sotabencheval/object_detection/coco.py

@@ -3,7 +3,7 @@
 from sotabenchapi.core import BenchmarkResult, check_inputs
 import time
 
-from sotabencheval.utils import calculate_batch_hash, is_server, AverageMeter
+from sotabencheval.utils import calculate_batch_hash, is_server, AverageMeter, get_max_memory_allocated
 from sotabencheval.semantic_segmentation.utils import ConfusionMatrix
 
 
@@ -269,6 +269,7 @@ def get_results(self):
                }
 
         self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.inference_time.count) / self.inference_time.sum
+        self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 

sotabencheval/question_answering/squad.py

@@ -3,7 +3,7 @@
 from sotabenchapi.core import BenchmarkResult, check_inputs
 import time
 
-from sotabencheval.utils import calculate_batch_hash, is_server, AverageMeter
+from sotabencheval.utils import calculate_batch_hash, is_server, AverageMeter, get_max_memory_allocated
 from sotabencheval.semantic_segmentation.utils import ConfusionMatrix
 
 
@@ -244,6 +244,7 @@ def get_results(self):
                }
 
         self.speed_mem_metrics['Tasks Per Second (Total)'] = len(self.inference_time.count) / self.inference_time.sum
+        self.speed_mem_metrics['Max Memory Allocated (Total)'] = get_max_memory_allocated()
 
         return self.results
 

sotabencheval/semantic_segmentation/ade20k.py

@@ -86,6 +86,15 @@ def set_env_on_server(env_name, value):
     return False
 
 
+def get_max_memory_allocated(device='cuda'):
+    try:
+        import torch
+        max_mem = torch.cuda.max_memory_allocated(device='cuda')
+        torch.cuda.reset_max_memory_allocated(device='cuda')
+        return max_mem
+    except ImportError:
+        return None
+
 # below utilities are taken from the torchvision repository
 
 

sotabencheval/semantic_segmentation/pascalvoc.py

@@ -15,6 +15,6 @@ def __repr__(self):
             f"build={self.build})"
         )
 
-version = Version(0, 0, 29)
+version = Version(0, 0, 30)
 
 __version__ = str(version)