Merge pull request #122 from Continvvm/fix_metrics

Fix metrics

Author: arthurdouillard

continuum/metrics/base_logger.py

@@ -43,7 +43,6 @@ def __init__(self, root_log=None, list_keywords=["performance"], list_subsets=["
         self._update_dict_architecture(update_task=True)
 
     def add(self, value, keyword="performance", subset="train"):
-
         assert keyword in self.list_keywords, f"Keyword {keyword} is not declared in list_keywords {self.list_keywords}"
         assert subset in self.list_subsets, f"Field {subset} is not declared in list_keywords {self.list_subsets}"
 
@@ -89,7 +88,7 @@ def _add_perf(self, predictions, targets, task_ids=None, subset="train"):
         targets = np.concatenate([self._get_current_targets(subset), targets])
         self.logger_dict[subset]["performance"][self.current_task][self.current_epoch]["targets"] = targets
 
-        if (task_ids is not None) and self._get_current_task_ids(subset).size > 0:
+        if task_ids is not None:
             task_ids = np.concatenate([self._get_current_task_ids(subset), task_ids])
             self.logger_dict[subset]["performance"][self.current_task][self.current_epoch]["task_ids"] = task_ids
 

continuum/metrics/logger.py

@@ -1,3 +1,5 @@
+import statistics
+
 import numpy as np
 
 from continuum.metrics.base_logger import _BaseLogger
@@ -37,7 +39,7 @@ def _get_best_epochs_perf(self, subset):
         last_epoch_pred = []
         last_epoch_targets = []
         last_epoch_task_ids = []
-        for task_id in range(self.current_task):
+        for task_id in range(len(self.logger_dict[subset]["performance"])):
             predictions = self.logger_dict[subset]["performance"][task_id][-1]["predictions"]
             targets = self.logger_dict[subset]["performance"][task_id][-1]["targets"]
             task_id = self.logger_dict[subset]["performance"][task_id][-1]["task_ids"]
@@ -48,20 +50,15 @@ def _get_best_epochs_perf(self, subset):
 
         return last_epoch_pred, last_epoch_targets, last_epoch_task_ids
 
-
     def _get_best_epochs_data(self, keyword, subset):
-
         assert keyword != "performance", f"this method is not mode for performance keyword use _get_best_epochs_perf"
         list_values = []
         for task_id in range(self.current_task):
             list_values.append(self.logger_dict[subset][keyword][task_id][-1])
         return list_values
 
-
-
     def _get_best_epochs(self, keyword="performance", subset="train"):
-
-        if keyword=="performance":
+        if keyword == "performance":
             values = self._get_best_epochs_perf(subset)
         else:
             values = self._get_best_epochs_data(keyword, subset)
@@ -84,7 +81,6 @@ def online_accuracy(self):
         return accuracy(predictions, targets)
 
     @property
-    @cache
     @require_subset("test")
     def accuracy(self):
         return accuracy(
@@ -93,7 +89,6 @@ def accuracy(self):
         )
 
     @property
-    @cache
     @require_subset("test")
     def accuracy_per_task(self):
         """Returns all task accuracy individually."""
@@ -104,7 +99,6 @@ def accuracy_per_task(self):
         ]
 
     @property
-    @cache
     @require_subset("train")
     def online_cumulative_performance(self):
         """Computes the accuracy of last task on the train set.
@@ -122,7 +116,6 @@ def online_cumulative_performance(self):
         return accuracy(preds, targets)
 
     @property
-    @cache
     @require_subset("test")
     def average_incremental_accuracy(self):
         """Computes the average of the accuracies computed after each task.
@@ -132,56 +125,48 @@ def average_incremental_accuracy(self):
           Rebuffi et al. CVPR 2017
         """
         all_preds, all_targets, _ = self._get_best_epochs(subset="test")
-
         return statistics.mean([
             accuracy(all_preds[t], all_targets[t])
             for t in range(len(all_preds))
         ])
 
     @property
-    @cache
     @require_subset("test")
     def backward_transfer(self):
         all_preds, all_targets, task_ids = self._get_best_epochs(subset="test")
         return backward_transfer(all_preds, all_targets, task_ids)
 
     @property
-    @cache
     @require_subset("test")
     def forward_transfer(self):
         all_preds, all_targets, task_ids = self._get_best_epochs(subset="test")
         return forward_transfer(all_preds, all_targets, task_ids)
 
     @property
-    @cache
     @require_subset("test")
     def positive_backward_transfer(self):
         all_preds, all_targets, task_ids = self._get_best_epochs(subset="test")
         return positive_backward_transfer(all_preds, all_targets, task_ids)
 
     @property
-    @cache
     @require_subset("test")
     def remembering(self):
         all_preds, all_targets, task_ids = self._get_best_epochs(subset="test")
         return remembering(all_preds, all_targets, task_ids)
 
     @property
-    @cache
     @require_subset("test")
     def accuracy_A(self):
         all_preds, all_targets, task_ids = self._get_best_epochs(subset="test")
         return accuracy_A(all_preds, all_targets, task_ids)
 
     @property
-    @cache
     @require_subset("test")
     def forgetting(self):
         all_preds, all_targets, task_ids = self._get_best_epochs(subset="test")
         return forgetting(all_preds, all_targets, task_ids)
 
     @property
-    @cache
     def model_size_growth(self):
         assert "model_size" in self.list_keywords
         sizes = self._get_best_epochs("model_size")

continuum/metrics/metrics.py

@@ -1,5 +1,7 @@
 from functools import reduce
 
+import numpy as np
+
 
 def accuracy(task_preds, task_targets):
     """Computes the accuracy of a given task.
@@ -14,7 +16,7 @@ def accuracy(task_preds, task_targets):
     assert task_targets.size == task_preds.size, f"{task_targets.size} vs {task_preds.size}"
 
     metric = (task_preds == task_targets).mean()
-    assert 0. <= metric <= 1.0
+    assert 0. <= metric <= 1.0, metric
     return metric
 
 
@@ -48,7 +50,7 @@ def accuracy_A(all_preds, all_targets, all_tasks):
             A += _get_R_ij(i, j, all_preds, all_targets, all_tasks)
 
     metric = A / (T * (T + 1) / 2)
-    assert 0. <= metric <= 1.0
+    assert 0. <= metric <= 1.0, metric
     return metric
 
 
@@ -78,7 +80,7 @@ def backward_transfer(all_preds, all_targets, all_tasks):
             bwt += (r_ij - r_jj)
 
     metric = bwt / (T * (T - 1) / 2)
-    assert -1. <= metric <= 1.0
+    assert -1. <= metric <= 1.0, metric
     return metric
 
 
@@ -96,7 +98,7 @@ def positive_backward_transfer(all_preds, all_targets, all_tasks):
     """
     bwt = backward_transfer(all_preds, all_targets, all_tasks)
     metric = 1 - abs(min(bwt, 0.))
-    assert 0. <= metric <= 1.0
+    assert 0. <= metric <= 1.0, metric
     return metric
 
 
@@ -114,7 +116,7 @@ def remembering(all_preds, all_targets, all_tasks):
     """
     bwt = backward_transfer(all_preds, all_targets, all_tasks)
     metric = max(bwt, 0.)
-    assert 0. <= metric <= 1.0
+    assert 0. <= metric <= 1.0, metric
     return metric
 
 
@@ -141,7 +143,7 @@ def forward_transfer(all_preds, all_targets, all_tasks):
             fwt += _get_R_ij(i, j, all_preds, all_targets, all_tasks)
 
     metric = fwt / (T * (T - 1) / 2)
-    assert -1. <= metric <= 1.0
+    assert -1. <= metric <= 1.0, metric
     return metric
 
 
@@ -164,7 +166,7 @@ def forgetting(all_preds, all_targets, all_tasks):
         f += (r_lj - r_kj)
 
     metric = f / (T - 1)
-    assert 0. <= metric <= 1.0
+    assert 0. <= metric <= 1.0, metric
     return metric
 
 
@@ -241,5 +243,5 @@ def get_model_size_growth(model_sizes):
         ms += (model_sizes[0][0] / model_sizes[i][-1])
 
     metric = min(1., ms / T)
-    assert 0. <= metric <= 1.0
+    assert 0. <= metric <= 1.0, metric
     return metric

continuum/metrics/utils.py

@@ -21,4 +21,4 @@ def wrapper(self):
             v = func(self)
             self.__dict__[name] = v
         return v
-    return wrapper
+    return wrapper

continuum/scenarios/base.py

@@ -115,10 +115,12 @@ def _select_data_by_task(
         x, y, t = self.dataset  # type: ignore
 
         if isinstance(task_index, slice):
-            start = task_index.start or 0
-            stop = task_index.stop or len(self) + 1
-            step = task_index.step or 1
+            start = task_index.start if task_index.start is not None else 0
+            stop = task_index.stop if task_index.stop is not None else len(self) + 1
+            step = task_index.step if task_index.step is not None else 1
             task_index = list(range(start, stop, step))
+            if len(task_index) == 0:
+                raise ValueError(f"Invalid slicing resulting in no data (start={start}, end={stop}, step={step}).")
             task_index = [
                 t if t >= 0 else _handle_negative_indexes(t, len(self)) for t in task_index
             ]

tests/test_slicing.py

@@ -49,7 +49,6 @@ def test_slicing_nc(index, classes):
     (0, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]),
     (3, [6, 7, 8, 9]),
     (4, [8, 9]),
-    (12, [])
 ])
 def test_slicing_nc_no_end(start_index, classes):
     train, test = gen_data()
@@ -61,6 +60,25 @@ def test_slicing_nc_no_end(start_index, classes):
     assert (targets == np.array(classes)).all(), (targets, classes)
 
 
+@pytest.mark.parametrize("start,end", [
+    (12, 100),
+    (0, 0),
+    (3, 3)
+])
+def test_slicing_empty(start, end):
+    train, test = gen_data()
+    dummy = InMemoryDataset(*train)
+    scenario = ClassIncremental(dummy, increment=2)
+
+    has_failed = False
+    try:
+        taskset = scenario[start_index:]
+    except:
+        has_failed = True
+
+    assert has_failed
+
+
 def test_slicing_nc_no_index():
     train, test = gen_data()
     dummy = InMemoryDataset(*train)
@@ -72,3 +90,5 @@ def test_slicing_nc_no_index():
 
     assert len(targets) == len(classes)
     assert (targets == np.array(classes)).all(), (targets, classes)
+
+