Commiting before merging christian branch

Author: Seilmast

.gitignore

@@ -1,6 +1,7 @@
 __pycache__/
 .ipynb_checkpoints/
 Data/*
+data/*
 Results/*
 Experiments/*
 _build/
@@ -14,9 +15,7 @@ doc/autoapi
 
 #Magnus specific
 job*
-env2/*
-ruffian.sh 
-localtest.sh
+local*
 
 # Johanthings
 formatting.x 

CollaborativeCoding/dataloaders/download.py

@@ -87,7 +87,7 @@ def _get_labels(path: Path) -> np.ndarray:
 
     def svhn(self, data_dir: Path) -> tuple[np.ndarray, np.ndarray]:
         def download_svhn(path, train: bool = True):
-            SVHN()
+            SVHN(path, split="train" if train else "test", download=True)
 
         parent_path = data_dir / "SVHN"
 

CollaborativeCoding/load_data.py

@@ -64,7 +64,7 @@ def load_data(dataset: str, *args, **kwargs) -> tuple:
         case "svhn":
             dataset = SVHNDataset
             train_labels, test_labels = downloader.svhn(data_dir=data_dir)
-            labels = np.arange(10)
+            labels = np.unique(train_labels)
         case "mnist_4-9":
             dataset = MNISTDataset4_9
             train_labels, test_labels = downloader.mnist(data_dir=data_dir)
@@ -78,6 +78,10 @@ def load_data(dataset: str, *args, **kwargs) -> tuple:
     train_indices = np.arange(len(train_labels))
     test_indices = np.arange(len(test_labels))
 
+    print(train_indices.shape)
+    print(np.asarray(train_labels).shape)
+    print(labels.shape)
+
     # Filter the labels to only get indices of the wanted labels
     train_samples = train_indices[np.isin(train_labels, labels)]
     test_samples = test_indices[np.isin(test_labels, labels)]

CollaborativeCoding/load_metric.py

@@ -94,3 +94,28 @@ def getmetrics(self, str_prefix: str = None):
     def resetmetric(self):
         for key in self.metrics:
             self.metrics[key].__reset__()
+
+
+if __name__ == "__main__":
+    import torch as th
+
+    metrics = ["entropy", "f1", "recall", "precision", "accuracy"]
+
+    class_sizes = [3, 6, 10]
+    for class_size in class_sizes:
+        y_true = th.rand((5, class_size)).argmax(dim=1)
+        y_pred = th.rand((5, class_size))
+
+        metricwrapper = MetricWrapper(
+            metric,
+            num_classes=class_size,
+            macro_averaging=True if class_size % 2 == 0 else False,
+        )
+
+        metricwrapper(y_true, y_pred)
+        metric = metricwrapper.getmetrics()
+        assert metric is not None
+
+        metricwrapper.resetmetric()
+        metric2 = metricwrapper.getmetrics()
+        assert metric != metric2

tests/test_dataloaders.py

@@ -8,6 +8,7 @@
 
 from CollaborativeCoding.dataloaders import (
     MNISTDataset0_3,
+    SVHNDataset,
     USPSDataset0_6,
     USPSH5_Digit_7_9_Dataset,
 )
@@ -20,6 +21,7 @@
         ("usps_0-6", USPSDataset0_6),
         ("usps_7-9", USPSH5_Digit_7_9_Dataset),
         ("mnist_0-3", MNISTDataset0_3),
+        ("svhn", SVHNDataset),
         # TODO: Add more datasets here
     ],
 )

tests/test_wrappers.py

@@ -1,6 +1,6 @@
 from pathlib import Path
 
-from CollaborativeCoding import load_data, load_metric, load_model
+from CollaborativeCoding import MetricWrapper, load_data, load_model
 
 
 def test_load_model():
@@ -36,13 +36,7 @@ def test_load_data():
     import torch as th
     from torchvision import transforms
 
-    dataset_names = [
-        "usps_0-6",
-        "mnist_0-3",
-        "usps_7-9",
-        "svhn",
-        # 'mnist_4-9' #Uncomment when implemented
-    ]
+    dataset_names = ["usps_0-6", "mnist_0-3", "usps_7-9", "svhn", "mnist_4-9"]
 
     trans = transforms.Compose(
         [
@@ -64,4 +58,25 @@ def test_load_data():
 
 
 def test_load_metric():
-    pass
+    import torch as th
+
+    metrics = ("entropy", "f1", "recall", "precision", "accuracy")
+
+    class_sizes = [3, 6, 10]
+    for class_size in class_sizes:
+        y_true = th.rand((5, class_size)).argmax(dim=1)
+        y_pred = th.rand((5, class_size))
+
+        metricwrapper = MetricWrapper(
+            *metrics,
+            num_classes=class_size,
+            macro_averaging=True if class_size % 2 == 0 else False,
+        )
+
+        metricwrapper(y_true, y_pred)
+        metric = metricwrapper.getmetrics()
+        assert metric is not None
+
+        metricwrapper.resetmetric()
+        metric2 = metricwrapper.getmetrics()
+        assert metric != metric2