Merged with jan-branch and synched main

Author: Seilmast

.python-version

@@ -0,0 +1 @@
+3.12

environment.yml

@@ -9,7 +9,8 @@ dependencies:
   - sphinx-autobuild
   - sphinx-rtd-theme
   - pip
-  - h5py
+  - h5py==3.12.1
+  - hdf5==1.14.4
   - black
   - isort
   - jupyterlab
@@ -20,6 +21,8 @@ dependencies:
   - scalene
   - tqdm
   - scipy
+  - wandb
+  - scikit-learn
   - pip:
     - torch
     - torchvision

main.py

@@ -5,6 +5,7 @@
 from torch.utils.data import DataLoader
 from torchvision import transforms
 from tqdm import tqdm
+from wandb_api import WANDB_API
 
 from utils import MetricWrapper, createfolders, get_args, load_data, load_model
 
@@ -29,33 +30,38 @@ def main():
 
     device = args.device
 
-    if args.dataset.lower() in ["usps_0-6", "uspsh5_7_9"]:
-        augmentations = transforms.Compose(
+    if args.dataset.lower() in ["usps_0-6", "usps_7-9"]:
+        transform = transforms.Compose(
             [
                 transforms.Resize((16, 16)),
                 transforms.ToTensor(),
             ]
         )
     else:
-        augmentations = transforms.Compose([transforms.ToTensor()])
+        transform = transforms.Compose([transforms.ToTensor()])
 
-    # Dataset
-    traindata = load_data(
+    traindata, validata, testdata = load_data(
         args.dataset,
-        train=True,
-        data_path=args.datafolder,
-        download=args.download_data,
-        transform=augmentations,
-    )
-    validata = load_data(
-        args.dataset,
-        train=False,
-        data_path=args.datafolder,
-        download=args.download_data,
-        transform=augmentations,
+        data_dir=args.datafolder,
+        transform=transform,
+        val_size=args.val_size,
     )
 
-    metrics = MetricWrapper(traindata.num_classes, *args.metric)
+    train_metrics = MetricWrapper(
+        *args.metric,
+        num_classes=traindata.num_classes,
+        macro_averaging=args.macro_averaging,
+    )
+    val_metrics = MetricWrapper(
+        *args.metric,
+        num_classes=traindata.num_classes,
+        macro_averaging=args.macro_averaging,
+    )
+    test_metrics = MetricWrapper(
+        *args.metric,
+        num_classes=traindata.num_classes,
+        macro_averaging=args.macro_averaging,
+    )
 
     # Find the shape of the data, if is 2D, add a channel dimension
     data_shape = traindata[0][0].shape
@@ -80,6 +86,9 @@ def main():
     valiloader = DataLoader(
         validata, batch_size=args.batchsize, shuffle=False, pin_memory=True
     )
+    testloader = DataLoader(
+        testdata, batch_size=args.batchsize, shuffle=False, pin_memory=True
+    )
 
     criterion = nn.CrossEntropyLoss()
     optimizer = th.optim.Adam(model.parameters(), lr=args.learning_rate)
@@ -104,17 +113,17 @@ def main():
             optimizer.step()
             optimizer.zero_grad(set_to_none=True)
 
-            metrics(y, logits)
+            train_metrics(y, logits)
 
             break
-        print(metrics.accumulate())
+        print(train_metrics.accumulate())
         print("Dry run completed successfully.")
         exit()
 
     # wandb.login(key=WANDB_API)
     wandb.init(
         entity="ColabCode",
-        project="Magnus Runs",
+        project=args.run_name,
         tags=[args.modelname, args.dataset],
         config=args,
     )
@@ -135,33 +144,49 @@ def main():
             optimizer.zero_grad(set_to_none=True)
             trainingloss.append(loss.item())
 
-            metrics(y, logits)
+            train_metrics(y, logits)
 
-        wandb.log(metrics.accumulate(str_prefix="Train "))
-        metrics.reset()
-
-        evalloss = []
-        # Eval loop start
+        valloss = []
+        # Validation loop start
         model.eval()
         with th.no_grad():
             for x, y in tqdm(valiloader, desc="Validation"):
                 x, y = x.to(device), y.to(device)
                 logits = model.forward(x)
                 loss = criterion(logits, y)
-                evalloss.append(loss.item())
-
-                metrics(y, logits)
+                valloss.append(loss.item())
 
-        wandb.log(metrics.accumulate(str_prefix="Evaluation "))
-        metrics.reset()
+                val_metrics(y, logits)
 
         wandb.log(
             {
                 "Epoch": epoch,
                 "Train loss": np.mean(trainingloss),
-                "Evaluation Loss": np.mean(evalloss),
+                "Validation loss": np.mean(valloss),
             }
+            | train_metrics.__getmetrics__(str_prefix="Train ")
+            | val_metrics.__getmetrics__(str_prefix="Validation ")
         )
+        train_metrics.__resetmetrics__()
+        val_metrics.__resetmetrics__()
+
+    testloss = []
+    model.eval()
+    with th.no_grad():
+        for x, y in tqdm(testloader, desc="Testing"):
+            x, y = x.to(device), y.to(device)
+            logits = model.forward(x)
+            loss = criterion(logits, y)
+            testloss.append(loss.item())
+
+            preds = th.argmax(logits, dim=1)
+            test_metrics(y, preds)
+
+    wandb.log(
+        {"Epoch": 1, "Test loss": np.mean(testloss)}
+        | test_metrics.__getmetrics__(str_prefix="Test ")
+    )
+    test_metrics.__resetmetrics__()
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -1,3 +1,29 @@
+[project]
+name = "collaborative-coding-exam"
+version = "0.1.0"
+description = "Exam project in the collaborative coding course."
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "black>=25.1.0",
+    "h5py>=3.12.1",
+    "isort>=6.0.0",
+    "jupyterlab>=4.3.5",
+    "numpy>=2.2.2",
+    "pandas>=2.2.3",
+    "pip>=25.0",
+    "pytest>=8.3.4",
+    "ruff>=0.9.4",
+    "scalene>=1.5.51",
+    "scikit-learn>=1.6.1",
+    "sphinx>=8.1.3",
+    "sphinx-autoapi>=3.4.0",
+    "sphinx-autobuild>=2024.10.3",
+    "sphinx-rtd-theme>=3.0.2",
+    "torch>=2.6.0",
+    "torchvision>=0.21.0",
+    "tqdm>=4.67.1",
+]
 [tool.isort]
 profile = "black"
 line_length = 88

tests/test_dataloaders.py

@@ -0,0 +1,41 @@
+from utils.dataloaders.usps_0_6 import USPSDataset0_6
+
+
+def test_uspsdataset0_6():
+    from pathlib import Path
+    from tempfile import TemporaryDirectory
+
+    import h5py
+    import numpy as np
+    from torchvision import transforms
+
+    # Create a temporary directory (deleted after the test)
+    with TemporaryDirectory() as tempdir:
+        tempdir = Path(tempdir)
+
+        tf = tempdir / "usps.h5"
+
+        # Create a h5 file
+        with h5py.File(tf, "w") as f:
+            targets = np.array([6, 5, 4, 3, 2, 1, 0, 0, 0, 0])
+            indices = np.arange(len(targets))
+            # Populate the file with data
+            f["train/data"] = np.random.rand(10, 16 * 16)
+            f["train/target"] = targets
+
+        trans = transforms.Compose(
+            [
+                transforms.Resize((16, 16)),
+                transforms.ToTensor(),
+            ]
+        )
+        dataset = USPSDataset0_6(
+            data_path=tempdir,
+            sample_ids=indices,
+            train=True,
+            transform=trans,
+        )
+        assert len(dataset) == 10
+        data, target = dataset[0]
+        assert data.shape == (1, 16, 16)
+        assert target == 6