Add files via upload

Author: yzhuoning

examples/scripts/02_optimizing_auroc_with_resnet20_on_imbalanced_cifar10.py

@@ -1,30 +1,33 @@
-"""Optimizing AUROC loss on imbalanced dataset**
+"""02_Optimizing_AUROC_with_ResNet20_on_Imbalanced_CIFAR10.ipynb
 
- Author: Zhuoning Yuan
+**Author**: Zhuoning Yuan
 
-If you find this tutorial helpful in your work,  please acknowledge our library and cite the following paper:
+**Introduction**
+In this tutorial, you will learn how to quickly train a ResNet20 model by optimizing **AUROC** using our novel [AUCMLoss](https://arxiv.org/abs/2012.03173) and `PESG` optimizer on a binary image classification task on Cifar10. After completion of this tutorial, you should be able to use LibAUC to train your own models on your own datasets.
+
+**Useful Resources**:
+* Website: https://libauc.org
+* Github: https://github.com/Optimization-AI/LibAUC
 
+**Reference**:  
+If you find this tutorial helpful in your work,  please acknowledge our library and cite the following paper:
 @inproceedings{yuan2021large,
   title={Large-scale robust deep auc maximization: A new surrogate loss and empirical studies on medical image classification},
   author={Yuan, Zhuoning and Yan, Yan and Sonka, Milan and Yang, Tianbao},
   booktitle={Proceedings of the IEEE/CVF International Conference on Computer Vision},
   pages={3040--3049},
   year={2021}
 }
-
-@misc{libauc2022,
-      title={LibAUC: A Deep Learning Library for X-Risk Optimization.},
-      author={Zhuoning Yuan, Zi-Hao Qiu, Gang Li, Dixian Zhu, Zhishuai Guo, Quanqi Hu, Bokun Wang, Qi Qi, Yongjian Zhong, Tianbao Yang},
-      year={2022}
-    }
 """
 
+
 from libauc.losses import AUCMLoss
 from libauc.optimizers import PESG
 from libauc.models import resnet20 as ResNet20
 from libauc.datasets import CIFAR10
 from libauc.utils import ImbalancedDataGenerator
 from libauc.sampler import DualSampler
+from libauc.metrics import auc_roc_score
 
 import torch 
 from PIL import Image
@@ -33,7 +36,6 @@
 from torch.utils.data import Dataset
 from sklearn.metrics import roc_auc_score
 
-
 def set_all_seeds(SEED):
     # REPRODUCIBILITY
     torch.manual_seed(SEED)
@@ -71,82 +73,122 @@ def __getitem__(self, idx):
         return image, target
 
 
-# paramaters
+# HyperParameters
 SEED = 123
 BATCH_SIZE = 128
 imratio = 0.1 # for demo 
+total_epochs = 100
+decay_epochs = [50, 75]
+
 lr = 0.1
-gamma = 500
-weight_decay = 1e-4
 margin = 1.0
+epoch_decay = 0.003 # refers gamma in the paper
+weight_decay = 0.0001
 
+# oversampling minority class, you can tune it in (0, 0.5]
+# e.g., sampling_rate=0.2 is that num of positive samples in mini-batch is sampling_rate*batch_size=13
+sampling_rate = 0.2
 
-# dataloader 
+# load data as numpy arrays 
 train_data, train_targets = CIFAR10(root='./data', train=True)
 test_data, test_targets  = CIFAR10(root='./data', train=False)
 
+# generate imbalanced data
 generator = ImbalancedDataGenerator(verbose=True, random_seed=0)
 (train_images, train_labels) = generator.transform(train_data, train_targets, imratio=imratio)
 (test_images, test_labels) = generator.transform(test_data, test_targets, imratio=0.5) 
 
-trainloader = torch.utils.data.DataLoader(ImageDataset(train_images, train_labels), batch_size=BATCH_SIZE, shuffle=True, num_workers=1, pin_memory=True, drop_last=True)
-testloader = torch.utils.data.DataLoader( ImageDataset(test_images, test_labels, mode='test'), batch_size=BATCH_SIZE, shuffle=False, num_workers=1,  pin_memory=True)
+# data augmentations 
+trainSet = ImageDataset(train_images, train_labels)
+trainSet_eval = ImageDataset(train_images, train_labels, mode='test')
+testSet = ImageDataset(test_images, test_labels, mode='test')
 
+# dataloaders
+sampler = DualSampler(trainSet, BATCH_SIZE, sampling_rate=sampling_rate)
+trainloader = torch.utils.data.DataLoader(trainSet, batch_size=BATCH_SIZE, sampler=sampler, num_workers=2)
+trainloader_eval = torch.utils.data.DataLoader(trainSet_eval, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)
+testloader = torch.utils.data.DataLoader(testSet, batch_size=BATCH_SIZE, shuffle=False, num_workers=2)
 
+"""# **Creating models & AUC Optimizer**"""
+# You can include sigmoid/l2 activations on model's outputs before computing loss
 model = ResNet20(pretrained=False, last_activation=None, num_classes=1)
 model = model.cuda()
 
-Loss = AUCMLoss()
+# You can also pass Loss.a, Loss.b, Loss.alpha to optimizer (for old version users)
+loss_fn = AUCMLoss()
 optimizer = PESG(model, 
-                 a=Loss.a, 
-                 b=Loss.b, 
-                 alpha=Loss.alpha, 
+                 loss_fn=loss_fn,
                  lr=lr, 
-                 gamma=gamma, 
+                 momentum=0.9,
                  margin=margin, 
+                 epoch_decay=epoch_decay, 
                  weight_decay=weight_decay)
 
 
+"""# **Training**"""
 print ('Start Training')
 print ('-'*30)
-for epoch in range(100):
-    
-     if epoch == 50 or epoch==75:
-         # decrease learning rate by 10x & update regularizer
-         optimizer.update_regularizer(decay_factor=10)
+
+train_log = []
+test_log = []
+for epoch in range(total_epochs):
+     if epoch in decay_epochs:
+         optimizer.update_regularizer(decay_factor=10) # decrease learning rate by 10x & update regularizer
 
-     train_pred = []
-     train_true = []
+     train_loss = []
      model.train()    
      for data, targets in trainloader:
          data, targets  = data.cuda(), targets.cuda()
          y_pred = model(data)
          y_pred = torch.sigmoid(y_pred)
-         loss = Loss(y_pred, targets)
+         loss = loss_fn(y_pred, targets)
          optimizer.zero_grad()
          loss.backward()
          optimizer.step()
-        
-         train_pred.append(y_pred.cpu().detach().numpy())
-         train_true.append(targets.cpu().detach().numpy())
-
-     train_true = np.concatenate(train_true)
-     train_pred = np.concatenate(train_pred)
-     train_auc = roc_auc_score(train_true, train_pred) 
-
+         train_loss.append(loss.item())
+    
+     # evaluation on train & test sets
      model.eval()
-     test_pred = []
-     test_true = [] 
-     for j, data in enumerate(testloader):
-         test_data, test_targets = data
+     train_pred_list = []
+     train_true_list = []
+     for train_data, train_targets in trainloader_eval:
+         train_data = train_data.cuda()
+         train_pred = model(train_data)
+         train_pred_list.append(train_pred.cpu().detach().numpy())
+         train_true_list.append(train_targets.numpy())
+     train_true = np.concatenate(train_true_list)
+     train_pred = np.concatenate(train_pred_list)
+     train_auc = auc_roc_score(train_true, train_pred)
+     train_loss = np.mean(train_loss)
+  
+     test_pred_list = []
+     test_true_list = [] 
+     for test_data, test_targets in testloader:
          test_data = test_data.cuda()
-         y_pred = model(test_data)
-         test_pred.append(y_pred.cpu().detach().numpy())
-         test_true.append(test_targets.numpy())
-     test_true = np.concatenate(test_true)
-     test_pred = np.concatenate(test_pred)
-     val_auc =  roc_auc_score(test_true, test_pred) 
+         test_pred = model(test_data)
+         test_pred_list.append(test_pred.cpu().detach().numpy())
+         test_true_list.append(test_targets.numpy())
+     test_true = np.concatenate(test_true_list)
+     test_pred = np.concatenate(test_pred_list)
+     val_auc =  auc_roc_score(test_true, test_pred) 
      model.train()
-   
+ 
      # print results
-     print("epoch: {}, train_loss: {:4f}, train_auc:{:4f}, test_auc:{:4f}, lr:{:4f}".format(epoch, loss.item(), train_auc, val_auc, optimizer.lr ))
+     print("epoch: %s, train_loss: %.4f, train_auc: %.4f, test_auc: %.4f, lr: %.4f"%(epoch, train_loss, train_auc, val_auc, optimizer.lr ))    
+     train_log.append(train_auc) 
+     test_log.append(val_auc)
+
+
+"""# **Visualization**
+Now, let's see the learning curve of optimizing AUROC on train and tes sets. 
+"""
+import matplotlib.pyplot as plt
+plt.rcParams["figure.figsize"] = (9,5)
+x=np.arange(len(train_log))
+plt.figure()
+plt.plot(x, train_log, LineStyle='-', label='Train Set', linewidth=3)
+plt.plot(x, test_log,  LineStyle='-', label='Test Set', linewidth=3)
+plt.title('AUCMLoss (10% CIFAR10)',fontsize=25)
+plt.legend(fontsize=15)
+plt.ylabel('AUROC', fontsize=25)
+plt.xlabel('Epoch', fontsize=25)