Add files via upload

Author: yzhuoning

examples/08_Optimizing_AUROC_Loss_with_DenseNet121_on_Melanoma.ipynb

@@ -0,0 +1,316 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "name": "08.Optimizing_AUROC_Loss_with_DenseNet121_on_Melanoma.ipynb",
+      "provenance": [],
+      "collapsed_sections": []
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "ud6jh8GWr_qT"
+      },
+      "source": [
+        "* Author: Zhuoning Yuan\n",
+        "* Project: https://github.com/Optimization-AI/LibAUC\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "dcqqRTWMsFKY"
+      },
+      "source": [
+        "# **Installing LibAUC**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "Ter36EFG2k2g"
+      },
+      "source": [
+        "!pip install libauc"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "rZL_V9NutBio"
+      },
+      "source": [
+        "# **Downloading Melanoma Dataset**\n",
+        " \n",
+        "*  You can download dataset from here: https://www.kaggle.com/cdeotte/jpeg-melanoma-256x256. \n",
+        "*  In this tutorial, I am going to use JPEG Melanoma 256x256.\n",
+        "\n"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "gXlAZOYRs_rf"
+      },
+      "source": [
+        "!cp /content/drive/MyDrive/LibAUC/melanoma_256x256.zip /content/\n",
+        "!unzip melanoma_224x224.zip -d /content/melanoma/"
+      ],
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "SWwE3v_R5Oku"
+      },
+      "source": [
+        "\n",
+        "# **Importing LibAUC**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "fDfVpPkp4o9O"
+      },
+      "source": [
+        "from libauc.losses import AUCMLoss\n",
+        "from libauc.optimizers import PESG\n",
+        "from libauc.models import DenseNet121, DenseNet169\n",
+        "from libauc.datasets import Melanoma\n",
+        "from libauc.utils import auroc\n",
+        "\n",
+        "import torch \n",
+        "from PIL import Image\n",
+        "import numpy as np\n",
+        "import torchvision.transforms as transforms\n",
+        "from torch.utils.data import Dataset"
+      ],
+      "execution_count": 1,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "l2swK5Mo7Kca"
+      },
+      "source": [
+        "# **Reproducibility**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "OiiT5oEp7J3C"
+      },
+      "source": [
+        "def set_all_seeds(SEED):\n",
+        "    # REPRODUCIBILITY\n",
+        "    torch.manual_seed(SEED)\n",
+        "    np.random.seed(SEED)\n",
+        "    torch.backends.cudnn.deterministic = True\n",
+        "    torch.backends.cudnn.benchmark = False"
+      ],
+      "execution_count": 2,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "G-StcokOURv9"
+      },
+      "source": [
+        "# **Data Augmentation**"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "id": "sQlAJDeoUTtR"
+      },
+      "source": [
+        "import albumentations as A\n",
+        "from albumentations.pytorch.transforms import ToTensor\n",
+        "\n",
+        "def augmentations(image_size=256, is_test=True):\n",
+        "    # https://www.kaggle.com/vishnus/a-simple-pytorch-starter-code-single-fold-93\n",
+        "    imagenet_stats = {'mean':[0.485, 0.456, 0.406], 'std':[0.229, 0.224, 0.225]}\n",
+        "    train_tfms = A.Compose([\n",
+        "        A.Cutout(p=0.5),\n",
+        "        A.RandomRotate90(p=0.5),\n",
+        "        A.Flip(p=0.5),\n",
+        "        A.OneOf([\n",
+        "            A.RandomBrightnessContrast(brightness_limit=0.2,\n",
+        "                                       contrast_limit=0.2,\n",
+        "                                       ),\n",
+        "            A.HueSaturationValue(\n",
+        "                hue_shift_limit=20,\n",
+        "                sat_shift_limit=50,\n",
+        "                val_shift_limit=50)\n",
+        "        ], p=0.5),\n",
+        "        A.OneOf([\n",
+        "            A.IAAAdditiveGaussianNoise(),\n",
+        "            A.GaussNoise(),\n",
+        "        ], p=0.5),\n",
+        "        A.OneOf([\n",
+        "            A.MotionBlur(p=0.2),\n",
+        "            A.MedianBlur(blur_limit=3, p=0.1),\n",
+        "            A.Blur(blur_limit=3, p=0.1),\n",
+        "        ], p=0.5),\n",
+        "        A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.2, rotate_limit=45, p=0.5),\n",
+        "        A.OneOf([\n",
+        "            A.OpticalDistortion(p=0.3),\n",
+        "            A.GridDistortion(p=0.1),\n",
+        "            A.IAAPiecewiseAffine(p=0.3),\n",
+        "        ], p=0.5), \n",
+        "        ToTensor(normalize=imagenet_stats)\n",
+        "        ])\n",
+        "    \n",
+        "    test_tfms = A.Compose([ToTensor(normalize=imagenet_stats)])\n",
+        "    if is_test:\n",
+        "        return test_tfms\n",
+        "    else:\n",
+        "        return train_tfms"
+      ],
+      "execution_count": 3,
+      "outputs": []
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "fP4adNO97YBV"
+      },
+      "source": [
+        "# **Optimizing AUCM Loss**\n",
+        "* Installation of `albumentations` is required!"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 519
+        },
+        "id": "Xx7j0UYe6V0J",
+        "outputId": "3bce73dc-ea0a-4639-b3ba-888b276a7d97"
+      },
+      "source": [
+        "# dataset\n",
+        "trainSet = Melanoma(root='./melanoma/', is_test=False, test_size=0.2, transforms=augmentations)\n",
+        "testSet = Melanoma(root='./melanoma/', is_test=True, test_size=0.2, transforms=augmentations)\n",
+        "\n",
+        "# paramaters\n",
+        "SEED = 123\n",
+        "BATCH_SIZE = 64\n",
+        "lr = 0.1 \n",
+        "gamma = 500\n",
+        "imratio = trainSet.imratio\n",
+        "weight_decay = 1e-5\n",
+        "margin = 1.0\n",
+        "\n",
+        "# model\n",
+        "set_all_seeds(SEED)\n",
+        "model = DenseNet121(pretrained=True, last_activation=None, activations='relu', num_classes=1)\n",
+        "model = model.cuda()\n",
+        "\n",
+        "trainloader = torch.utils.data.DataLoader(trainSet, batch_size=BATCH_SIZE, num_workers=2, shuffle=True)\n",
+        "testloader =  torch.utils.data.DataLoader(testSet, batch_size=BATCH_SIZE, num_workers=2, shuffle=False)\n",
+        "\n",
+        "# load your own pretrained model here\n",
+        "#  PATH = 'ce_pretrained_model.pth' \n",
+        "#  state_dict = torch.load(PATH)\n",
+        "#  state_dict.pop('classifier.weight', None)\n",
+        "#  state_dict.pop('classifier.bias', None) \n",
+        "#  model.load_state_dict(state_dict, strict=False)\n",
+        "\n",
+        "# define loss & optimizer\n",
+        "Loss = AUCMLoss(imratio=imratio)\n",
+        "optimizer = PESG(model, \n",
+        "                 a=Loss.a, \n",
+        "                 b=Loss.b, \n",
+        "                 alpha=Loss.alpha, \n",
+        "                 lr=lr, \n",
+        "                 gamma=gamma, \n",
+        "                 margin=margin, \n",
+        "                 weight_decay=weight_decay)\n",
+        "\n",
+        "total_epochs = 16\n",
+        "best_val_auc = 0\n",
+        "for epoch in range(total_epochs):\n",
+        "\n",
+        "  # reset stages \n",
+        "  if epoch== int(total_epochs*0.5) or epoch== int(total_epochs*0.75):\n",
+        "     optimizer.update_regularizer(decay_factor=10) \n",
+        "\n",
+        "  # training \n",
+        "  for idx, data in enumerate(trainloader):\n",
+        "      train_data, train_labels = data\n",
+        "      train_data, train_labels = train_data.cuda(), train_labels.cuda()\n",
+        "      y_pred = model(train_data)\n",
+        "      y_pred = torch.sigmoid(y_pred)\n",
+        "      loss = Loss(y_pred, train_labels)\n",
+        "      optimizer.zero_grad()\n",
+        "      loss.backward()\n",
+        "      optimizer.step()\n",
+        "\n",
+        "  # validation\n",
+        "  model.eval()\n",
+        "  with torch.no_grad():    \n",
+        "       test_pred = []\n",
+        "       test_true = [] \n",
+        "       for jdx, data in enumerate(testloader):\n",
+        "           test_data, test_label = data\n",
+        "           test_data = test_data.cuda()\n",
+        "           y_pred = model(test_data)\n",
+        "           y_pred = torch.sigmoid(y_pred)\n",
+        "           test_pred.append(y_pred.cpu().detach().numpy())\n",
+        "           test_true.append(test_label.numpy())\n",
+        "              \n",
+        "       test_true = np.concatenate(test_true)\n",
+        "       test_pred = np.concatenate(test_pred)\n",
+        "       val_auc =  auroc(test_true, test_pred) \n",
+        "       model.train()\n",
+        "\n",
+        "       if best_val_auc < val_auc:\n",
+        "          best_val_auc = val_auc\n",
+        "              \n",
+        "       print ('Epoch=%s, Loss=%.4f, Val_AUC=%.4f, lr=%.4f'%(epoch, loss, val_auc, optimizer.lr))\n",
+        "\n",
+        "print ('Best Val_AUC is %.4f'%best_val_auc)"
+      ],
+      "execution_count": 7,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "test set split is 0.2\n",
+            "Found 26134 image in total, 464 postive images, 25670 negative images.\n",
+            "test set split is 0.2\n",
+            "Found 6558 image in total, 117 postive images, 6441 negative images.\n",
+            "Epoch=0, Loss=0.0238, Val_AUC=0.8411, lr=0.1000\n",
+            "Epoch=1, Loss=0.0245, Val_AUC=0.8697, lr=0.1000\n",
+            "Epoch=2, Loss=0.0213, Val_AUC=0.8460, lr=0.1000\n",
+            "Epoch=3, Loss=-0.0234, Val_AUC=0.8783, lr=0.1000\n"
+          ]
+        },
+      ]
+    }
+  ]
+}