Merge pull request #404 from sergregory/t-sne-visualization

t-SNE visualization

Author: spmallick

README.md

@@ -13,6 +13,7 @@ Want to become an expert in AI? [AI Courses by OpenCV](https://opencv.org/course
 
 | Blog Post | |
 | ------------- |:-------------|
+|[t-SNE for ResNet feature visualization](https://www.learnopencv.com/t-sne-for-resnet-feature-visualization/)|[Code](https://github.com/spmallick/learnopencv/tree/master/TSNE)|
 |[Multi-Label Image Classification with Pytorch](https://www.learnopencv.com/multi-label-image-classification-with-pytorch/)|[Code](https://github.com/spmallick/learnopencv/tree/master/PyTorch-Multi-Label-Image-Classification)|
 |[CNN Receptive Field Computation Using Backprop](https://www.learnopencv.com/cnn-receptive-field-computation-using-backprop/)|[Code](https://github.com/spmallick/learnopencv/tree/master/PyTorch-Receptive-Field-With-Backprop)|
 |[Augmented Reality using AruCo Markers in OpenCV(C++ and Python)](https://www.learnopencv.com/augmented-reality-using-aruco-markers-in-opencv-(c++-python)/) |[Code](https://github.com/spmallick/learnopencv/tree/master/AugmentedRealityWithArucoMarkers)|

TSNE/README.md

@@ -0,0 +1,69 @@
+# Setup
+
+This code was tested with python 3.7, however, it should work with any python 3.
+
+1. Create and activate virtual environment for experiments with t-SNE.
+
+```bash
+python3 -m venv venv
+source venv/bin/activate
+```
+
+2. install the dependencies
+
+```bash
+python3 -m pip install -r requirements.txt
+```
+
+# Data downloading
+
+Download data from Kaggle and unzip it.
+The easiest way is to use kaggle console API. To setup it, follow [this guide](https://www.kaggle.com/general/74235).
+However, you can download the data using your browser - results will be the same.
+
+After that, execute the following commands:
+
+```bash
+
+kaggle datasets download alessiocorrado99/animals10
+
+mkdir -p data
+
+cd data
+
+unzip ../animals10.zip
+
+cd ..
+
+```
+
+# Executing the T-SNE visualization
+
+```bash
+
+python3 tsne.py
+
+```
+
+Additional options:
+
+```bash
+python3 tsne.py -h
+
+usage: tsne.py [-h] [--path PATH] [--batch BATCH] [--num_images NUM_IMAGES]
+
+optional arguments:
+  -h, --help            show this help message and exit
+  --path PATH
+  --batch BATCH
+  --num_images NUM_IMAGES
+
+```
+
+You can change the data directory with `--path` argument.
+
+Tweak the `--num_images` to speed-up the process - by default it is 500, you can make it smaller.
+
+Tweak the `--batch` to better utilize your PC's resources. The script uses GPU automatically if it available. You may
+want to increase the batch size to utilize the GPU better or decrease it if the default batch size does not fit your
+GPU.

TSNE/animals_dataset.py

@@ -0,0 +1,93 @@
+from os import path, listdir
+import torch
+from torchvision import transforms
+import random
+
+from PIL import Image, ImageFile
+ImageFile.LOAD_TRUNCATED_IMAGES = True
+
+
+colors_per_class = {
+    'dog' : [254, 202, 87],
+    'horse' : [255, 107, 107],
+    'elephant' : [10, 189, 227],
+    'butterfly' : [255, 159, 243],
+    'chicken' : [16, 172, 132],
+    'cat' : [128, 80, 128],
+    'cow' : [87, 101, 116],
+    'sheep' : [52, 31, 151],
+    'spider' : [0, 0, 0],
+    'squirrel' : [100, 100, 255],
+}
+
+
+# processes Animals10 dataset: https://www.kaggle.com/alessiocorrado99/animals10
+class AnimalsDataset(torch.utils.data.Dataset):
+    def __init__(self, data_path, num_images=1000):
+        translation = {'cane' : 'dog',
+                       'cavallo' : 'horse',
+                       'elefante' : 'elephant',
+                       'farfalla' : 'butterfly',
+                       'gallina' : 'chicken',
+                       'gatto' : 'cat',
+                       'mucca' : 'cow',
+                       'pecora' : 'sheep',
+                       'ragno' : 'spider',
+                       'scoiattolo' : 'squirrel'}
+
+        self.classes = translation.values()
+
+        if not path.exists(data_path):
+            raise Exception(data_path + ' does not exist!')
+
+        self.data = []
+
+        folders = listdir(data_path)
+        for folder in folders:
+            label = translation[folder]
+
+            full_path = path.join(data_path, folder)
+            images = listdir(full_path)
+
+            current_data = [(path.join(full_path, image), label) for image in images]
+            self.data += current_data
+
+        num_images = min(num_images, len(self.data))
+        self.data = random.sample(self.data, num_images) # only use num_images images
+
+        # We use the transforms described in official PyTorch ResNet inference example:
+        # https://pytorch.org/hub/pytorch_vision_resnet/.
+        self.transform = transforms.Compose([
+            transforms.Resize(256),
+            transforms.CenterCrop(224),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ])
+
+
+    def __len__(self):
+        return len(self.data)
+
+
+    def __getitem__(self, index):
+        image_path, label = self.data[index]
+
+        image = Image.open(image_path)
+
+        try:
+            image = self.transform(image) # some images in the dataset cannot be processed - we'll skip them
+        except Exception:
+            return None
+
+        dict_data = {
+            'image' : image,
+            'label' : label,
+            'image_path' : image_path
+        }
+        return dict_data
+
+
+# Skips empty samples in a batch
+def collate_skip_empty(batch):
+    batch = [sample for sample in batch if sample] # check that sample is not None
+    return torch.utils.data.dataloader.default_collate(batch)

TSNE/requirements.txt

@@ -0,0 +1,9 @@
+opencv-python>=3.4.1.15
+numpy>=1.18.1
+matplotlib>=3.2.0
+tqdm==4.45.0
+Pillow==7.0.0
+scikit-learn==0.22.2.post1
+scipy==1.4.1
+torch==1.4.0
+torchvision==0.5.0

TSNE/resnet.py

@@ -0,0 +1,37 @@
+import torch
+from torchvision import models
+from torch.hub import load_state_dict_from_url
+
+
+# Define the architecture by modifying resnet.
+# Original code is here
+# https://github.com/pytorch/vision/blob/b2e95657cd5f389e3973212ba7ddbdcc751a7878/torchvision/models/resnet.py
+class ResNet101(models.ResNet):
+    def __init__(self, num_classes=1000, pretrained=True, **kwargs):
+        # Start with standard resnet101 defined here
+        # https://github.com/pytorch/vision/blob/b2e95657cd5f389e3973212ba7ddbdcc751a7878/torchvision/models/resnet.py
+        super().__init__(block=models.resnet.Bottleneck, layers=[3, 4, 23, 3], num_classes=num_classes, **kwargs)
+        if pretrained:
+            state_dict = load_state_dict_from_url(models.resnet.model_urls['resnet101'], progress=True)
+            self.load_state_dict(state_dict)
+
+    # Reimplementing forward pass.
+    # Replacing the following code
+    # https://github.com/pytorch/vision/blob/b2e95657cd5f389e3973212ba7ddbdcc751a7878/torchvision/models/resnet.py#L197-L213
+    def _forward_impl(self, x):
+        # Standard forward for resnet
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        x = self.maxpool(x)
+
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        # Notice there is no forward pass through the original classifier.
+        x = self.avgpool(x)
+        x = torch.flatten(x, 1)
+
+        return x