Merge branch 'main' of https://github.com/NUS-HPC-AI-Lab/DD-Ranking

Author: lizekai-richard

.github/workflows/doc_deploy.yml

@@ -0,0 +1,39 @@
+name: Deploy
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # To push a branch
+      pages: write  # To push to a GitHub Pages site
+      id-token: write # To update the deployment status
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install latest mdbook
+        run: |
+          tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name')
+          url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz"
+          mkdir mdbook
+          curl -sSL $url | tar -xz --directory=./mdbook
+          echo `pwd`/mdbook >> $GITHUB_PATH
+      - name: Build Book
+        run: |
+          # This assumes your book is in the root of your repository.
+          # Just add a `cd` here if you need to change to another directory.
+          mdbook build
+      - name: Setup Pages
+        uses: actions/configure-pages@v4
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          # Upload entire repository
+          path: 'book'
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

book.toml

@@ -0,0 +1,10 @@
+# Documentation
+# * mdbook   https://rust-lang.github.io/mdBook/
+# * template https://github.com/kg4zow/mdbook-template/
+
+[book]
+authors             = ["DD-Ranking Team"]
+language            = "en"
+multilingual        = false
+src                 = "doc"
+title               = "DD-Ranking Benchmark"

configs/Demo_Hard_Label.yaml

@@ -1,7 +1,18 @@
+
+# real data
 dataset: "CIFAR10"
 real_data_path: "./dataset/"
+custom_val_trans: None
+
+# synthetic data
 ipc: 10
+im_size: (32, 32)
+
+# agent model
 model_name: "ConvNet-3"
+use_torchvision: False
+
+# data augmentation
 data_aug_func: "dsa"
 aug_params: {
   "prob_flip": 0.5, 
@@ -13,16 +24,20 @@ aug_params: {
   "ratio_crop_pad": 0.125, 
   "ratio_cutout": 0.5
 }
+use_zca: False
+
+# training specifics
 optimizer: "sgd"
 lr_scheduler: "step"
 weight_decay: 0.0005
 momentum: 0.9
 num_eval: 5
-im_size: (32, 32)
 num_epochs: 1000
-use_zca: False
-batch_size: 256
+syn_batch_size: 128
+real_batch_size: 256
 default_lr: 0.01
 num_workers: 4
-save_path: "./results/my_method_hard_label_scores.csv"
-device: "cuda"
+device: "cuda"
+
+# save path
+save_path: "./results/my_method_hard_label_scores.csv"

configs/Demo_Soft_Label.yaml

@@ -1,7 +1,19 @@
+# real data
 dataset: "CIFAR10"
 real_data_path: "./dataset/"
+custom_val_trans: None
+
+# synthetic data
 ipc: 10
+im_size: (32, 32)
+
+# agent model
 model_name: "ConvNet-3"
+stu_use_torchvision: False
+tea_use_torchvision: False
+teacher_dir: "./teacher_models"
+
+# data augmentation
 data_aug_func: "dsa"
 aug_params: {
     "prob_flip": 0.5,
@@ -13,19 +25,25 @@ aug_params: {
     "ratio_crop_pad": 0.125,
     "ratio_cutout": 0.5
 }
+use_zca: True
+
+# soft label settings
 soft_label_mode: "S"
 soft_label_criterion: "sce"
+temperature: 1.0
+
+# training specifics
 optimizer: "sgd"
 lr_scheduler: "step"
-temperature: 20.0
 weight_decay: 0.0005
 momentum: 0.9
 num_eval: 5
-im_size: (32, 32)
 num_epochs: 1000
-use_zca: True
-batch_size: 256
 default_lr: 0.01
 num_workers: 4
-save_path: "./results/my_method_soft_label_scores.csv"
 device: "cuda"
+syn_batch_size: 128
+real_batch_size: 256
+
+# save path
+save_path: "./results/my_method_soft_label_scores.csv"

dd_ranking/aug/cutmix.py

@@ -5,15 +5,6 @@
 
 class Cutmix_Augmentation:
     def __init__(self, params: dict):
-        # self.transform = kornia.augmentation.RandomCutMixV2(
-        #     num_mix = params["times"],
-        #     cut_size = params["size"],
-        #     same_on_batch = params["same_on_batch"],
-        #     beta = params["beta"],
-        #     keepdim = params["keep_dim"],
-        #     p = params["prob"]
-        # )
-
         self.cutmix_p = params["cutmix_p"]
 
     def rand_bbox(self, size, lam):

dd_ranking/aug/mixup.py

@@ -5,13 +5,6 @@
 
 class Mixup_Augmentation:
     def __init__(self, params: dict):
-        # self.transform = kornia.augmentation.RandomMixUpV2(
-        #     lambda_val = params["lambda_range"],
-        #     same_on_batch = params["same_on_batch"],
-        #     keepdim = params["keepdim"],
-        #     p = params["prob"]
-        # )
-
         self.mixup_p = params["mixup_p"]
 
     def mixup(self, images):

dd_ranking/loss/kl.py

@@ -1,17 +1,16 @@
 import torch
 import torch.nn as nn
 import torch.nn.functional as F
-from torch.nn import KLDivLoss
 
 
 class KLDivergenceLoss(nn.Module):
     def __init__(self, temperature=1.2):
         super(KLDivergenceLoss, self).__init__()
         self.temperature = temperature
-        self.kl = KLDivLoss(reduction='batchmean')
 
     def forward(self, stu_outputs, tea_outputs):
         stu_probs = F.log_softmax(stu_outputs / self.temperature, dim=1)
-        tea_probs = F.softmax(tea_outputs / self.temperature, dim=1)
-        loss = self.kl(stu_probs, tea_probs)
+        with torch.no_grad():
+            tea_probs = F.softmax(tea_outputs / self.temperature, dim=1)
+        loss = F.kl_div(stu_probs, tea_probs, reduction='batchmean')
         return loss

dd_ranking/loss/sce.py

@@ -4,12 +4,13 @@
 
 
 class SoftCrossEntropyLoss(nn.Module):
-    def __init__(self):
+    def __init__(self, temperature=1.2):
         super(SoftCrossEntropyLoss, self).__init__()
+        self.temperature = temperature
 
     def forward(self, stu_outputs, tea_outputs):
-        input_log_likelihood = -F.log_softmax(stu_outputs, dim=1)
-        target_log_likelihood = F.softmax(tea_outputs, dim=1)
+        input_log_likelihood = -F.log_softmax(stu_outputs / self.temperature, dim=1)
+        target_log_likelihood = F.softmax(tea_outputs / self.temperature, dim=1)
         batch_size = stu_outputs.size(0)
         loss = torch.sum(torch.mul(input_log_likelihood, target_log_likelihood)) / batch_size
         return loss