Skip to content

Commit e389504

Browse files
thucbx99thucbx99
committed
Update format and docs for Pre-trained Model Selection
1 parent 1f23eee commit e389504

File tree

15 files changed

+177
-146
lines changed

15 files changed

+177
-146
lines changed

‎docs/index.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ Transfer Learning
1414
tllib/reweight
1515
tllib/normalization
1616
tllib/regularization
17+
tllib/ranking
1718

1819

1920
.. toctree::

‎docs/tllib/ranking.rst‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
=====================
2+
Ranking
3+
=====================
4+
5+
6+
7+
.. _H_score:
8+
9+
H-score
10+
-------------------------------------------
11+
12+
.. autofunction:: tllib.ranking.hscore.h_score
13+
14+
15+
.. _LEEP:
16+
17+
LEEP: Log Expected Empirical Prediction
18+
-------------------------------------------
19+
20+
.. autofunction:: tllib.ranking.leep.log_expected_empirical_prediction
21+
22+
23+
.. _NCE:
24+
25+
NCE: Negative Conditional Entropy
26+
-------------------------------------------
27+
28+
.. autofunction:: tllib.ranking.nce.negative_conditional_entropy
29+
30+
31+
.. _LogME:
32+
33+
LogME: Log Maximum Evidence
34+
-------------------------------------------
35+
36+
.. autofunction:: tllib.ranking.logme.log_maximum_evidence
37+

‎examples/model_selection/hscore.py‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
sys.path.append('.')
1818
import utils
1919

20-
2120
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
2221

2322

@@ -36,22 +35,26 @@ def main(args):
3635
data_transform = utils.get_transform(resizing=args.resizing)
3736
print("data_transform: ", data_transform)
3837
model = utils.get_model(args.arch, args.pretrained).to(device)
39-
score_dataset, num_classes = utils.get_dataset(args.data, args.root, data_transform, args.sample_rate, args.num_samples_per_classes)
40-
score_loader = DataLoader(score_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True)
38+
score_dataset, num_classes = utils.get_dataset(args.data, args.root, data_transform, args.sample_rate,
39+
args.num_samples_per_classes)
40+
score_loader = DataLoader(score_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
41+
pin_memory=True)
4142
print(f'Using {len(score_dataset)} samples for ranking')
42-
features, predictions, targets = utils.forwarding_dataset(score_loader, model, layer=eval(f'model.{args.layer}'), device=device)
43+
features, predictions, targets = utils.forwarding_dataset(score_loader, model,
44+
layer=eval(f'model.{args.layer}'), device=device)
4345
if args.save_features:
4446
np.save(os.path.join(logger.get_save_dir(), 'features.npy'), features)
4547
np.save(os.path.join(logger.get_save_dir(), 'preds.npy'), predictions)
4648
np.save(os.path.join(logger.get_save_dir(), 'targets.npy'), targets)
4749

4850
print('Conducting transferability calculation')
4951
result = h_score(features, targets)
50-
51-
logger.write(f'# {result:.4f} # data_{args.data}_sr{args.sample_rate}_sc{args.num_samples_per_classes}_model_{args.arch}_layer_{args.layer}\n')
52+
53+
logger.write(
54+
f'# {result:.4f} # data_{args.data}_sr{args.sample_rate}_sc{args.num_samples_per_classes}_model_{args.arch}_layer_{args.layer}\n')
5255
print(f'Results saved in {logger.get_result_dir()}')
5356
logger.close()
54-
57+
5558

5659
if __name__ == '__main__':
5760
parser = argparse.ArgumentParser(description='Ranking pre-trained models with HScore')
@@ -68,22 +71,22 @@ def main(args):
6871
parser.add_argument('-sc', '--num-samples-per-classes', default=None, type=int,
6972
help='number of samples per classes.')
7073
parser.add_argument('-b', '--batch-size', default=48, type=int,
71-
metavar='N', help='mini-batch size (default: 48)')
74+
metavar='N', help='mini-batch size (default: 48)')
7275
parser.add_argument('--resizing', default='res.', type=str)
73-
76+
7477
# model
7578
parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50',
7679
choices=utils.get_model_names(),
7780
help='model to be ranked: ' +
78-
' | '.join(utils.get_model_names()) +
79-
' (default: resnet50)')
81+
' | '.join(utils.get_model_names()) +
82+
' (default: resnet50)')
8083
parser.add_argument('-l', '--layer', default='fc',
8184
help='before which layer features are extracted')
8285
parser.add_argument('--pretrained', default=None,
83-
help="pretrained checkpoint of the backbone. "
84-
"(default: None, use the ImageNet supervised pretrained backbone)")
86+
help="pretrained checkpoint of the backbone. "
87+
"(default: None, use the ImageNet supervised pretrained backbone)")
8588
parser.add_argument("--save_features", action='store_true',
86-
help="whether to save extracted features")
89+
help="whether to save extracted features")
8790

8891
args = parser.parse_args()
89-
main(args)
92+
main(args)

‎examples/model_selection/hscore.sh‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env bash
22

3-
43
# Ranking Pre-trained Model
54
# ======================================================================================================================
65
# CIFAR10
@@ -54,7 +53,6 @@ CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/caltech101 -d Caltech101 -a dense
5453
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/caltech101 -d Caltech101 -a mobilenet_v2 -l classifier[-1] --save_features
5554
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/caltech101 -d Caltech101 -a mnasnet1_0 -l classifier[-1] --save_features
5655

57-
5856
# ======================================================================================================================
5957
# DTD
6058
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/dtd -d DTD -a resnet50 -l fc --save_features
@@ -81,7 +79,6 @@ CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/Oxford-IIIT -d OxfordIIITPets -a
8179
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/Oxford-IIIT -d OxfordIIITPets -a mobilenet_v2 -l classifier[-1] --save_features
8280
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/Oxford-IIIT -d OxfordIIITPets -a mnasnet1_0 -l classifier[-1] --save_features
8381

84-
8582
# ======================================================================================================================
8683
# StanfordCars
8784
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/stanford_cars -d StanfordCars -a resnet50 -l fc --save_features
@@ -95,7 +92,6 @@ CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/stanford_cars -d StanfordCars -a
9592
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/stanford_cars -d StanfordCars -a mobilenet_v2 -l classifier[-1] --save_features
9693
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/stanford_cars -d StanfordCars -a mnasnet1_0 -l classifier[-1] --save_features
9794

98-
9995
# ======================================================================================================================
10096
# SUN397
10197
CUDA_VISIBLE_DEVICES=0 python hscore.py ./data/SUN397 -d SUN397 -a resnet50 -l fc --save_features

‎examples/model_selection/leep.py‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
sys.path.append('.')
1818
import utils
1919

20-
2120
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
2221

2322

@@ -36,24 +35,29 @@ def main(args):
3635
data_transform = utils.get_transform(resizing=args.resizing)
3736
print("data_transform: ", data_transform)
3837
model = utils.get_model(args.arch, args.pretrained).to(device)
39-
score_dataset, num_classes = utils.get_dataset(args.data, args.root, data_transform, args.sample_rate, args.num_samples_per_classes)
40-
score_loader = DataLoader(score_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True)
38+
score_dataset, num_classes = utils.get_dataset(args.data, args.root, data_transform, args.sample_rate,
39+
args.num_samples_per_classes)
40+
score_loader = DataLoader(score_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
41+
pin_memory=True)
4142
print(f'Using {len(score_dataset)} samples for ranking')
42-
features, predictions, targets = utils.forwarding_dataset(score_loader, model, layer=eval(f'model.{args.layer}'), device=device)
43+
features, predictions, targets = utils.forwarding_dataset(score_loader, model,
44+
layer=eval(f'model.{args.layer}'), device=device)
4345
if args.save_features:
4446
np.save(os.path.join(logger.get_save_dir(), 'features.npy'), features)
4547
np.save(os.path.join(logger.get_save_dir(), 'preds.npy'), predictions)
4648
np.save(os.path.join(logger.get_save_dir(), 'targets.npy'), targets)
4749

4850
print('Conducting transferability calculation')
4951
result = leep(predictions, targets)
50-
51-
logger.write(f'# {result:.4f} # data_{args.data}_sr{args.sample_rate}_sc{args.num_samples_per_classes}_model_{args.arch}_layer_{args.layer}\n')
52+
53+
logger.write(
54+
f'# {result:.4f} # data_{args.data}_sr{args.sample_rate}_sc{args.num_samples_per_classes}_model_{args.arch}_layer_{args.layer}\n')
5255
logger.close()
53-
56+
5457

5558
if __name__ == '__main__':
56-
parser = argparse.ArgumentParser(description='Ranking pre-trained models with LEEP (Log Expected Empirical Prediction)')
59+
parser = argparse.ArgumentParser(
60+
description='Ranking pre-trained models with LEEP (Log Expected Empirical Prediction)')
5761

5862
# dataset
5963
parser.add_argument('root', metavar='DIR',
@@ -67,22 +71,22 @@ def main(args):
6771
parser.add_argument('-sc', '--num-samples-per-classes', default=None, type=int,
6872
help='number of samples per classes.')
6973
parser.add_argument('-b', '--batch-size', default=48, type=int,
70-
metavar='N', help='mini-batch size (default: 48)')
74+
metavar='N', help='mini-batch size (default: 48)')
7175
parser.add_argument('--resizing', default='res.', type=str)
72-
76+
7377
# model
7478
parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50',
7579
choices=utils.get_model_names(),
7680
help='model to be ranked: ' +
77-
' | '.join(utils.get_model_names()) +
78-
' (default: resnet50)')
81+
' | '.join(utils.get_model_names()) +
82+
' (default: resnet50)')
7983
parser.add_argument('-l', '--layer', default='fc',
8084
help='before which layer features are extracted')
8185
parser.add_argument('--pretrained', default=None,
82-
help="pretrained checkpoint of the backbone. "
83-
"(default: None, use the ImageNet supervised pretrained backbone)")
86+
help="pretrained checkpoint of the backbone. "
87+
"(default: None, use the ImageNet supervised pretrained backbone)")
8488
parser.add_argument("--save_features", action='store_true',
85-
help="whether to save extracted features")
89+
help="whether to save extracted features")
8690

8791
args = parser.parse_args()
88-
main(args)
92+
main(args)

‎examples/model_selection/leep.sh‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env bash
22

3-
43
# Ranking Pre-trained Model
54
# ======================================================================================================================
65
# CIFAR10
@@ -54,7 +53,6 @@ CUDA_VISIBLE_DEVICES=0 python leep.py ./data/caltech101 -d Caltech101 -a densene
5453
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/caltech101 -d Caltech101 -a mobilenet_v2 -l classifier[-1] --save_features
5554
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/caltech101 -d Caltech101 -a mnasnet1_0 -l classifier[-1] --save_features
5655

57-
5856
# ======================================================================================================================
5957
# DTD
6058
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/dtd -d DTD -a resnet50 -l fc --save_features
@@ -81,7 +79,6 @@ CUDA_VISIBLE_DEVICES=0 python leep.py ./data/Oxford-IIIT -d OxfordIIITPets -a de
8179
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/Oxford-IIIT -d OxfordIIITPets -a mobilenet_v2 -l classifier[-1] --save_features
8280
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/Oxford-IIIT -d OxfordIIITPets -a mnasnet1_0 -l classifier[-1] --save_features
8381

84-
8582
# ======================================================================================================================
8683
# StanfordCars
8784
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/stanford_cars -d StanfordCars -a resnet50 -l fc --save_features
@@ -95,7 +92,6 @@ CUDA_VISIBLE_DEVICES=0 python leep.py ./data/stanford_cars -d StanfordCars -a de
9592
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/stanford_cars -d StanfordCars -a mobilenet_v2 -l classifier[-1] --save_features
9693
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/stanford_cars -d StanfordCars -a mnasnet1_0 -l classifier[-1] --save_features
9794

98-
9995
# ======================================================================================================================
10096
# SUN397
10197
CUDA_VISIBLE_DEVICES=0 python leep.py ./data/SUN397 -d SUN397 -a resnet50 -l fc --save_features

‎examples/model_selection/logme.py‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
sys.path.append('.')
1818
import utils
1919

20-
2120
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
2221

2322

@@ -26,7 +25,6 @@ def main(args):
2625
print(args)
2726
print(f'Calc Transferabilities of {args.arch} on {args.data}')
2827

29-
3028
try:
3129
features = np.load(os.path.join(logger.get_save_dir(), 'features.npy'))
3230
predictions = np.load(os.path.join(logger.get_save_dir(), 'preds.npy'))
@@ -37,21 +35,25 @@ def main(args):
3735
data_transform = utils.get_transform(resizing=args.resizing)
3836
print("data_transform: ", data_transform)
3937
model = utils.get_model(args.arch, args.pretrained).to(device)
40-
score_dataset, num_classes = utils.get_dataset(args.data, args.root, data_transform, args.sample_rate, args.num_samples_per_classes)
41-
score_loader = DataLoader(score_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True)
38+
score_dataset, num_classes = utils.get_dataset(args.data, args.root, data_transform, args.sample_rate,
39+
args.num_samples_per_classes)
40+
score_loader = DataLoader(score_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers,
41+
pin_memory=True)
4242
print(f'Using {len(score_dataset)} samples for ranking')
43-
features, predictions, targets = utils.forwarding_dataset(score_loader, model, layer=eval(f'model.{args.layer}'), device=device)
43+
features, predictions, targets = utils.forwarding_dataset(score_loader, model,
44+
layer=eval(f'model.{args.layer}'), device=device)
4445
if args.save_features:
4546
np.save(os.path.join(logger.get_save_dir(), 'features.npy'), features)
4647
np.save(os.path.join(logger.get_save_dir(), 'preds.npy'), predictions)
4748
np.save(os.path.join(logger.get_save_dir(), 'targets.npy'), targets)
4849

4950
print('Conducting transferability calculation')
5051
result = logme(features, targets)
51-
52-
logger.write(f'# {result:.4f} # data_{args.data}_sr{args.sample_rate}_sc{args.num_samples_per_classes}_model_{args.arch}_layer_{args.layer}\n')
52+
53+
logger.write(
54+
f'# {result:.4f} # data_{args.data}_sr{args.sample_rate}_sc{args.num_samples_per_classes}_model_{args.arch}_layer_{args.layer}\n')
5355
logger.close()
54-
56+
5557

5658
if __name__ == '__main__':
5759
parser = argparse.ArgumentParser(description='Ranking pre-trained models with LogME (Log Maximum Evidence)')
@@ -68,22 +70,22 @@ def main(args):
6870
parser.add_argument('-sc', '--num-samples-per-classes', default=None, type=int,
6971
help='number of samples per classes.')
7072
parser.add_argument('-b', '--batch-size', default=48, type=int,
71-
metavar='N', help='mini-batch size (default: 48)')
73+
metavar='N', help='mini-batch size (default: 48)')
7274
parser.add_argument('--resizing', default='res.', type=str)
7375

7476
# model
7577
parser.add_argument('-a', '--arch', metavar='ARCH', default='resnet50',
7678
choices=utils.get_model_names(),
7779
help='model to be ranked: ' +
78-
' | '.join(utils.get_model_names()) +
79-
' (default: resnet50)')
80+
' | '.join(utils.get_model_names()) +
81+
' (default: resnet50)')
8082
parser.add_argument('-l', '--layer', default='fc',
8183
help='before which layer features are extracted')
8284
parser.add_argument('--pretrained', default=None,
83-
help="pretrained checkpoint of the backbone. "
84-
"(default: None, use the ImageNet supervised pretrained backbone)")
85+
help="pretrained checkpoint of the backbone. "
86+
"(default: None, use the ImageNet supervised pretrained backbone)")
8587
parser.add_argument("--save_features", action='store_true',
86-
help="whether to save extracted features")
88+
help="whether to save extracted features")
8789

8890
args = parser.parse_args()
89-
main(args)
91+
main(args)

‎examples/model_selection/logme.sh‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
#!/usr/bin/env bash
22

3-
43
# Ranking Pre-trained Model
54
# ======================================================================================================================
65
# CIFAR10
@@ -54,7 +53,6 @@ CUDA_VISIBLE_DEVICES=0 python logme.py ./data/caltech101 -d Caltech101 -a densen
5453
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/caltech101 -d Caltech101 -a mobilenet_v2 -l classifier[-1] --save_features
5554
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/caltech101 -d Caltech101 -a mnasnet1_0 -l classifier[-1] --save_features
5655

57-
5856
# ======================================================================================================================
5957
# DTD
6058
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/dtd -d DTD -a resnet50 -l fc --save_features
@@ -81,7 +79,6 @@ CUDA_VISIBLE_DEVICES=0 python logme.py ./data/Oxford-IIIT -d OxfordIIITPets -a d
8179
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/Oxford-IIIT -d OxfordIIITPets -a mobilenet_v2 -l classifier[-1] --save_features
8280
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/Oxford-IIIT -d OxfordIIITPets -a mnasnet1_0 -l classifier[-1] --save_features
8381

84-
8582
# ======================================================================================================================
8683
# StanfordCars
8784
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/stanford_cars -d StanfordCars -a resnet50 -l fc --save_features
@@ -95,7 +92,6 @@ CUDA_VISIBLE_DEVICES=0 python logme.py ./data/stanford_cars -d StanfordCars -a d
9592
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/stanford_cars -d StanfordCars -a mobilenet_v2 -l classifier[-1] --save_features
9693
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/stanford_cars -d StanfordCars -a mnasnet1_0 -l classifier[-1] --save_features
9794

98-
9995
# ======================================================================================================================
10096
# SUN397
10197
CUDA_VISIBLE_DEVICES=0 python logme.py ./data/SUN397 -d SUN397 -a resnet50 -l fc --save_features

0 commit comments

Comments
 (0)