Merge pull request #8 from mirrorboat/main

fix bug for gen image eval

Author: mirrorboat

configs/eval/gen_image.yaml

@@ -0,0 +1,35 @@
+model_cache_path: '../ckpt' # Path to cache models
+num_workers: 2
+dependencies: [image]
+
+# data:
+#   image:
+#     meta_data_path: "demos/image_eval/image.json"
+#     data_path: "demos/image_eval/images"
+#     ref_meta_data_path: "demos/image_eval/image_copy.json"
+#     ref_data_path: "demos/image_eval/images"
+#     image_key: 'image'
+#     formatter: 'GenImageFormatter'
+
+data:
+  image:
+    meta_data_path: "demos/image_eval/gen_image.json"
+    data_path: "demos/image_eval/gen_images"
+    ref_meta_data_path: "demos/image_eval/ref_image.json"
+    ref_data_path: "demos/image_eval/images"
+    image_key: 'image'
+    formatter: 'GenImageFormatter'
+
+scorers:
+  ISScorer:
+    batch_size: 32
+    device: cpu
+    dims: 2048
+    resize: True
+    splits: 10
+  KIDScorer:
+    batch_size: 50
+    num_workers: 8
+    dims: 2048
+    device: cpu
+    model: inception

configs/eval/gen_image_example.yaml

@@ -4,10 +4,8 @@ dependencies: [image]
 
 data:
   image:
-    meta_data_path: "../data/image_data.json"
-    data_path: "../data/images"
-    ref_meta_data_path: "../data/data4IS2.jsonl"
-    ref_data_path: "../data/synth_challenge"
+    meta_data_path: "demos/image_eval/image.json"
+    data_path: "demos/image_eval/images"
     image_key: 'image'
     id_key: 'id'
     formatter: 'PureImageFormatter'
@@ -16,58 +14,58 @@ scorers:
   LiqeScorer:
       batch_size: 2
       device: "cuda"
-  ArniqaScorer:
-      batch_size: 2
-      device: "cuda"
-  TopiqScorer:
-      batch_size: 2
-      device: "cuda"
-  ClipiqaScorer:
-      batch_size: 2
-      device: "cuda"
-  QalignScorer:
-      batch_size: 2
-      device: "cuda"
-  ManiqaScorer:
-      batch_size: 2
-      device: "cuda"
-  MusiqScorer:
-      batch_size: 2
-      device: "cuda"
-  DbcnnScorer:
-      batch_size: 2
-      device: "cuda"
-  Pqa2piqScorer:
-      batch_size: 2
-      device: "cuda"
-  HyperiqaScorer:
-      batch_size: 2
-      device: "cuda"
-  NimaScorer:
-      batch_size: 2
-      device: "cuda"
-  WadiqamScoreer:
-      batch_size: 2
-      device: "cuda"
-  CnniqaScorer:
-      batch_size: 2
-      device: "cuda"
-  NrqmScoreer:
-      batch_size: 2
-      device: "cuda"
-  PiScorer:
-      batch_size: 2
-      device: "cuda"
-  BrisqueScorer:
-      batch_size: 2
-      device: "cuda"
-  IlniqeScorer:
-      batch_size: 2
-      device: "cuda"
-  NiqeScorer:
-      batch_size: 2
-      device: "cuda"
-  PiqeScorer:
-      batch_size: 2
-      device: "cuda"
+#   ArniqaScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   TopiqScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   ClipiqaScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   QalignScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   ManiqaScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   MusiqScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   DbcnnScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   Pqa2piqScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   HyperiqaScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   NimaScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   WadiqamScoreer:
+#       batch_size: 2
+#       device: "cuda"
+#   CnniqaScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   NrqmScoreer:
+#       batch_size: 2
+#       device: "cuda"
+#   PiScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   BrisqueScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   IlniqeScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   NiqeScorer:
+#       batch_size: 2
+#       device: "cuda"
+#   PiqeScorer:
+#       batch_size: 2
+#       device: "cuda"
 

configs/eval/gen_image_new.yaml

@@ -56,7 +56,7 @@ def inception_score(self, sample, device, batch_size=32, resize=False, splits=1)
 
         # Load inception model
         inception_model = inception_v3(pretrained=True, transform_input=False).type(dtype)
-        inception_model.eval();
+        inception_model.eval()
         up = nn.Upsample(size=(299, 299), mode='bilinear').type(dtype)
 
         def get_pred(x):

configs/eval/image_eval.yaml

@@ -13,7 +13,7 @@
 from torch.nn.functional import adaptive_avg_pool2d
 import torchvision.transforms as transforms
 from dataflow.Eval.image.kid.inception import InceptionV3
-from dataflow.Eval.image..kid.lenet import LeNet5
+from dataflow.Eval.image.kid.lenet import LeNet5
 from PIL import Image
 import sys
 
@@ -125,7 +125,6 @@ def calculate_kid_given_paths(self, sample, ref_sample, batch_size, cuda, dims,
             model.load_state_dict(torch.load('./kid/lenet.pth'))
         if cuda:
             model.cuda()
-
         act_true = self._compute_activations(ref_sample, model, batch_size, dims, cuda, model_type)
         results = []
         actj = self._compute_activations(sample, model, batch_size, dims, cuda, model_type)

dataflow/Eval/image/is_scorer.py

@@ -333,10 +333,6 @@ def evaluate(self, dataset):
         idx_list = list(range(len(dataset)))
         dataset.score_record.item_score[self.__class__.__name__]['Default'][idx_list] = score_list
 
-        # assert len(score_list) == len(id_list), "The number of scores and ids should be the same."
-        # id_score_dict = dict(zip(id_list, score_list))
-        # return id_score_dict
-        # dataset.scores_list[self.__class__.__name__] = score_list
         return self.__class__.__name__, dataset.score_record.item_score[self.__class__.__name__]
 
     def __call__(self, dataset):
@@ -387,14 +383,13 @@ def evaluate(self, datasets):
             else:
                 ref_dataloader = DataLoader(ref_dataset, batch_size=len(ref_dataset), shuffle=False, num_workers=self.num_workers)
             for data, ref_data in zip(dataloader, ref_dataloader):
-                scores = self.evaluate_batch(data[1], ref_data[1])
+                scores = self.evaluate_batch(data, ref_data)
                 if isinstance(scores, torch.Tensor):
                     scores = scores.cpu().detach().numpy
                 score_list.append(scores) # list
 
-        dataset.score_record.meta_score[self.scorer_name]['Default'] = score_list[0]
-        # if ref_dataset is not None:
-        #     ref_dataset.meta_score[self.scorer_name] = score_list[0]
+        dataset.score_record.meta_score[self.scorer_name] = score_list[0][0]
+
         return self.scorer_name, dataset.score_record.meta_score[self.scorer_name]
 
 class VideoScorer(Scorer):

dataflow/Eval/image/kid_scorer.py

@@ -0,0 +1,32 @@
+[
+    {
+        "image": "1e558d58-f53e-422e-9715-19d4c12c093d.png"
+    },
+    {
+        "image": "202a9b83-28e5-4002-9858-64a4cb734f91.png"
+    },
+    {
+        "image": "65250ac7-c2ed-4ed1-b375-0916649c849b.png"
+    },
+    {
+        "image": "d10f73b1-ecff-4562-a3b9-3d2d0fc0b82a.png"
+    },
+    {
+        "image": "bda58b80-f069-45f5-9eb3-5d023fb731ec.png"
+    },
+    {
+        "image": "a34a8f2f-e3f5-42b7-9f18-cad076ad70bb.png"
+    },
+    {
+        "image": "67d2b934-2be9-470b-898e-98251a81e74d.png"
+    },
+    {
+        "image": "1de8c101-7d18-45c9-90ad-6b27bc9b565f.png"
+    },
+    {
+        "image": "cd2eed6b-8aea-40f1-a635-3cb9b4f1f460.png"
+    },
+    {
+        "image": "e5a47fa8-f901-40cb-9f3d-39b048152fe2.png"
+    }
+]