set cls_mtbc as nonexclusive classification

Author: taigw

pymic/net_run/agent_cls.py

@@ -37,6 +37,7 @@ class ClassificationAgent(NetRunAgent):
     def __init__(self, config, stage = 'train'):
         super(ClassificationAgent, self).__init__(config, stage)
         self.transform_dict  = TransformDict
+        assert(self.task_type in  ["cls", "cls_nexcl"])
 
     def get_stage_dataset_from_config(self, stage):
         assert(stage in ['train', 'valid', 'test'])
@@ -106,7 +107,7 @@ def get_evaluation_score(self, outputs, labels):
             _, preds = torch.max(outputs, 1)
             consis= self.convert_tensor_type(preds ==  labels.data)
             score = torch.mean(consis)
-        elif(self.task_type == "cls_mtbc"): #multi-task binary classification
+        elif(self.task_type == "cls_nexcl"): #nonexclusive classification
             preds = self.convert_tensor_type(outputs > 0.5)
             consis= self.convert_tensor_type(preds ==  labels.data)
             score = torch.mean(consis) 
@@ -299,7 +300,7 @@ def infer(self):
                 if (self.task_type == "cls"):
                     out_prob  = nn.Softmax(dim = 1)(out_digit).detach().cpu().numpy()
                     out_lab   = np.argmax(out_prob, axis=1)
-                else: #self.task_type == "cls_mtbc"
+                else: #self.task_type == "cls_nexcl"
                     out_prob  = nn.Sigmoid()(out_digit).detach().cpu().numpy() 
                     out_lab   = np.asarray(out_prob > 0.5, np.uint8)              
                 for i in range(len(names)):

pymic/util/evaluation_cls.py

@@ -52,8 +52,7 @@ def get_evaluation_score(gt_label, pred_prob, metric):
         raise ValueError("undefined metric: {0:}".format(metric))
     return score
 
-def binary_evaluation(config_file):
-    config = parse_config(config_file)['evaluation']
+def binary_evaluation(config):
     metric_list = config['metric_list']
     gt_csv  = config['ground_truth_csv']
     prob_csv= config['predict_prob_csv']
@@ -71,6 +70,55 @@ def binary_evaluation(config_file):
         score_list.append(score)
         print("{0:}: {1:}".format(metric, score))
 
+    out_csv = prob_csv.replace("prob", "eval")
+    with open(out_csv, mode='w') as csv_file:
+        csv_writer = csv.writer(csv_file, delimiter=',', 
+                            quotechar='"',quoting=csv.QUOTE_MINIMAL)
+        csv_writer.writerow(metric_list)
+        csv_writer.writerow(score_list)
+
+def nexcl_evaluation(config):
+    """
+    evaluation for nonexclusive classification
+    """
+    metric_list = config['metric_list']
+    gt_csv    = config['ground_truth_csv']
+    prob_csv  = config['predict_prob_csv']
+    gt_items  = pd.read_csv(gt_csv)
+    prob_items= pd.read_csv(prob_csv)
+    assert(len(gt_items) == len(prob_items))
+    for i in range(len(gt_items)):
+        assert(gt_items.iloc[i, 0] == prob_items.iloc[i, 0])
+    
+    cls_names = gt_items.columns[1:]
+    cls_num   = len(cls_names)
+    gt_data  = np.asarray(gt_items.iloc[:, 1:cls_num + 1])
+    prob_data = np.asarray(prob_items.iloc[:, 1:cls_num + 1])
+    score_list= []
+    for metric in metric_list:
+        print(metric)
+        score_m = []
+        for c in range(cls_num):
+            gt_data_c = gt_data[:, c:c+1]
+            prob_c = prob_data[:, c]
+            prob_c = np.asarray([1.0 - prob_c, prob_c])
+            prob_c = np.transpose(prob_c)
+            score = get_evaluation_score(gt_data_c, prob_c, metric)
+            score_m.append(score)
+            print(cls_names[c], score)
+        score_avg = np.asarray(score_m).mean()
+        print('avg', score_avg)
+        score_m.append(score_avg)
+        score_list.append(score_m)
+
+    out_csv = prob_csv.replace("prob", "eval")
+    with open(out_csv, mode='w') as csv_file:
+        csv_writer = csv.writer(csv_file, delimiter=',', 
+                            quotechar='"',quoting=csv.QUOTE_MINIMAL)
+        csv_writer.writerow(['metric'] + list(cls_names) + ['avg'])
+        for i in range(len(score_list)):
+            item = metric_list[i : i+1] + score_list[i]
+            csv_writer.writerow(item)
 
 def main():
     if(len(sys.argv) < 2):
@@ -79,7 +127,12 @@ def main():
         exit()
     config_file = str(sys.argv[1])
     assert(os.path.isfile(config_file))
-    binary_evaluation(config_file)
+    config = parse_config(config_file)['evaluation']
+    task_type = config.get('task_type', "cls")
+    if(task_type == "cls"):  # default exclusive classification
+        binary_evaluation(config)
+    else:                    # non exclusive classification
+        nexcl_evaluation(config)
 
 if __name__ == '__main__':
     main()