Bug fix: difference between "otx eval" score and evaluation while training (#2307)

* sort labels in right order

* fix data adapter

* refine bugfix

* refine bugfix

* implement unit test

* align with pre-commit

Author: eunwoosh

src/otx/algorithms/segmentation/adapters/openvino/task.py

@@ -260,8 +260,6 @@ def add_prediction(
 
             if dump_soft_prediction:
                 for label_index, label in self._label_dictionary.items():
-                    if label_index == 0:
-                        continue
                     current_label_soft_prediction = soft_prediction[:, :, label_index]
                     if process_soft_prediction:
                         current_label_soft_prediction = get_activation_map(current_label_soft_prediction)

src/otx/algorithms/segmentation/task.py

@@ -91,7 +91,7 @@ def __init__(self, task_environment: TaskEnvironment, output_path: Optional[str]
         self._model_name = task_environment.model_template.name
         self._train_type = self._hyperparams.algo_backend.train_type
         self.metric = "mDice"
-        self._label_dictionary = dict(enumerate(sorted(self._labels), 1))
+        self._label_dictionary = dict(enumerate(self._labels, 1))  # It should have same order as model class order
 
         self._model_dir = os.path.join(
             os.path.abspath(os.path.dirname(self._task_environment.model_template.model_template_path)),
@@ -287,8 +287,6 @@ def _add_predictions_to_dataset(self, prediction_results, dataset, dump_soft_pre
 
             if dump_soft_prediction:
                 for label_index, label in self._label_dictionary.items():
-                    if label_index == 0:
-                        continue
                     current_label_soft_prediction = soft_prediction[:, :, label_index]
                     if process_soft_prediction:
                         current_label_soft_prediction = get_activation_map(current_label_soft_prediction)

src/otx/api/usecases/evaluation/dice.py

@@ -114,7 +114,6 @@ def __compute_dice_averaged_over_pixels(
         resultset_labels = set(resultset.prediction_dataset.get_labels() + resultset.ground_truth_dataset.get_labels())
         model_labels = set(resultset.model.configuration.get_label_schema().get_labels(include_empty=False))
         labels = sorted(resultset_labels.intersection(model_labels))
-        labels_map = {label: i + 1 for i, label in enumerate(labels)}
         hard_predictions = []
         hard_references = []
         for prediction_item, reference_item in zip(
@@ -126,6 +125,8 @@ def __compute_dice_averaged_over_pixels(
             except:
                 # when item consists of masks with Image properties
                 # TODO (sungchul): how to add condition to check if polygon or mask?
+                labels_map = {label: i + 1 for i, label in enumerate(labels)}
+
                 def combine_masks(annotations):
                     combined_mask = None
                     for annotation in annotations:

src/otx/core/data/adapter/segmentation_dataset_adapter.py

@@ -53,10 +53,9 @@ def get_otx_dataset(self) -> DatasetEntity:
         self.updated_label_id: Dict[int, int] = {}
 
         if hasattr(self, "data_type_candidates"):
-            if self.data_type_candidates[0] == "voc":
+            if "voc" in self.data_type_candidates[0]:
                 self.set_voc_labels()
-
-            if self.data_type_candidates[0] == "common_semantic_segmentation":
+            elif self.data_type_candidates[0] == "common_semantic_segmentation":
                 self.set_common_labels()
 
         else:

tests/unit/algorithms/segmentation/adapters/mmseg/datasets/test_dataset.py

@@ -67,7 +67,7 @@ def test_classes_sorted(self, mocker) -> None:
         self.otx_dataset: DatasetEntity = DatasetEntity(items=[dataset_item()])
         self.pipeline: list[dict] = [{"type": "LoadImageFromOTXDataset", "to_float32": True}]
         self.classes: list[str] = [f"class_{i+1}" for i in range(11)]
-        labels_entities = [label_entity(name, i) for i, name in enumerate(self.classes)]
+        labels_entities = [label_entity(name, str(i)) for i, name in enumerate(self.classes)]
 
         mocker.patch.object(MPASegDataset, "filter_labels", return_value=labels_entities)
 

tests/unit/algorithms/segmentation/adapters/test_otx_segmentation_task.py

@@ -8,6 +8,8 @@
 import pytest
 from mmcv import ConfigDict
 
+from otx.api.entities.label import LabelEntity, Domain
+from otx.api.entities.id import ID
 from otx.algorithms.segmentation.adapters.mmseg.task import MMSegmentationTask
 from otx.algorithms.segmentation.adapters.mmseg.models.heads import otx_head_factory
 from otx.api.configuration.helper import create
@@ -67,3 +69,18 @@ def test_save_model(self, otx_model, mocker):
 
         mocker_load.assert_called_once()
         mocker_save.assert_called_once()
+
+    @e2e_pytest_unit
+    def test_label_order(self, mocker):
+        mocker.patch("otx.algorithms.segmentation.task.os")
+        mocker.patch("otx.algorithms.segmentation.task.TRAIN_TYPE_DIR_PATH")
+        mock_environemnt = mocker.MagicMock()
+
+        fake_label = []
+        for i in range(20):
+            fake_label.append(LabelEntity(name=f"class_{i}", domain=Domain.SEGMENTATION, id=ID(str(i))))
+        mock_environemnt.get_labels.return_value = fake_label
+        task = MMSegmentationTask(mock_environemnt)
+
+        for i, label_entity in task._label_dictionary.items():
+            assert label_entity.name == f"class_{i-1}"