Merge back 1.2.3 -> 1.3.1 (#2192)

* Optimize counting train & inference speed and memory consumption (#2172)

* Do not skip full img tile classifier + Fix Sequencial Export Issue (#2174)

* Add warning message to tiling parameter (#2193)

* Change the way to pad for instance-segmentation

* Hotfix: use 0 confidence threshold when computing best threshold based on F1

* Fix possible bw issue in exportable code (#2212)

* Fix ellipse load (#2214)

---------

Signed-off-by: Songki Choi <[email protected]>
Co-authored-by: Sungman Cho <[email protected]>
Co-authored-by: jaegukhyun <[email protected]>
Co-authored-by: Eugene Liu <[email protected]>
Co-authored-by: sungmanc <[email protected]>
Co-authored-by: Vladislav Sovrasov <[email protected]>

Author: 6 people

CHANGELOG.md

@@ -6,7 +6,7 @@ All notable changes to this project will be documented in this file.
 
 ### Enhancements
 
--
+- n/a
 
 ### Bug fixes
 
@@ -46,6 +46,23 @@ All notable changes to this project will be documented in this file.
 
 ### Known issues
 
+- OpenVINO(==2022.3) IR inference is not working well on 2-stage models (e.g. Mask-RCNN) exported from torch==1.13.1
+  (working well up to torch==1.12.1) (<https://github.com/openvinotoolkit/training_extensions/issues/1906>)
+
+## \[v1.2.3\]
+
+### Bug fixes
+
+- Hotfix: use 0 confidence threshold when computing best threshold based on F1
+
+## \[v1.2.2\]
+
+### Enhancements
+
+- Improve warning message for tiling configurable parameter
+
+### Known issues
+
 - OpenVINO(==2022.3) IR inference is not working well on 2-stage models (e.g. Mask-RCNN) exported from torch==1.13.1
   (working well up to torch==1.12.1) (<https://github.com/openvinotoolkit/training_extensions/issues/1906>)
 
@@ -57,12 +74,14 @@ All notable changes to this project will be documented in this file.
 - Integrate new ignored loss in semantic segmentation (<https://github.com/openvinotoolkit/training_extensions/pull/2065>, <https://github.com/openvinotoolkit/training_extensions/pull/2111>)
 - Optimize YOLOX data pipeline (<https://github.com/openvinotoolkit/training_extensions/pull/2075>)
 - Tiling Spatial Concatenation for OpenVINO IR (<https://github.com/openvinotoolkit/training_extensions/pull/2052>)
+- Optimize counting train & inference speed and memory consumption (<https://github.com/openvinotoolkit/training_extensions/pull/2172>)
 
 ### Bug fixes
 
 - Bug fix: value of validation variable is changed after auto decrease batch size (<https://github.com/openvinotoolkit/training_extensions/pull/2053>)
 - Fix tiling 0 stride issue in parameter adapter (<https://github.com/openvinotoolkit/training_extensions/pull/2078>)
 - Fix Tiling NNCF (<https://github.com/openvinotoolkit/training_extensions/pull/2081>)
+- Do not skip full img tile classifier + Fix Sequencial Export Issue (<https://github.com/openvinotoolkit/training_extensions/pull/2174>)
 
 ## \[v1.2.0\]
 

docs/source/guide/release_notes/index.rst

@@ -1,6 +1,18 @@
 Releases
 ========
 
+***************
+[v1.3.0] (2Q23)
+***************
+
+- Support direct annotation input for COCO format
+- Action task supports multi GPU training
+- Support storage cache in Apache Arrow using Datumaro for action tasks
+- Add a simplified greedy labels postprocessing for hierarchical classification
+- Support auto adapting batch size
+- Support auto adapting num_workers
+- Support noisy label detection for detection tasks
+
 *************
 v1.2.0 (1Q23)
 *************

otx/algorithms/common/adapters/mmcv/tasks/exporter.py

@@ -35,6 +35,14 @@ def run(self, cfg, **kwargs):  # noqa: C901
             pipeline = dataset.get("pipeline", [])
             pipeline += cfg.data.test.get("pipeline", [])
             cfg.data.test.pipeline = pipeline
+        for pipeline in cfg.data.test.pipeline:
+            if pipeline.get("transforms", None):
+                transforms = pipeline.transforms
+                for transform in transforms:
+                    if transform.type == "Collect":
+                        for collect_key in transform["keys"]:
+                            if collect_key != "img":
+                                transform["keys"].remove(collect_key)
 
         model_builder = kwargs.get("model_builder")
         try:

otx/algorithms/common/adapters/mmdeploy/apis.py

@@ -261,7 +261,7 @@ def extract_partition(
             model_name: str = "model",
         ):
             """Function for extracting partition."""
-
+            reset_mark_function_count()
             model_onnx = MMdeployExporter.torch2onnx(
                 output_dir,
                 input_data,

otx/algorithms/common/configs/training_base.py

@@ -227,6 +227,13 @@ class BasePostprocessing(ParameterGroup):
             affects_outcome_of=ModelLifecycle.INFERENCE,
         )
 
+        use_ellipse_shapes = configurable_boolean(
+            default_value=False,
+            header="Use ellipse shapes",
+            description="Use direct ellipse shape in inference instead of polygon from mask",
+            affects_outcome_of=ModelLifecycle.INFERENCE,
+        )
+
     @attrs
     class BaseNNCFOptimization(ParameterGroup):
         """BaseNNCFOptimization for OTX Algorithms."""
@@ -328,7 +335,9 @@ class BaseTilingParameters(ParameterGroup):
             default_value=False,
             header="Enable tiling",
             description="Set to True to allow tiny objects to be better detected.",
-            warning="Tiling trades off speed for accuracy as it increases the number of images to be processed.",
+            warning="Tiling trades off speed for accuracy as it increases the number of images to be processed. "
+            "Important: In the current version, depending on the dataset size and the available hardware resources, "
+            "a model may not train successfully when tiling is enabled.",
             affects_outcome_of=ModelLifecycle.NONE,
         )
 
@@ -357,7 +366,7 @@ class BaseTilingParameters(ParameterGroup):
             description="Tile Image Size",
             default_value=400,
             min_value=100,
-            max_value=1024,
+            max_value=4096,
             affects_outcome_of=ModelLifecycle.NONE,
         )
 
@@ -375,7 +384,7 @@ class BaseTilingParameters(ParameterGroup):
             description="Max object per image",
             default_value=1500,
             min_value=1,
-            max_value=10000,
+            max_value=5000,
             affects_outcome_of=ModelLifecycle.NONE,
         )
 
@@ -395,4 +404,26 @@ class BaseTilingParameters(ParameterGroup):
             affects_outcome_of=ModelLifecycle.NONE,
         )
 
+        tile_sampling_ratio = configurable_float(
+            header="Sampling Ratio for entire tiling",
+            description="Since tiling train and validation to all tile from large image, "
+            "usually it takes lots of time than normal training."
+            "The tile_sampling_ratio is ratio for sampling entire tile dataset."
+            "Sampling tile dataset would save lots of time for training and validation time."
+            "Note that sampling will be applied to training and validation dataset, not test dataset.",
+            default_value=1.0,
+            min_value=0.000001,
+            max_value=1.0,
+            affects_outcome_of=ModelLifecycle.NONE,
+        )
+
+        object_tile_ratio = configurable_float(
+            header="Object tile ratio",
+            description="The desired ratio of min object size and tile size.",
+            default_value=0.03,
+            min_value=0.00,
+            max_value=1.00,
+            affects_outcome_of=ModelLifecycle.NONE,
+        )
+
     tiling_parameters = add_parameter_group(BaseTilingParameters)

otx/algorithms/detection/adapters/mmdet/datasets/dataset.py

@@ -14,7 +14,6 @@
 # See the License for the specific language governing permissions
 # and limitations under the License.
 
-import tempfile
 from collections import OrderedDict
 from copy import copy
 from typing import Any, Dict, List, Sequence, Tuple, Union
@@ -32,6 +31,7 @@
 from otx.api.entities.dataset_item import DatasetItemEntity
 from otx.api.entities.datasets import DatasetEntity
 from otx.api.entities.label import Domain, LabelEntity
+from otx.api.entities.subset import Subset
 from otx.api.utils.shape_factory import ShapeFactory
 
 from .tiling import Tile
@@ -274,6 +274,7 @@ def evaluate(  # pylint: disable=too-many-branches
             if metric not in allowed_metrics:
                 raise KeyError(f"metric {metric} is not supported")
             annotations = [self.get_ann_info(i) for i in range(len(self))]
+            assert len(annotations) == len(results), "annotation length does not match prediction results"
             iou_thrs = [iou_thr] if isinstance(iou_thr, float) else iou_thr
             if metric == "mAP":
                 assert isinstance(iou_thrs, list)
@@ -306,7 +307,7 @@ def evaluate(  # pylint: disable=too-many-branches
 
 # pylint: disable=too-many-arguments
 @DATASETS.register_module()
-class ImageTilingDataset:
+class ImageTilingDataset(OTXDetDataset):
     """A wrapper of tiling dataset.
 
     Suitable for training small object dataset. This wrapper composed of `Tile`
@@ -330,6 +331,8 @@ class ImageTilingDataset:
             after NMS, only top max_per_img will be kept. Defaults to 200.
         max_annotation (int, optional): Limit the number of ground truth by
             randomly select 5000 due to RAM OOM. Defaults to 5000.
+        sampling_ratio (flaot): Ratio for sampling entire tile dataset.
+        include_full_img (bool): Whether to include full image in the dataset.
     """
 
     def __init__(
@@ -344,28 +347,29 @@ def __init__(
         max_annotation=5000,
         filter_empty_gt=True,
         test_mode=False,
+        sampling_ratio=1.0,
+        include_full_img=False,
     ):
         self.dataset = build_dataset(dataset)
         self.CLASSES = self.dataset.CLASSES
-        self.tmp_dir = tempfile.TemporaryDirectory()  # pylint: disable=consider-using-with
 
         self.tile_dataset = Tile(
             self.dataset,
             pipeline,
-            tmp_dir=self.tmp_dir,
             tile_size=tile_size,
             overlap=overlap_ratio,
             min_area_ratio=min_area_ratio,
             iou_threshold=iou_threshold,
             max_per_img=max_per_img,
             max_annotation=max_annotation,
-            filter_empty_gt=False if test_mode else filter_empty_gt,
+            filter_empty_gt=filter_empty_gt if self.dataset.otx_dataset[0].subset != Subset.TESTING else False,
+            sampling_ratio=sampling_ratio if self.dataset.otx_dataset[0].subset != Subset.TESTING else 1.0,
+            include_full_img=include_full_img if self.dataset.otx_dataset[0].subset != Subset.TESTING else True,
         )
         self.flag = np.zeros(len(self), dtype=np.uint8)
         self.pipeline = Compose(pipeline)
         self.test_mode = test_mode
         self.num_samples = len(self.dataset)  # number of original samples
-        self.merged_results: Union[List[Tuple[np.ndarray, list]], List[np.ndarray]] = []
 
     def __len__(self) -> int:
         """Get the length of the dataset."""
@@ -383,18 +387,16 @@ def __getitem__(self, idx: int) -> Dict:
         """
         return self.pipeline(self.tile_dataset[idx])
 
-    def evaluate(self, results, **kwargs) -> Dict[str, float]:
-        """Evaluation on Tile dataset.
+    def get_ann_info(self, idx):
+        """Get annotation information of a tile.
 
         Args:
-            results (list[list | tuple]): Testing results of the dataset.
-            **kwargs: Addition keyword arguments.
+            idx (int): Index of data.
 
         Returns:
-            dict[str, float]: evaluation metric.
+            dict: Annotation information of a tile.
         """
-        self.merged_results = self.tile_dataset.merge(results)
-        return self.dataset.evaluate(self.merged_results, **kwargs)
+        return self.tile_dataset.get_ann_info(idx)
 
     def merge(self, results) -> Union[List[Tuple[np.ndarray, list]], List[np.ndarray]]:
         """Merge tile-level results to image-level results.
@@ -405,10 +407,4 @@ def merge(self, results) -> Union[List[Tuple[np.ndarray, list]], List[np.ndarray
         Returns:
             merged_results (list[list | tuple]): Merged results of the dataset.
         """
-        self.merged_results = self.tile_dataset.merge(results)
-        return self.merged_results
-
-    def __del__(self):
-        """Delete the temporary directory when the object is deleted."""
-        if getattr(self, "tmp_dir", False):
-            self.tmp_dir.cleanup()
+        return self.tile_dataset.merge(results)