Fix auto input size mismatch in eval & export (#2530)

* Fix auto input size mismatch in eval & export

* Re-enable E2E tests for Issue#2518

* Add input size check in export testing

* Format float numbers in log

* Fix NNCF export shape mismatch

* Fix saliency map issue

* Disable auto input size if tiling enabled

---------

Signed-off-by: Songki Choi <[email protected]>

Author: Songki Choi

src/otx/algorithms/classification/adapters/mmcls/configurer.py

@@ -1,9 +1,9 @@
 """Base configurer for mmdet config."""
+
 # Copyright (C) 2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-#
 
-from typing import Optional
+from typing import Optional, Tuple
 
 import torch
 from mmcv import build_from_cfg
@@ -22,7 +22,6 @@
     recursively_update_cfg,
     update_or_add_custom_hook,
 )
-from otx.algorithms.common.configs.configuration_enums import InputSizePreset
 from otx.algorithms.common.utils.logger import get_logger
 
 logger = get_logger()
@@ -162,16 +161,19 @@ def configure_topk(cfg):
 
     @staticmethod
     def configure_input_size(
-        cfg, input_size_config: InputSizePreset = InputSizePreset.DEFAULT, model_ckpt_path: Optional[str] = None
+        cfg, input_size=Optional[Tuple[int, int]], model_ckpt_path: Optional[str] = None, training=True
     ):
         """Change input size if necessary."""
-        manager = InputSizeManager(cfg)
-        input_size = manager.get_configured_input_size(input_size_config, model_ckpt_path)
         if input_size is None:  # InputSizePreset.DEFAULT
             return
 
+        manager = InputSizeManager(cfg)
+
         if input_size == (0, 0):  # InputSizePreset.AUTO
-            input_size = BaseConfigurer.adapt_input_size_to_dataset(cfg, manager)
+            if training:
+                input_size = BaseConfigurer.adapt_input_size_to_dataset(cfg, manager, use_annotations=False)
+            else:
+                input_size = manager.get_trained_input_size(model_ckpt_path)
             if input_size is None:
                 return
 

src/otx/algorithms/classification/adapters/mmcls/nncf/task.py

@@ -1,18 +1,7 @@
 """NNCF Task for OTX Classification."""
 
-# Copyright (C) 2022 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the 'License');
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an 'AS IS' BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
+# Copyright (C) 2022-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
 from functools import partial
 from typing import List, Optional
@@ -121,3 +110,6 @@ def _generate_training_metrics_group(self, learning_curves):
             output.append(LineMetricsGroup(metrics=[metric_curve], visualization_info=visualization_info))
 
         return output, best_acc
+
+    def _save_model_post_hook(self, modelinfo):
+        modelinfo["input_size"] = self._input_size

src/otx/algorithms/classification/adapters/mmcls/task.py

@@ -1,18 +1,7 @@
 """Task of OTX Classification using mmclassification training backend."""
 
 # Copyright (C) 2023 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
 
 import glob
 import os
@@ -194,11 +183,12 @@ def configure(
             ir_options,
             data_classes,
             model_classes,
-            self._hyperparams.learning_parameters.input_size,
+            self._input_size,
             options_for_patch_datasets=options_for_patch_datasets,
             options_for_patch_evaluation=options_for_patch_evaluation,
         )
         self._config = cfg
+        self._input_size = cfg.model.pop("input_size", None)
         return cfg
 
     def build_model(

src/otx/algorithms/classification/task.py

@@ -1,18 +1,7 @@
 """Task of OTX Classification."""
 
 # Copyright (C) 2023 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
+# SPDX-License-Identifier: Apache-2.0
 
 import io
 import json
@@ -34,6 +23,7 @@
     get_multihead_class_info as get_hierarchical_info,
 )
 from otx.algorithms.common.configs import TrainType
+from otx.algorithms.common.configs.configuration_enums import InputSizePreset
 from otx.algorithms.common.tasks.base_task import TRAIN_TYPE_DIR_PATH, OTXTask
 from otx.algorithms.common.utils import embed_ir_model_data
 from otx.algorithms.common.utils.callback import TrainingProgressCallback
@@ -130,6 +120,12 @@ def __init__(self, task_environment: TaskEnvironment, output_path: Optional[str]
         if self._task_environment.model is not None:
             self._load_model()
 
+        if hasattr(self._hyperparams.learning_parameters, "input_size"):
+            input_size_cfg = InputSizePreset(self._hyperparams.learning_parameters.input_size.value)
+        else:
+            input_size_cfg = InputSizePreset.DEFAULT
+        self._input_size = input_size_cfg.tuple
+
     def _is_multi_label(self, label_groups: List[LabelGroup], all_labels: List[LabelEntity]):
         """Check whether the current training mode is multi-label or not."""
         # NOTE: In the current Geti, multi-label should have `___` symbol for all group names.
@@ -479,6 +475,7 @@ def save_model(self, output_model: ModelEntity):
             "model": model_ckpt,
             "config": hyperparams_str,
             "labels": labels,
+            "input_size": self._input_size,
             "VERSION": 1,
         }
 

src/otx/algorithms/common/adapters/mmcv/configurer.py

@@ -26,7 +26,6 @@
     recursively_update_cfg,
     update_or_add_custom_hook,
 )
-from otx.algorithms.common.configs.configuration_enums import InputSizePreset
 from otx.algorithms.common.tasks.base_task import OnHookInitialized
 from otx.algorithms.common.utils import UncopiableDefaultDict, append_dist_rank_suffix
 from otx.algorithms.common.utils.data import compute_robust_dataset_statistics
@@ -74,7 +73,7 @@ def configure(
         ir_options: Optional[Config] = None,
         data_classes: Optional[List[str]] = None,
         model_classes: Optional[List[str]] = None,
-        input_size: InputSizePreset = InputSizePreset.DEFAULT,
+        input_size: Optional[Tuple[int, int]] = None,
         **kwargs: Dict[Any, Any],
     ) -> Config:
         """Create MMCV-consumable config from given inputs."""
@@ -228,7 +227,7 @@ def configure_data_pipeline(self, cfg, input_size, model_ckpt_path, **kwargs):
         """Configuration data pipeline settings."""
 
         patch_color_conversion(cfg)
-        self.configure_input_size(cfg, input_size, model_ckpt_path)
+        self.configure_input_size(cfg, input_size, model_ckpt_path, self.training)
 
     def configure_recipe(self, cfg, **kwargs):
         """Configuration training recipe settings."""
@@ -533,7 +532,15 @@ def adapt_input_size_to_dataset(
         stat = compute_robust_dataset_statistics(dataset, use_annotations)
         if not stat:
             return None
-        logger.info(f"Dataset stat: {json.dumps(stat, indent=4)}")
+
+        def format_float(obj):
+            if isinstance(obj, float):
+                return f"{obj:.2f}"
+            if isinstance(obj, dict):
+                return {k: format_float(v) for k, v in obj.items()}
+            return obj
+
+        logger.info(f"Dataset stat: {json.dumps(format_float(stat), indent=4)}")
 
         # Fit to typical large image size (conservative)
         # -> "avg" size might be preferrable for efficiency

src/otx/algorithms/common/adapters/mmcv/utils/config_utils.py

@@ -682,12 +682,12 @@ def set_input_size(self, input_size: Union[int, List[int], Tuple[int, int]]):
                     self._set_pipeline_size_value(pipelines, resize_ratio)
 
         # Set model size
-        # - needed only for YOLOX
         model_cfg = self._config.get("model", {})
+        model_cfg["input_size"] = input_size
         if model_cfg.get("type", "") == "CustomYOLOX":
+            # - needed only for YOLOX
             if input_size[0] % 32 != 0 or input_size[1] % 32 != 0:
                 raise ValueError("YOLOX should have input size being multiple of 32.")
-            model_cfg["input_size"] = input_size
 
     @property
     def base_input_size(self) -> Union[Tuple[int, int], Dict[str, Tuple[int, int]]]:
@@ -862,38 +862,28 @@ def _set_size_value(pipeline: Dict, attr: str, scale: Tuple[Union[int, float], U
             pipeline[attr] = (round(pipeline[attr][0] * scale[0]), round(pipeline[attr][1] * scale[1]))
 
     @staticmethod
-    def get_configured_input_size(
-        input_size_config: InputSizePreset = InputSizePreset.DEFAULT, model_ckpt: Optional[str] = None
-    ) -> Optional[Tuple[int, int]]:
-        """Get configurable input size configuration. If it doesn't exist, return None.
+    def get_trained_input_size(model_ckpt: Optional[str] = None) -> Optional[Tuple[int, int]]:
+        """Get trained input size from checkpoint. If it doesn't exist, return None.
 
         Args:
-            input_size_config (InputSizePreset, optional): Input size setting. Defaults to InputSizePreset.DEFAULT.
             model_ckpt (Optional[str], optional): Model weight to load. Defaults to None.
 
         Returns:
             Optional[Tuple[int, int]]: Pair of width and height. If there is no input size configuration, return None.
         """
-        input_size = None
-        if input_size_config == InputSizePreset.DEFAULT:
-            if model_ckpt is None:
-                return None
-
-            model_info = torch.load(model_ckpt, map_location="cpu")
-            for key in ["config", "learning_parameters", "input_size", "value"]:
-                if key not in model_info:
-                    return None
-                model_info = model_info[key]
-            input_size = model_info
-
-            if input_size == InputSizePreset.DEFAULT.value:
-                return None
-            logger.info("Given model weight was trained with {} input size.".format(input_size))
+        if model_ckpt is None:
+            return None
 
-        else:
-            input_size = input_size_config.value
+        model_info = torch.load(model_ckpt, map_location="cpu")
+        if model_info is None:
+            return None
 
-        return InputSizePreset.parse(input_size)
+        input_size = model_info.get("input_size", None)
+        if not input_size:
+            return None
+
+        logger.info("Given model weight was trained with {} input size.".format(input_size))
+        return input_size
 
     @staticmethod
     def select_closest_size(input_size: Tuple[int, int], preset_sizes: List[Tuple[int, int]]):

src/otx/algorithms/common/utils/utils.py

@@ -1,18 +1,7 @@
 """Collections of Utils for common OTX algorithms."""
 
-# Copyright (C) 2022 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
+# Copyright (C) 2022-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
 import importlib
 import inspect
@@ -98,7 +87,7 @@ def get_arg_spec(  # noqa: C901  # pylint: disable=too-many-branches
     return tuple(args)
 
 
-def set_random_seed(seed, logger, deterministic=False):
+def set_random_seed(seed, logger=None, deterministic=False):
     """Set random seed.
 
     Args:
@@ -116,7 +105,8 @@ def set_random_seed(seed, logger, deterministic=False):
     torch.manual_seed(seed)
     torch.cuda.manual_seed_all(seed)
     os.environ["PYTHONHASHSEED"] = str(seed)
-    logger.info(f"Training seed was set to {seed} w/ deterministic={deterministic}.")
+    if logger:
+        logger.info(f"Training seed was set to {seed} w/ deterministic={deterministic}.")
     if deterministic:
         torch.backends.cudnn.deterministic = True
         torch.backends.cudnn.benchmark = False

src/otx/algorithms/detection/adapters/mmdet/configurer.py

@@ -1,9 +1,9 @@
 """Base configurer for mmdet config."""
+
 # Copyright (C) 2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
-#
 
-from typing import Optional
+from typing import Optional, Tuple
 
 from mmcv.utils import ConfigDict
 
@@ -13,7 +13,6 @@
 from otx.algorithms.common.adapters.mmcv.utils.config_utils import (
     InputSizeManager,
 )
-from otx.algorithms.common.configs.configuration_enums import InputSizePreset
 from otx.algorithms.common.utils.logger import get_logger
 from otx.algorithms.detection.adapters.mmdet.utils import (
     cluster_anchors,
@@ -154,9 +153,12 @@ def configure_bbox_head(self, cfg):
 
     @staticmethod
     def configure_input_size(
-        cfg, input_size_config: InputSizePreset = InputSizePreset.DEFAULT, model_ckpt_path: Optional[str] = None
+        cfg, input_size=Optional[Tuple[int, int]], model_ckpt_path: Optional[str] = None, training=True
     ):
         """Change input size if necessary."""
+        if input_size is None:  # InputSizePreset.DEFAULT
+            return
+
         # YOLOX tiny has a different input size in train and val data pipeline
         base_input_size = None
         model_cfg = cfg.get("model")
@@ -168,15 +170,13 @@ def configure_input_size(
                     "test": (416, 416),
                     "unlabeled": (992, 736),
                 }
-
         manager = InputSizeManager(cfg, base_input_size)
 
-        input_size = manager.get_configured_input_size(input_size_config, model_ckpt_path)
-        if input_size is None:  # InputSizePreset.DEFAULT
-            return
-
         if input_size == (0, 0):  # InputSizePreset.AUTO
-            input_size = BaseConfigurer.adapt_input_size_to_dataset(cfg, manager, use_annotations=True)
+            if training:
+                input_size = BaseConfigurer.adapt_input_size_to_dataset(cfg, manager, use_annotations=True)
+            else:
+                input_size = manager.get_trained_input_size(model_ckpt_path)
             if input_size is None:
                 return
 

src/otx/algorithms/detection/adapters/mmdet/nncf/task.py

@@ -1,19 +1,7 @@
 """NNCF Task of OTX Detection."""
 
-# Copyright (C) 2022 Intel Corporation
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions
-# and limitations under the License.
-
+# Copyright (C) 2022-2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
 
 from functools import partial
 from typing import Optional
@@ -124,3 +112,4 @@ def _save_model_post_hook(self, modelinfo):
             self._update_anchors(modelinfo["anchors"], self.config.model.bbox_head.anchor_generator)
 
         modelinfo["confidence_threshold"] = self.confidence_threshold
+        modelinfo["input_size"] = self._input_size