Add e2e test for visual prompting (#2360)

* (WIP) otx optimize

* pre-commit

* (WIP) set e2e

* Remove nncf config

* Add visual prompting requirement

* Add visual prompting in tox

* Add visual prompting in setup.py

* Fix typo

* Delete unused configuration.yaml

* Edit test_name

* Add to limit activation range

* Update from `vp` to `visprompt`

* Fix about no returning the first label

* pre-commit

* (WIP) otx optimize

* pre-commit

* (WIP) set e2e

* Remove nncf config

* Add visual prompting requirement

* Add visual prompting in tox

* Add visual prompting in setup.py

* Fix typo

* pre-commit

* Add actions

* Update tests/e2e/cli/visual_prompting/test_visual_prompting.py

Co-authored-by: Jaeguk Hyun <[email protected]>

* Skip PTQ e2e test

* Change task name

* Remove skipped tc

---------

Co-authored-by: Jaeguk Hyun <[email protected]>

Author: sungchul1

.github/workflows/daily.yml

@@ -24,6 +24,8 @@ jobs:
             test_dir: "tests/e2e/cli/instance_segmentation"
           - task: "seg"
             test_dir: "tests/e2e/cli/semantic_segmentation"
+          - task: "visprompt"
+            test_dir: "tests/e2e/cli/visual_prompting"
     name: E2E-Test-py310-${{ matrix.task }}
     uses: ./.github/workflows/run_tests_in_tox.yml
     with:

.github/workflows/pre_merge.yml

@@ -104,6 +104,8 @@ jobs:
             test_dir: "tests/integration/cli/action"
           - task: "ano"
             test_dir: "tests/integration/cli/anomaly"
+          - task: "visprompt"
+            test_dir: "tests/integration/cli/visual_prompting"
     name: Integration-Test-py310-${{ matrix.task }}
     # This is what will cancel the job concurrency
     concurrency:

requirements/visual_prompting.txt

@@ -0,0 +1,4 @@
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# Visual Prompting Requirements.
+scikit-image
+pytorch-lightning>=1.7.0,<1.10.0

setup.py

@@ -149,13 +149,18 @@ def _cython_modules():
             "base", "openvino", "segmentation",
         ]
     ),
+    "visual_prompting": get_requirements(requirement_files=[
+            "base", "openvino", "visual_prompting",
+        ]
+    ),
     "full": get_requirements(requirement_files=[
             "base",
             "openvino",
             "anomaly",
             "classification",
             "detection",
             "segmentation",
+            "visual_prompting",
             "action",
         ]
     ),

src/otx/algorithms/visual_prompting/configs/configuration.yaml

@@ -85,62 +85,6 @@ learning_parameters:
       visible_in_ui: true
       warning: null
       auto_hpo_state: NOT_POSSIBLE
-nncf_optimization:
-  description: Optimization by NNCF
-  enable_pruning:
-    affects_outcome_of: NONE
-    auto_hpo_state: not_possible
-    auto_hpo_value: null
-    default_value: false
-    description: Enable filter pruning algorithm
-    editable: true
-    header: Enable filter pruning algorithm
-    type: BOOLEAN
-    ui_rules:
-      action: DISABLE_EDITING
-      operator: AND
-      rules: []
-      type: UI_RULES
-    value: false
-    visible_in_ui: true
-    warning: null
-  enable_quantization:
-    affects_outcome_of: NONE
-    auto_hpo_state: not_possible
-    auto_hpo_value: null
-    default_value: true
-    description: Enable quantization algorithm
-    editable: true
-    header: Enable quantization algorithm
-    type: BOOLEAN
-    ui_rules:
-      action: DISABLE_EDITING
-      operator: AND
-      rules: []
-      type: UI_RULES
-    value: true
-    visible_in_ui: true
-    warning: null
-  header: Optimization by NNCF
-  pruning_supported:
-    affects_outcome_of: TRAINING
-    auto_hpo_state: not_possible
-    auto_hpo_value: null
-    default_value: false
-    description: Whether filter pruning is supported
-    editable: false
-    header: Whether filter pruning is supported
-    type: BOOLEAN
-    ui_rules:
-      action: DISABLE_EDITING
-      operator: AND
-      rules: []
-      type: UI_RULES
-    value: false
-    visible_in_ui: false
-    warning: null
-  type: PARAMETER_GROUP
-  visible_in_ui: true
 pot_parameters:
   description: POT Parameters
   header: POT Parameters

src/otx/algorithms/visual_prompting/configs/sam_vit_b/configuration.yaml

@@ -0,0 +1,23 @@
+"""PTQ config file."""
+from nncf.common.quantization.structs import QuantizationPreset
+from nncf.quantization.advanced_parameters import AdvancedQuantizationParameters
+from nncf.quantization.range_estimator import (
+    AggregatorType,
+    RangeEstimatorParameters,
+    StatisticsCollectorParameters,
+    StatisticsType,
+)
+
+advanced_parameters = AdvancedQuantizationParameters(
+    activations_range_estimator_params=RangeEstimatorParameters(
+        min=StatisticsCollectorParameters(
+            statistics_type=StatisticsType.QUANTILE, aggregator_type=AggregatorType.MIN, quantile_outlier_prob=1e-4
+        ),
+        max=StatisticsCollectorParameters(
+            statistics_type=StatisticsType.QUANTILE, aggregator_type=AggregatorType.MAX, quantile_outlier_prob=1e-4
+        ),
+    ),
+    # backend_params={"use_pot": True},
+)
+
+preset = QuantizationPreset.MIXED

src/otx/algorithms/visual_prompting/configs/sam_vit_b/ptq_optimization_config.py

@@ -28,13 +28,14 @@
 import nncf
 import numpy as np
 import openvino.runtime as ov
+from addict import Dict as ADDict
 from nncf.common.quantization.structs import QuantizationPreset
 from openvino.model_api.adapters import OpenvinoAdapter, create_core
 from openvino.model_api.models import Model
 
+from otx.algorithms.common.utils import get_default_async_reqs_num, read_py_config
 from otx.algorithms.common.utils.ir import check_if_quantized
 from otx.algorithms.common.utils.logger import get_logger
-from otx.algorithms.common.utils.utils import get_default_async_reqs_num
 from otx.algorithms.visual_prompting.adapters.openvino import model_wrappers
 from otx.algorithms.visual_prompting.adapters.pytorch_lightning.datasets.dataset import (
     OTXVisualPromptingDataset,
@@ -447,13 +448,17 @@ def optimize(
             if optimization_parameters is not None:
                 optimization_parameters.update_progress(10 * i + 35 * (i - 1), None)
 
-            stat_subset_size = self.hparams.pot_parameters.stat_subset_size
-            preset = QuantizationPreset(self.hparams.pot_parameters.preset.name.lower())
-
-            compressed_model = nncf.quantize(
-                ov_model, quantization_dataset, subset_size=min(stat_subset_size, len(data_loader)), preset=preset
+            optimization_config_path = os.path.join(self._base_dir, "ptq_optimization_config.py")
+            ptq_config = ADDict()
+            if os.path.exists(optimization_config_path):
+                ptq_config = read_py_config(optimization_config_path)
+            ptq_config.update(
+                subset_size=min(self.hparams.pot_parameters.stat_subset_size, len(data_loader)),
+                preset=QuantizationPreset(self.hparams.pot_parameters.preset.name.lower()),
             )
 
+            compressed_model = nncf.quantize(ov_model, quantization_dataset, **ptq_config)
+
             if optimization_parameters is not None:
                 optimization_parameters.update_progress(45 * i, None)
 

src/otx/algorithms/visual_prompting/tasks/openvino.py

@@ -60,7 +60,7 @@ def get_otx_dataset(self) -> DatasetEntity:
                             if self.use_mask:
                                 # use masks loaded in datumaro as-is
                                 if self.data_type == "common_semantic_segmentation":
-                                    if new_label := self.updated_label_id.get(ann.label, None):
+                                    if (new_label := self.updated_label_id.get(ann.label, None)) is not None:
                                         ann.label = new_label
                                     else:
                                         continue
@@ -70,7 +70,7 @@ def get_otx_dataset(self) -> DatasetEntity:
                                 # convert masks to polygons, they will be converted to masks again
                                 datumaro_polygons = MasksToPolygons.convert_mask(ann)
                                 for d_polygon in datumaro_polygons:
-                                    if new_label := self.updated_label_id.get(d_polygon.label, None):
+                                    if (new_label := self.updated_label_id.get(d_polygon.label, None)) is not None:
                                         d_polygon.label = new_label
                                     else:
                                         continue

src/otx/core/data/adapter/visual_prompting_dataset_adapter.py

@@ -0,0 +1,3 @@
+# Copyright (C) 2023 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+#