feat(LAB-3459): import geospatial image through SDK (#1863)

Co-authored-by: paulruelle <paul.ruelle@kili-technology.com>

Author: RuellePaul

src/kili/core/constants.py

@@ -12,6 +12,7 @@
         "application/vnd.nitf, image/jp2,image/jpeg,image/png,image/bmp,image/gif,image/webp,"
         "image/x-icon,image/tiff,image/vnd.microsoft.icon,image/svg+xml,image/avif,image/apng"
     ),
+    "Geospatial": ("application/vnd.nitf, image/jp2, image/tiff"),
     "Pdf": "application/pdf",
     "Text": "text/plain",
     "TimeSeries": "text/csv",
@@ -21,6 +22,7 @@
 mime_extensions_for_IV2 = {
     "AUDIO": mime_extensions["Audio"],
     "IMAGE": mime_extensions["Image"],
+    "GEOSPATIAL": mime_extensions["Geospatial"],
     "NA": "",
     "PDF": mime_extensions["Pdf"],
     "TEXT": mime_extensions["Text"],

src/kili/domain/project.py

@@ -11,7 +11,7 @@
 
 ProjectId = NewType("ProjectId", str)
 InputType = Literal[
-    "IMAGE", "PDF", "TEXT", "VIDEO", "LLM_RLHF", "LLM_INSTR_FOLLOWING", "LLM_STATIC"
+    "IMAGE", "GEOSPATIAL", "PDF", "TEXT", "VIDEO", "LLM_RLHF", "LLM_INSTR_FOLLOWING", "LLM_STATIC"
 ]
 
 

src/kili/services/asset_import/__init__.py

@@ -52,11 +52,13 @@ def import_assets(  # pylint: disable=too-many-arguments
 
     if input_type not in importer_by_type:
         raise NotImplementedError(f"There is no importer for the input type: {input_type}")
-    if input_type != "IMAGE" and any(asset.get("multi_layer_content") for asset in assets):
+    if input_type not in ["IMAGE", "GEOSPATIAL"] and any(
+        asset.get("multi_layer_content") for asset in assets
+    ):
         raise ImportValidationError(
             f"Import of multi-layer assets is not supported for input type: {input_type}"
         )
     asset_importer = importer_by_type[input_type](*importer_params)
     casted_assets = cast(List[AssetLike], assets)
     asset_importer.check_asset_contents(casted_assets)
-    return asset_importer.import_assets(assets=casted_assets)
+    return asset_importer.import_assets(assets=casted_assets, input_type=input_type)

src/kili/services/asset_import/base.py

@@ -53,7 +53,6 @@
 
 FILTER_EXISTING_BATCH_SIZE = 1000
 
-
 if TYPE_CHECKING:
     from kili.client import Kili
 
@@ -63,6 +62,7 @@ class BatchParams(NamedTuple):
 
     is_asynchronous: bool
     is_hosted: bool
+    input_type: Optional[InputType] = None
 
 
 class ProcessingParams(NamedTuple):
@@ -103,7 +103,9 @@ def __init__(
         self.logger = logging.getLogger(__name__)
         self.logger.setLevel(logging.INFO)
 
-    def import_batch(self, assets: ListOrTuple[AssetLike], verify: bool) -> List[str]:
+    def import_batch(  # pylint: disable=unused-argument
+        self, assets: ListOrTuple[AssetLike], verify: bool, input_type: Optional[InputType] = None
+    ) -> List[str]:
         """Base actions to import a batch of asset.
 
         Returns:
@@ -236,7 +238,7 @@ def are_native_videos(assets) -> bool:
 
     def _async_import_to_kili(self, assets: List[KiliResolverAsset]):
         """Import assets with asynchronous resolver."""
-        if self.input_type == "IMAGE":
+        if self.input_type in ["IMAGE", "GEOSPATIAL"]:
             upload_type = "GEO_SATELLITE"
         elif self.input_type in ("VIDEO", "VIDEO_LEGACY"):
             upload_type = "NATIVE_VIDEO" if self.are_native_videos(assets) else "FRAME_VIDEO"
@@ -297,7 +299,9 @@ def import_to_kili(self, assets: List[KiliResolverAsset]):
 class ContentBatchImporter(BaseBatchImporter):
     """Class defining the methods to import a batch of assets with content."""
 
-    def import_batch(self, assets: List[AssetLike], verify: bool):
+    def import_batch(
+        self, assets: List[AssetLike], verify: bool, input_type: Optional[InputType] = None
+    ):
         """Method to import a batch of asset with content."""
         assets = self.add_ids(assets)
         if not self.is_hosted:
@@ -312,7 +316,7 @@ def import_batch(self, assets: List[AssetLike], verify: bool):
                 if not asset.get("content") and not asset.get("multi_layer_content")
             ]
             if len(assets_with_content) > 0:
-                assets += self.upload_local_content_to_bucket(assets_with_content)
+                assets += self.upload_local_content_to_bucket(assets_with_content, input_type)
         return super().import_batch(assets, verify)
 
     def get_content_type_and_data_from_content(
@@ -332,7 +336,9 @@ def get_type_and_data_from_content_array(
         """Returns the data of the content (path) and its content type for each element in the array."""
         return list(map(self.get_content_type_and_data_from_content, content_array))
 
-    def upload_local_content_to_bucket(self, assets: List[AssetLike]):
+    def upload_local_content_to_bucket(
+        self, assets: List[AssetLike], input_type: Optional[InputType] = None
+    ):
         """Upload local content to a bucket."""
         project_bucket_path = self.generate_project_bucket_path()
         # tuple containing (bucket_path, file_path, asset_index, content_index)
@@ -343,12 +349,14 @@ def upload_local_content_to_bucket(self, assets: List[AssetLike]):
             if multi_layer_content:
                 for j, item in enumerate(multi_layer_content):
                     bucket_path = BaseBatchImporter.build_url_from_parts(
-                        project_bucket_path, asset_id, "content", str(j)
+                        project_bucket_path, asset_id, "content", f"{j!s}.tif"
                     )
                     to_upload.append((bucket_path, item.get("path"), i, j))
             else:
                 bucket_path = BaseBatchImporter.build_url_from_parts(
-                    project_bucket_path, asset_id, "content"
+                    project_bucket_path,
+                    asset_id,
+                    "content.tif" if input_type == "GEOSPATIAL" else "content",
                 )
                 to_upload.append((bucket_path, asset.get("content"), i, None))
         signed_urls = bucket.request_signed_urls(
@@ -418,7 +426,9 @@ def upload_json_content_to_bucket(self, assets: List[AssetLike]):
             )
         return [AssetLike(**{**asset, "json_content": url}) for asset, url in zip(assets, url_gen)]  # type: ignore
 
-    def import_batch(self, assets: List[AssetLike], verify: bool):
+    def import_batch(
+        self, assets: List[AssetLike], verify: bool, input_type: Optional[InputType] = None
+    ):
         """Method to import a batch of asset with json content."""
         assets = self.add_ids(assets)
         assets = self.loop_on_batch(self.stringify_json_content)(assets)
@@ -443,7 +453,7 @@ def __init__(
         self.pbar = tqdm(disable=logger_params.disable_tqdm)
 
     @abc.abstractmethod
-    def import_assets(self, assets: List[AssetLike]) -> List[str]:
+    def import_assets(self, assets: List[AssetLike], input_type: InputType) -> List[str]:
         """Import assets into Kili.
 
         Returns:
@@ -611,6 +621,7 @@ def import_assets_by_batch(
         assets: List[AssetLike],
         batch_importer: BaseBatchImporter,
         batch_size=IMPORT_BATCH_SIZE,
+        input_type: Optional[InputType] = None,
     ):
         """Split assets by batch and import them with a given batch importer."""
         batch_generator = batcher(assets, batch_size)
@@ -622,5 +633,5 @@ def import_assets_by_batch(
         for i, batch_assets in enumerate(batch_generator):
             # check last batch only
             verify = i == (nb_batch - 1) and self.verify
-            created_asset_ids += batch_importer.import_batch(batch_assets, verify)
+            created_asset_ids += batch_importer.import_batch(batch_assets, verify, input_type)
         return created_asset_ids

src/kili/services/asset_import/image.py

@@ -5,16 +5,17 @@
 
 from kili.core.constants import mime_extensions_that_need_post_processing
 from kili.core.helpers import get_mime_type
+from kili.domain.project import InputType
 
 from .base import BaseAbstractAssetImporter, BatchParams, ContentBatchImporter
 from .constants import LARGE_IMAGE_THRESHOLD_SIZE
 from .types import AssetLike
 
 
 class ImageDataImporter(BaseAbstractAssetImporter):
-    """Class for importing assets into an IMAGE project."""
+    """Class for importing assets into an IMAGE or GEOSPATIAL project."""
 
-    def import_assets(self, assets: List[AssetLike]):
+    def import_assets(self, assets: List[AssetLike], input_type: InputType):
         """Import IMAGE assets into Kili."""
         self._check_upload_is_allowed(assets)
         is_hosted = self.is_hosted_content(assets)
@@ -24,17 +25,25 @@ def import_assets(self, assets: List[AssetLike]):
         sync_assets, async_assets = self.split_asset_by_upload_type(assets, is_hosted)
         created_asset_ids: List[str] = []
         if len(sync_assets) > 0:
-            sync_batch_params = BatchParams(is_hosted=is_hosted, is_asynchronous=False)
+            sync_batch_params = BatchParams(
+                is_hosted=is_hosted, is_asynchronous=False, input_type=input_type
+            )
             batch_importer = ContentBatchImporter(
                 self.kili, self.project_params, sync_batch_params, self.pbar
             )
-            created_asset_ids += self.import_assets_by_batch(sync_assets, batch_importer)
+            created_asset_ids += self.import_assets_by_batch(
+                sync_assets, batch_importer, input_type=input_type
+            )
         if len(async_assets) > 0:
-            async_batch_params = BatchParams(is_hosted=is_hosted, is_asynchronous=True)
+            async_batch_params = BatchParams(
+                is_hosted=is_hosted, is_asynchronous=True, input_type=input_type
+            )
             batch_importer = ContentBatchImporter(
                 self.kili, self.project_params, async_batch_params, self.pbar
             )
-            created_asset_ids += self.import_assets_by_batch(async_assets, batch_importer)
+            created_asset_ids += self.import_assets_by_batch(
+                async_assets, batch_importer, input_type=input_type
+            )
         return created_asset_ids
 
     @staticmethod

src/kili/services/asset_import/llm.py

@@ -6,6 +6,7 @@
 from typing import List, Optional, Tuple
 
 from kili.core.helpers import is_url
+from kili.domain.project import InputType
 
 from .base import (
     BaseAbstractAssetImporter,
@@ -71,7 +72,7 @@ def transform_asset_content(asset_content, json_metadata):
 
         return transformed_asset_content, changed_json_metadata
 
-    def import_assets(self, assets: List[AssetLike]):
+    def import_assets(self, assets: List[AssetLike], input_type: InputType):
         """Import LLM assets into Kili."""
         self._check_upload_is_allowed(assets)
         data_type = self.get_data_type(assets)

src/kili/services/asset_import/pdf.py

@@ -2,14 +2,16 @@
 
 from typing import List
 
+from kili.domain.project import InputType
+
 from .base import BaseAbstractAssetImporter, BatchParams, ContentBatchImporter
 from .types import AssetLike
 
 
 class PdfDataImporter(BaseAbstractAssetImporter):
     """Class for importing data into a PDF project."""
 
-    def import_assets(self, assets: List[AssetLike]):
+    def import_assets(self, assets: List[AssetLike], input_type: InputType):
         """Import PDF assets into Kili."""
         self._check_upload_is_allowed(assets)
         is_hosted = self.is_hosted_content(assets)

src/kili/services/asset_import/text.py

@@ -5,6 +5,7 @@
 from typing import List, Optional, Tuple
 
 from kili.core.helpers import is_url
+from kili.domain.project import InputType
 
 from .base import (
     BaseAbstractAssetImporter,
@@ -62,7 +63,7 @@ def get_data_type(assets: List[AssetLike]) -> TextDataType:
             return TextDataType.HOSTED_FILE
         return TextDataType.RAW_TEXT
 
-    def import_assets(self, assets: List[AssetLike]):
+    def import_assets(self, assets: List[AssetLike], input_type: InputType):
         """Import TEXT assets into Kili."""
         self._check_upload_is_allowed(assets)
         data_type = self.get_data_type(assets)

src/kili/services/asset_import/video.py

@@ -5,9 +5,10 @@
 from enum import Enum
 from itertools import repeat
 from json import JSONDecodeError
-from typing import List
+from typing import List, Optional
 
 from kili.core.helpers import get_mime_type, is_url
+from kili.domain.project import InputType
 from kili.services.asset_import.base import (
     BaseAbstractAssetImporter,
     BaseBatchImporter,
@@ -74,7 +75,9 @@ def add_video_processing_parameters(self, asset):
         json_metadata = {**json_metadata, "processingParameters": processing_parameters}
         return AssetLike(**{**asset, "json_metadata": json_metadata})  # type: ignore
 
-    def import_batch(self, assets: List[AssetLike], verify: bool):
+    def import_batch(
+        self, assets: List[AssetLike], verify: bool, input_type: Optional[InputType] = None
+    ):
         """Import a batch of video assets from content into Kili."""
         assets = self.loop_on_batch(self.add_video_processing_parameters)(assets)
         return super().import_batch(assets, verify)
@@ -90,7 +93,9 @@ def add_video_processing_parameters(self, asset):
         json_metadata = {**json_metadata, "processingParameters": processing_parameters}
         return AssetLike(**{**asset, "json_metadata": json_metadata})  # type: ignore
 
-    def import_batch(self, assets: List[AssetLike], verify: bool):
+    def import_batch(
+        self, assets: List[AssetLike], verify: bool, input_type: Optional[InputType] = None
+    ):
         """Import a batch of video assets from frames."""
         assets = self.add_ids(assets)
         if not self.is_hosted:
@@ -234,7 +239,7 @@ def videos_have_complete_processing_parameters(self, assets) -> bool:
                 return False
         return True
 
-    def import_assets(self, assets: List[AssetLike]):
+    def import_assets(self, assets: List[AssetLike], input_type: InputType):
         """Import video assets into Kili."""
         self._check_upload_is_allowed(assets)
         data_type = self.get_data_type(assets)

tests/integration/adapters/kili_api_gateway/test_paginated_graphql_query.py

@@ -156,8 +156,8 @@ def test_given_a_query_and_a_number_of_elements_to_query_i_have_a_progress_bar(
 
     # then
     captured = capsys.readouterr()
-    assert "0%|          | 0/250" in captured.err
-    assert "100%|██████████| 250/250" in captured.err
+    assert "0%" in captured.err and "0/250" in captured.err
+    assert "100%" in captured.err and "250/250" in captured.err
 
 
 def test_given_a_query_without_a_count_query_I_do_not_have_a_progress_bar(graphql_client, capsys):