Merge pull request #1908 from kili-technology/feature/lab-3753-aa-sdk-user-i-can-install-kili-with-optional-dependencies

chore(lab-3743): refine dependencies and make some optional

Author: baptiste-olivier

pyproject.toml

@@ -32,25 +32,19 @@ dependencies = [
   "pandas >= 1.0.0, < 3.0.0",
   "click >= 8.0.0, < 8.1.8",
   "requests >= 2.0.0, < 3.0.0",
-  "tabulate >= 0.9.0, < 0.10.0",
   "tenacity >= 8.0.0, < 9.0.0",
   "tqdm >= 4.0.0, < 5.0.0",
   "typeguard >= 4, < 5",
   "typing-extensions >= 4.1.0, < 5.0.0",
   "pyparsing >= 3.0.0, < 4.0.0",
   "websocket-client >= 1.0.0, < 2.0.0",
-  "pyyaml >= 6.0, < 7.0",
-  "Pillow >=9.0.0, <11.0.0",
   "cuid >= 0.4, < 0.5",
   "urllib3 >= 1.26, < 3",
-  "ffmpeg-python >= 0.2.0, < 0.3.0",
   "gql[requests,websockets] >= 3.5.0b5, < 4.0.0",
   "filelock >= 3.0.0, < 4.0.0",
   "pip-system-certs >= 4.0.0, < 5.0.0; platform_system=='Windows'",
   "pyrate-limiter >= 3, < 4",
-  "shapely >= 1.8, < 3",
-  "pyproj == 3.6.1",
-  "kili-formats == 0.1.8"
+  "kili-formats == 0.2.0"
 ]
 urls = { homepage = "https://github.com/kili-technology/kili-python-sdk" }
 
@@ -89,12 +83,57 @@ dev = [
   # dead code detection
   "vulture==2.11",
   "dead==1.5.2",
-  # other optional dependencies
+  # optional dependencies
+  "kili-formats[all] == 0.2.0",
   "opencv-python >= 4.0.0, < 5.0.0",
   "azure-storage-blob >= 12.0.0, < 13.0.0",
+  # optional dependencies gis
+  "pyproj >= 2.6.1, < 3; python_version < '3.9'",
+  "pyproj == 3.6.1; python_version >= '3.9'",
+  "shapely >= 1.8, < 3",
+  # optional dependencies - cli
+  "tabulate >= 0.9.0, < 0.10.0",
+  # optional dependencies - yolo
+  "pyyaml >= 6.0, < 7.0",
+  # optional dependencies - image
+  "Pillow >=9.0.0, <11.0.0"
+]
+all = [
+  # aggregate all optional deps without dev
+  "azure-storage-blob >= 12.0.0, < 13.0.0",
+  "kili-formats[all] == 0.2.0",
+  "opencv-python >= 4.0.0, < 5.0.0",
+  "Pillow >=9.0.0, <11.0.0",
+  "pyproj == 3.6.1; python_version >= '3.9'",
+  "pyproj >= 2.6.1, < 3; python_version < '3.9'",
+  "pyyaml >= 6.0, < 7.0",
+  "shapely >= 1.8, < 3",
+  "tabulate >= 0.9.0, < 0.10.0"
 ]
-image-utils = ["opencv-python >= 4.0.0, < 5.0.0"]
 azure = ["azure-storage-blob >= 12.0.0, < 13.0.0"]
+cli = [
+  "tabulate >= 0.9.0, < 0.10.0"
+]
+coco = [
+  "kili-formats[coco] == 0.2.0"
+]
+gis = [
+  "pyproj >= 2.6.1, < 3; python_version < '3.9'",
+  "pyproj == 3.6.1; python_version >= '3.9'",
+  "shapely >= 1.8, < 3"
+]
+image = [
+  "Pillow >=9.0.0, <11.0.0",
+  "kili-formats[image] == 0.2.0"
+]
+image-utils = ["opencv-python >= 4.0.0, < 5.0.0"]
+
+video = [
+  "kili-formats[video] == 0.2.0"
+]
+yolo = [
+  "pyyaml >= 6.0, < 7.0"
+]
 
 [tool.pyright]
 exclude = ["**/__pycache__", ".github/scripts/upload_test_stats_datadog.py"]

src/kili/entrypoints/cli/project/create.py

@@ -4,7 +4,6 @@
 from typing import Optional
 
 import click
-from tabulate import tabulate
 
 from kili.domain.project import InputTypeEnum, ProjectId
 from kili.entrypoints.cli.common_args import Options
@@ -63,6 +62,10 @@ def create_project(
     To build a Kili project interface, please visit: \n
     https://docs.kili-technology.com/docs/customizing-the-interface-through-json-settings
     """
+    try:
+        from tabulate import tabulate  # pylint: disable=import-outside-toplevel
+    except ImportError as e:
+        raise ImportError("Install with `pip install kili[cli]` to use this feature.") from e
     kili = get_kili_client(api_key=api_key, api_endpoint=endpoint)
 
     if ((interface is not None) + (project_id_src is not None)) > 1:

src/kili/entrypoints/cli/project/describe.py

@@ -3,7 +3,6 @@
 from typing import Optional
 
 import click
-from tabulate import tabulate
 
 from kili.domain.project import ProjectId
 from kili.entrypoints.cli.common_args import Arguments, Options
@@ -27,6 +26,10 @@ def describe_project(api_key: Optional[str], endpoint: Optional[str], project_id
         kili project describe --project-id <project_id>
         ```
     """
+    try:
+        from tabulate import tabulate  # pylint: disable=import-outside-toplevel
+    except ImportError as e:
+        raise ImportError("Install with `pip install kili[cli]` to use this feature.") from e
     kili = get_kili_client(api_key=api_key, api_endpoint=endpoint)
     query_fields = [
         "title",

src/kili/entrypoints/cli/project/list_.py

@@ -5,7 +5,6 @@
 import click
 import numpy as np
 import pandas as pd
-from tabulate import tabulate
 
 from kili.adapters.kili_api_gateway.helpers.queries import QueryOptions
 from kili.domain.project import ProjectFilters
@@ -33,6 +32,10 @@ def list_projects(api_key: Optional[str], endpoint: Optional[str], tablefmt: str
         kili project list --max 10 --stdout-format pretty
         ```
     """
+    try:
+        from tabulate import tabulate  # pylint: disable=import-outside-toplevel
+    except ImportError as e:
+        raise ImportError("Install with `pip install kili[cli]` to use this feature.") from e
     kili = get_kili_client(api_key=api_key, api_endpoint=endpoint)
     projects = list(
         kili.kili_api_gateway.list_projects(

src/kili/entrypoints/cli/project/member/list_.py

@@ -4,7 +4,6 @@
 
 import click
 import pandas as pd
-from tabulate import tabulate
 
 from kili.adapters.kili_api_gateway.helpers.queries import QueryOptions
 from kili.core.graphql.operations.project_user.queries import (
@@ -31,6 +30,10 @@ def list_members(api_key: Optional[str], endpoint: Optional[str], project_id: st
         kili project member list <project_id> --stdout-format pretty
         ```
     """
+    try:
+        from tabulate import tabulate  # pylint: disable=import-outside-toplevel
+    except ImportError as e:
+        raise ImportError("Install with `pip install kili[cli]` to use this feature.") from e
     kili = get_kili_client(api_key=api_key, api_endpoint=endpoint)
     members_list = list(
         ProjectUserQuery(kili.graphql_client, kili.http_client)(

src/kili/services/asset_import/image.py

@@ -3,8 +3,6 @@
 import os
 from typing import List
 
-from PIL import Image, UnidentifiedImageError
-
 from kili.core.constants import mime_extensions_that_need_post_processing
 from kili.core.helpers import get_mime_type
 from kili.domain.project import InputType
@@ -13,8 +11,6 @@
 from .constants import LARGE_IMAGE_THRESHOLD_SIZE, MAX_WIDTH_OR_HEIGHT_NON_TILED
 from .types import AssetLike
 
-Image.MAX_IMAGE_PIXELS = None
-
 
 class ImageDataImporter(BaseAbstractAssetImporter):
     """Class for importing assets into an IMAGE or GEOSPATIAL project."""
@@ -53,6 +49,13 @@ def import_assets(self, assets: List[AssetLike], input_type: InputType):
     @staticmethod
     def get_is_large_image(image_path: str) -> bool:
         """Define if an image is too large and so on has to be tiled."""
+        try:
+            from PIL import Image  # pylint: disable=import-outside-toplevel
+
+            Image.MAX_IMAGE_PIXELS = None
+        except ImportError as e:
+            raise ImportError("Install with `pip install kili[image]` to use this feature.") from e
+
         if os.path.getsize(image_path) >= LARGE_IMAGE_THRESHOLD_SIZE:
             return True
 
@@ -63,6 +66,10 @@ def get_is_large_image(image_path: str) -> bool:
     @staticmethod
     def split_asset_by_upload_type(assets: List[AssetLike], is_hosted: bool):
         """Split assets into two groups, assets to to imported synchronously or asynchronously."""
+        try:
+            from PIL import UnidentifiedImageError  # pylint: disable=import-outside-toplevel
+        except ImportError as e:
+            raise ImportError("Install with `pip install kili[image]` to use this feature.") from e
         if is_hosted:
             return assets, []
         sync_assets, async_assets = [], []

src/kili/services/export/format/coco/__init__.py

@@ -51,10 +51,12 @@ def images_folder(self) -> Path:
     def process_and_save(self, assets: List[Dict], output_filename: Path):
         """Extract formatted annotations from labels."""
         clean_assets = self.preprocess_assets(assets)
-
-        self._save_assets_export(
-            clean_assets, self.export_root_folder, annotation_modifier=self.annotation_modifier
-        )
+        try:
+            self._save_assets_export(
+                clean_assets, self.export_root_folder, annotation_modifier=self.annotation_modifier
+            )
+        except ImportError as e:
+            raise ImportError("Install with `pip install kili[coco]` to use this feature.") from e
         self.create_readme_kili_file(self.export_root_folder)
         self.make_archive(self.export_root_folder, output_filename)
 

src/kili/services/export/format/kili/__init__.py

@@ -88,12 +88,17 @@ def _cut_video_assets(self, assets: List[Dict]) -> List[Dict]:
                 if nbr_frames == 0:
                     continue
                 leading_zeros = len(str(nbr_frames))
-                asset["jsonContent"] = cut_video(
-                    video_path=asset["content"],
-                    asset=asset,
-                    leading_zeros=leading_zeros,
-                    output_dir=self.images_folder,
-                )
+                try:
+                    asset["jsonContent"] = cut_video(
+                        video_path=asset["content"],
+                        asset=asset,
+                        leading_zeros=leading_zeros,
+                        output_dir=self.images_folder,
+                    )
+                except ImportError as e:
+                    raise ImportError(
+                        "Install with `pip install kili[video]` to use this feature."
+                    ) from e
         return assets
 
     def process_and_save(self, assets: List[Dict], output_filename: Path) -> None:

src/kili/services/export/format/voc/__init__.py

@@ -93,9 +93,14 @@ def _process_asset(
 
         # video with shouldUseNativeVideo set to True (no frames available)
         elif Path(asset["content"]).is_file():
-            width, height = get_video_dimensions(asset)
-            cut_video(asset["content"], asset, leading_zeros, Path(asset["content"]).parent)
-            frame_ext = ".jpg"
+            try:
+                width, height = get_video_dimensions(asset)
+                cut_video(asset["content"], asset, leading_zeros, Path(asset["content"]).parent)
+                frame_ext = ".jpg"
+            except ImportError as e:
+                raise ImportError(
+                    "Install with `pip install kili[video]` to use this feature."
+                ) from e
 
         else:
             raise FileNotFoundError(f"Could not find frames or video for asset {asset}")

src/kili/services/export/format/yolo/__init__.py

@@ -281,12 +281,15 @@ def _process_asset(
 
     # If the asset is a video, we need to cut it into frames
     if project_input_type == "VIDEO" and asset["jsonContent"] == "" and with_assets:
-        asset["jsonContent"] = cut_video(
-            video_path=asset["content"],
-            asset=asset,
-            leading_zeros=leading_zeros,
-            output_dir=images_folder,
-        )
+        try:
+            asset["jsonContent"] = cut_video(
+                video_path=asset["content"],
+                asset=asset,
+                leading_zeros=leading_zeros,
+                output_dir=images_folder,
+            )
+        except ImportError as e:
+            raise ImportError("Install with `pip install kili[video]` to use this feature.") from e
 
     content_frames = []
     if isinstance(asset["jsonContent"], list):