move gcs and s3 deps to extras

Author: hirosassa

README.md

@@ -54,6 +54,14 @@ Within the activated Python environment, use the following command to install go
 pip install gokart
 ```
 
+If you use S3 or GCS as a data store, install the corresponding extras:
+
+```bash
+pip install gokart[s3]   # S3 support
+pip install gokart[gcs]  # GCS support
+pip install gokart[all]  # both S3 and GCS
+```
+
 
 # Quickstart
 

docs/intro_to_gokart.rst

@@ -11,6 +11,13 @@ Within the activated Python environment, use the following command to install go
 
     pip install gokart
 
+If you use S3 or GCS as a data store, install the corresponding extras:
+
+.. code:: sh
+
+    pip install gokart[s3]   # S3 support
+    pip install gokart[gcs]  # GCS support
+    pip install gokart[all]  # both S3 and GCS
 
 
 Quickstart

docs/task_settings.rst

@@ -25,9 +25,15 @@ It is recommended to use the config file since it does not change much.
     gokart.add_config('base.ini')
 
 
-To use the S3 or GCS repository, please set the bucket path as ``s3://{YOUR_REPOSITORY_NAME}`` or ``gs://{YOUR_REPOSITORY_NAME}`` respectively.
+To use the S3 or GCS repository, please install the corresponding extras and set the bucket path as ``s3://{YOUR_REPOSITORY_NAME}`` or ``gs://{YOUR_REPOSITORY_NAME}`` respectively.
 
-If use S3 or GCS, please set credential information to Environment Variables.
+.. code:: sh
+
+    pip install gokart[s3]   # for S3 support
+    pip install gokart[gcs]  # for GCS support
+    pip install gokart[all]  # for both S3 and GCS
+
+Also, please set credential information to Environment Variables.
 
 .. code:: sh
 

gokart/gcs_config.py

@@ -2,11 +2,13 @@
 
 import json
 import os
-from typing import cast
+from typing import TYPE_CHECKING, cast
 
 import luigi
-import luigi.contrib.gcs
-from google.oauth2.service_account import Credentials
+
+if TYPE_CHECKING:
+    import luigi.contrib.gcs
+    from google.oauth2.service_account import Credentials
 
 
 class GCSConfig(luigi.Config):
@@ -19,9 +21,18 @@ def get_gcs_client(self) -> luigi.contrib.gcs.GCSClient:
         return self._client
 
     def _get_gcs_client(self) -> luigi.contrib.gcs.GCSClient:
+        try:
+            import luigi.contrib.gcs
+        except ImportError:
+            raise ImportError('GCS support requires additional dependencies. Install them with: pip install gokart[gcs]') from None
         return luigi.contrib.gcs.GCSClient(oauth_credentials=self._load_oauth_credentials())
 
     def _load_oauth_credentials(self) -> Credentials | None:
+        try:
+            from google.oauth2.service_account import Credentials
+        except ImportError:
+            raise ImportError('GCS support requires additional dependencies. Install them with: pip install gokart[gcs]') from None
+
         json_str = os.environ.get(self.gcs_credential_name)
         if not json_str:
             return None

gokart/gcs_obj_metadata_client.py

@@ -9,8 +9,6 @@
 from typing import Any, Final
 from urllib.parse import urlsplit
 
-from googleapiclient.model import makepatch
-
 from gokart.gcs_config import GCSConfig
 from gokart.required_task_output import RequiredTaskOutput
 from gokart.utils import FlattenableItems
@@ -39,6 +37,14 @@ def _path_to_bucket_and_key(path: str) -> tuple[str, str]:
         path_without_initial_slash = path[1:]
         return netloc, path_without_initial_slash
 
+    @staticmethod
+    def _makepatch(original: dict[str, Any], modified: dict[str, Any]) -> dict[str, Any]:
+        try:
+            from googleapiclient.model import makepatch
+        except ImportError:
+            raise ImportError('GCS support requires additional dependencies. Install them with: pip install gokart[gcs]') from None
+        return dict(makepatch(original, modified))
+
     @staticmethod
     def add_task_state_labels(
         path: str,
@@ -78,7 +84,7 @@ def add_task_state_labels(
                 .patch(
                     bucket=bucket,
                     object=obj,
-                    body=makepatch({'metadata': original_metadata}, {'metadata': patched_metadata}),
+                    body=GCSObjectMetadataClient._makepatch({'metadata': original_metadata}, {'metadata': patched_metadata}),
                 )
                 .execute()
             )

gokart/gcs_zip_client.py

@@ -4,12 +4,13 @@
 import shutil
 from typing import cast
 
-from gokart.gcs_config import GCSConfig
 from gokart.zip_client import ZipClient, _unzip_file
 
 
 class GCSZipClient(ZipClient):
     def __init__(self, file_path: str, temporary_directory: str) -> None:
+        from gokart.gcs_config import GCSConfig
+
         self._file_path = file_path
         self._temporary_directory = temporary_directory
         self._client = GCSConfig().get_gcs_client()

gokart/object_storage.py

@@ -4,14 +4,8 @@
 from typing import cast
 
 import luigi
-import luigi.contrib.gcs
-import luigi.contrib.s3
 from luigi.format import Format
 
-from gokart.gcs_config import GCSConfig
-from gokart.gcs_zip_client import GCSZipClient
-from gokart.s3_config import S3Config
-from gokart.s3_zip_client import S3ZipClient
 from gokart.zip_client import ZipClient
 
 object_storage_path_prefix = ['s3://', 'gs://']
@@ -28,26 +22,48 @@ def if_object_storage_path(path: str) -> bool:
     @staticmethod
     def get_object_storage_target(path: str, format: Format) -> luigi.target.FileSystemTarget:
         if path.startswith('s3://'):
+            try:
+                import luigi.contrib.s3
+            except ImportError:
+                raise ImportError('S3 support requires additional dependencies. Install them with: pip install gokart[s3]') from None
+
+            from gokart.s3_config import S3Config
+
             return luigi.contrib.s3.S3Target(path, client=S3Config().get_s3_client(), format=format)
         elif path.startswith('gs://'):
+            try:
+                import luigi.contrib.gcs
+            except ImportError:
+                raise ImportError('GCS support requires additional dependencies. Install them with: pip install gokart[gcs]') from None
+
+            from gokart.gcs_config import GCSConfig
+
             return luigi.contrib.gcs.GCSTarget(path, client=GCSConfig().get_gcs_client(), format=format)
         else:
             raise
 
     @staticmethod
     def exists(path: str) -> bool:
         if path.startswith('s3://'):
+            from gokart.s3_config import S3Config
+
             return cast(bool, S3Config().get_s3_client().exists(path))
         elif path.startswith('gs://'):
+            from gokart.gcs_config import GCSConfig
+
             return cast(bool, GCSConfig().get_gcs_client().exists(path))
         else:
             raise
 
     @staticmethod
     def get_timestamp(path: str) -> datetime:
         if path.startswith('s3://'):
+            from gokart.s3_config import S3Config
+
             return cast(datetime, S3Config().get_s3_client().get_key(path).last_modified)
         elif path.startswith('gs://'):
+            from gokart.gcs_config import GCSConfig
+
             # for gcs object
             # should PR to luigi
             bucket, obj = GCSConfig().get_gcs_client()._path_to_bucket_and_key(path)
@@ -59,12 +75,21 @@ def get_timestamp(path: str) -> datetime:
     @staticmethod
     def get_zip_client(file_path: str, temporary_directory: str) -> ZipClient:
         if file_path.startswith('s3://'):
+            from gokart.s3_zip_client import S3ZipClient
+
             return S3ZipClient(file_path=file_path, temporary_directory=temporary_directory)
         elif file_path.startswith('gs://'):
+            from gokart.gcs_zip_client import GCSZipClient
+
             return GCSZipClient(file_path=file_path, temporary_directory=temporary_directory)
         else:
             raise
 
     @staticmethod
     def is_buffered_reader(file: object) -> bool:
-        return not isinstance(file, luigi.contrib.s3.ReadableS3File)
+        try:
+            import luigi.contrib.s3
+
+            return not isinstance(file, luigi.contrib.s3.ReadableS3File)
+        except ImportError:
+            return True

gokart/s3_config.py

@@ -3,7 +3,6 @@
 import os
 
 import luigi
-import luigi.contrib.s3
 
 
 class S3Config(luigi.Config):
@@ -12,12 +11,16 @@ class S3Config(luigi.Config):
 
     _client = None
 
-    def get_s3_client(self) -> luigi.contrib.s3.S3Client:
+    def get_s3_client(self):
         if self._client is None:  # use cache as like singleton object
             self._client = self._get_s3_client()
         return self._client
 
-    def _get_s3_client(self) -> luigi.contrib.s3.S3Client:
+    def _get_s3_client(self):
+        try:
+            import luigi.contrib.s3
+        except ImportError:
+            raise ImportError('S3 support requires additional dependencies. Install them with: pip install gokart[s3]') from None
         return luigi.contrib.s3.S3Client(
             aws_access_key_id=os.environ.get(self.aws_access_key_id_name), aws_secret_access_key=os.environ.get(self.aws_secret_access_key_name)
         )

gokart/s3_zip_client.py

@@ -4,12 +4,13 @@
 import shutil
 from typing import cast
 
-from gokart.s3_config import S3Config
 from gokart.zip_client import ZipClient, _unzip_file
 
 
 class S3ZipClient(ZipClient):
     def __init__(self, file_path: str, temporary_directory: str) -> None:
+        from gokart.s3_config import S3Config
+
         self._file_path = file_path
         self._temporary_directory = temporary_directory
         self._client = S3Config().get_s3_client()

pyproject.toml

@@ -13,13 +13,10 @@ readme = "README.md"
 requires-python = ">=3.10, <4"
 dependencies = [
   "luigi>=3.8.0",
-  "boto3",
   "slack-sdk",
   "pandas",
   "numpy",
-  "google-auth",
   "pyarrow",
-  "google-api-python-client",
   "APScheduler",
   "redis",
   "dill",
@@ -39,6 +36,9 @@ classifiers = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
+s3 = ["boto3"]
+gcs = ["google-auth", "google-api-python-client"]
+all = ["gokart[s3]", "gokart[gcs]"]
 polars = ["polars>=0.19.0"]
 
 [project.urls]
@@ -48,6 +48,9 @@ Documentation = "https://gokart.readthedocs.io/en/latest/"
 
 [dependency-groups]
 test = [
+  "boto3",
+  "google-auth",
+  "google-api-python-client",
   "fakeredis",
   "lupa",
   "matplotlib",