big move

Author: sanderegg

services/storage/src/simcore_service_storage/api/rest/health.py

@@ -19,7 +19,7 @@
 from ...core.settings import Settings
 from ...db import get_engine_state
 from ...db import is_service_responsive as is_pg_responsive
-from ...s3 import get_s3_client
+from ...modules.s3 import get_s3_client
 
 _logger = logging.getLogger(__name__)
 

services/storage/src/simcore_service_storage/core/application.py

@@ -22,8 +22,8 @@
 from ..long_running_tasks import setup_rest_api_long_running_tasks
 from ..modules.db import setup_db
 from ..modules.redis import setup_redis
+from ..modules.s3 import setup_s3
 from ..routes import setup_rest_api_routes
-from ..s3 import setup_s3
 from .settings import Settings
 
 _ACCESS_LOG_FORMAT: Final[

services/storage/src/simcore_service_storage/s3.py renamed to services/storage/src/simcore_service_storage/modules/s3.py

@@ -12,8 +12,8 @@
 from tenacity.before_sleep import before_sleep_log
 from tenacity.wait import wait_fixed
 
-from .constants import APP_CONFIG_KEY, APP_S3_KEY, RETRY_WAIT_SECS
-from .core.settings import Settings
+from ..constants import APP_CONFIG_KEY, APP_S3_KEY, RETRY_WAIT_SECS
+from ..core.settings import Settings
 
 _logger = logging.getLogger(__name__)
 

services/storage/src/simcore_service_storage/simcore_s3_dsm.py

@@ -78,19 +78,19 @@
     UploadLinks,
     UserOrProjectFilter,
 )
-from .s3 import get_s3_client
-from .s3_utils import S3TransferDataCB, update_task_progress
-from .simcore_s3_dsm_utils import (
-    compute_file_id_prefix,
-    expand_directory,
-    get_directory_file_id,
-)
+from .modules.s3 import get_s3_client
 from .utils import (
     convert_db_to_model,
     download_to_file_or_raise,
     is_file_entry_valid,
     is_valid_managed_multipart_upload,
 )
+from .utils.s3_utils import S3TransferDataCB, update_task_progress
+from .utils.simcore_s3_dsm_utils import (
+    compute_file_id_prefix,
+    expand_directory,
+    get_directory_file_id,
+)
 
 _NO_CONCURRENCY: Final[int] = 1
 _MAX_PARALLEL_S3_CALLS: Final[NonNegativeInt] = 10

services/storage/src/simcore_service_storage/utils/__init__.py

@@ -12,10 +12,10 @@
 from pydantic import ByteSize, NonNegativeInt, TypeAdapter
 from servicelib.utils import ensure_ends_with
 
-from . import db_file_meta_data
-from .core.exceptions import FileMetaDataNotFoundError
-from .models import FileMetaData, FileMetaDataAtDB
-from .utils import convert_db_to_model
+from .. import db_file_meta_data
+from ..core.exceptions import FileMetaDataNotFoundError
+from ..models import FileMetaData, FileMetaDataAtDB
+from . import convert_db_to_model
 
 
 async def _list_all_files_in_folder(

services/storage/src/simcore_service_storage/s3_utils.py renamed to services/storage/src/simcore_service_storage/utils/s3_utils.py

@@ -54,7 +54,7 @@
 from simcore_service_storage.core.settings import Settings
 from simcore_service_storage.dsm import get_dsm_provider
 from simcore_service_storage.models import S3BucketName
-from simcore_service_storage.s3 import get_s3_client
+from simcore_service_storage.modules.s3 import get_s3_client
 from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager
 from tenacity.asyncio import AsyncRetrying
 from tenacity.retry import retry_if_exception_type

services/storage/src/simcore_service_storage/simcore_s3_dsm_utils.py renamed to services/storage/src/simcore_service_storage/utils/simcore_s3_dsm_utils.py

@@ -15,7 +15,7 @@
 from pydantic import ByteSize, TypeAdapter
 from simcore_service_storage import db_file_meta_data
 from simcore_service_storage.models import FileMetaData
-from simcore_service_storage.s3 import get_s3_client
+from simcore_service_storage.modules.s3 import get_s3_client
 from simcore_service_storage.simcore_s3_dsm import SimcoreS3DataManager
 
 pytest_simcore_core_services_selection = ["postgres"]

services/storage/tests/conftest.py

@@ -1,5 +1,5 @@
 import pytest
-from simcore_service_storage.simcore_s3_dsm_utils import compute_file_id_prefix
+from simcore_service_storage.utils.simcore_s3_dsm_utils import compute_file_id_prefix
 
 
 @pytest.mark.parametrize(