🐛 Concurrent S3 bucket creation attempt (#8045)

Author: giancarloromeo

services/storage/src/simcore_service_storage/modules/s3.py

@@ -7,19 +7,42 @@
 from common_library.json_serialization import json_dumps
 from fastapi import FastAPI
 from pydantic import TypeAdapter
-from servicelib.logging_utils import log_context
+from tenacity import retry, wait_random_exponential
 from tenacity.asyncio import AsyncRetrying
 from tenacity.before_sleep import before_sleep_log
 from tenacity.wait import wait_fixed
 from types_aiobotocore_s3.literals import BucketLocationConstraintType
 
 from ..constants import RETRY_WAIT_SECS
-from ..core.settings import get_application_settings
+from ..core.settings import ApplicationSettings, get_application_settings
 from ..exceptions.errors import ConfigurationError
 
 _logger = logging.getLogger(__name__)
 
 
+@retry(
+    wait=wait_random_exponential(),
+    before_sleep=before_sleep_log(_logger, logging.WARNING),
+    reraise=True,
+)
+async def _ensure_s3_bucket(
+    client: SimcoreS3API, settings: ApplicationSettings
+) -> None:
+    assert settings.STORAGE_S3  # nosec
+    if await client.bucket_exists(bucket=settings.STORAGE_S3.S3_BUCKET_NAME):
+        _logger.info(
+            "S3 bucket %s exists already, skipping creation",
+            settings.STORAGE_S3.S3_BUCKET_NAME,
+        )
+        return
+    await client.create_bucket(
+        bucket=settings.STORAGE_S3.S3_BUCKET_NAME,
+        region=TypeAdapter(
+            BucketLocationConstraintType | Literal["us-east-1"]
+        ).validate_python(settings.STORAGE_S3.S3_REGION),
+    )
+
+
 def setup_s3(app: FastAPI) -> None:
     async def _on_startup() -> None:
         app.state.s3_client = None
@@ -44,14 +67,7 @@ async def _on_startup() -> None:
                 assert client  # nosec
         app.state.s3_client = client
 
-        with log_context(_logger, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"):
-            assert settings.STORAGE_S3  # nosec
-            await client.create_bucket(
-                bucket=settings.STORAGE_S3.S3_BUCKET_NAME,
-                region=TypeAdapter(
-                    BucketLocationConstraintType | Literal["us-east-1"]
-                ).validate_python(settings.STORAGE_S3.S3_REGION),
-            )
+        await _ensure_s3_bucket(client, settings)
 
     async def _on_shutdown() -> None:
         if app.state.s3_client: