fix: concurrent bucket creation

Author: giancarloromeo

services/storage/src/simcore_service_storage/modules/s3.py

@@ -8,6 +8,7 @@
 from fastapi import FastAPI
 from pydantic import TypeAdapter
 from servicelib.logging_utils import log_context
+from tenacity import wait_exponential
 from tenacity.asyncio import AsyncRetrying
 from tenacity.before_sleep import before_sleep_log
 from tenacity.wait import wait_fixed
@@ -44,14 +45,29 @@ async def _on_startup() -> None:
                 assert client  # nosec
         app.state.s3_client = client
 
-        with log_context(_logger, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"):
+        async for attempt in AsyncRetrying(
+            wait=wait_exponential(min=1),  # 1, 2, 4, 8, ...
+            before_sleep=before_sleep_log(_logger, logging.WARNING),
+            reraise=True,
+        ):
             assert settings.STORAGE_S3  # nosec
-            await client.create_bucket(
-                bucket=settings.STORAGE_S3.S3_BUCKET_NAME,
-                region=TypeAdapter(
-                    BucketLocationConstraintType | Literal["us-east-1"]
-                ).validate_python(settings.STORAGE_S3.S3_REGION),
-            )
+            with attempt, log_context(
+                _logger, logging.DEBUG, msg="setup.s3_bucket.cleanup_ctx"
+            ):
+                if await client.bucket_exists(
+                    bucket=settings.STORAGE_S3.S3_BUCKET_NAME
+                ):
+                    _logger.info(
+                        "S3 bucket %s exists already, skipping creation",
+                        settings.STORAGE_S3.S3_BUCKET_NAME,
+                    )
+                    break
+                await client.create_bucket(
+                    bucket=settings.STORAGE_S3.S3_BUCKET_NAME,
+                    region=TypeAdapter(
+                        BucketLocationConstraintType | Literal["us-east-1"]
+                    ).validate_python(settings.STORAGE_S3.S3_REGION),
+                )
 
     async def _on_shutdown() -> None:
         if app.state.s3_client: