ensure we wait for the fastapi app to be up and running

Author: sanderegg

services/storage/src/simcore_service_storage/modules/celery/signals.py

@@ -1,4 +1,5 @@
 import asyncio
+import datetime
 import logging
 import threading
 from typing import Final
@@ -21,28 +22,33 @@
 _logger = logging.getLogger(__name__)
 
 _LIFESPAN_TIMEOUT: Final[int] = 10
+_FASTAPI_STARTUP_TIMEOUT: Final[float] = datetime.timedelta(minutes=1).total_seconds()
 
 
 def on_worker_init(sender, **_kwargs) -> None:
-    def _init_fastapi() -> None:
+    startup_complete_event = threading.Event()
+
+    def _init_fastapi(startup_complete_event: threading.Event) -> None:
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
         shutdown_event = asyncio.Event()
 
         fastapi_app = create_app(ApplicationSettings.create_from_envs())
 
-        async def lifespan():
+        async def lifespan(startup_complete_event: threading.Event) -> None:
             async with LifespanManager(
                 fastapi_app,
                 startup_timeout=_LIFESPAN_TIMEOUT,
                 shutdown_timeout=_LIFESPAN_TIMEOUT,
             ):
                 try:
+                    _logger.info("fastapi APP started!")
+                    startup_complete_event.set()
                     await shutdown_event.wait()
                 except asyncio.CancelledError:
                     _logger.warning("Lifespan task cancelled")
 
-        lifespan_task = loop.create_task(lifespan())
+        lifespan_task = loop.create_task(lifespan(startup_complete_event))
         fastapi_app.state.lifespan_task = lifespan_task
         fastapi_app.state.shutdown_event = shutdown_event
         set_event_loop(fastapi_app, loop)
@@ -52,8 +58,15 @@ async def lifespan():
 
         loop.run_forever()
 
-    thread = threading.Thread(target=_init_fastapi, daemon=True)
+    thread = threading.Thread(
+        target=_init_fastapi,
+        name="fastapi_app",
+        args=(startup_complete_event,),
+        daemon=True,
+    )
     thread.start()
+    # ensure the fastapi app is ready before going on
+    startup_complete_event.wait(_FASTAPI_STARTUP_TIMEOUT)
 
 
 def on_worker_shutdown(sender, **_kwargs):