🔨 Profile api server (#4754)

Author: bisgaard-itis

services/api-server/README.md

@@ -36,6 +36,11 @@ will start the api-server in development-mode together with a postgres db initia
 - http://127.0.0.1:8000/docs: redoc documentation
 - http://127.0.0.1:8000/dev/docs: swagger type of documentation
 
+### Profiling requests to the api server
+When in development mode (the environment variable `API_SERVER_DEV_FEATURES_ENABLED` is =1 in the running container) one can profile calls to the API server directly from the client side. On the server, the profiling is done using [Pyinstrument](https://github.com/joerick/pyinstrument). If we have our request in the form of a curl command, one simply adds the custom header `x-profile-api-server:true` to the command, in which case the profile is received under the `profile` key of the response body. This makes it easy to visualise the profiling report directly in bash:
+```bash
+<curl_command> -H 'x-profile-api-server: true' | jq -r .profile
+```
 
 ## Clients
 

services/api-server/requirements/_test.in

@@ -19,6 +19,7 @@ docker
 faker
 jsonref
 moto[server] # mock out tests based on AWS-S3
+pyinstrument
 pytest
 pytest-asyncio
 pytest-cov

services/api-server/requirements/_test.txt

@@ -234,6 +234,10 @@ pycparser==2.21
     # via
     #   -c requirements/_base.txt
     #   cffi
+pyinstrument==4.5.0
+    # via
+    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
 pyparsing==3.1.1
     # via moto
 pyrsistent==0.19.3

services/api-server/src/simcore_service_api_server/core/_profiler_middleware.py

@@ -0,0 +1,54 @@
+import json
+
+from fastapi import FastAPI
+from pyinstrument import Profiler
+from starlette.requests import Request
+
+
+def _generate_response_headers(content: bytes) -> list[tuple[bytes, bytes]]:
+    headers: dict = dict()
+    headers[b"content-length"] = str(len(content)).encode("utf8")
+    headers[b"content-type"] = b"application/json"
+    return list(headers.items())
+
+
+class ApiServerProfilerMiddleware:
+    """Following
+    https://www.starlette.io/middleware/#cleanup-and-error-handling
+    https://www.starlette.io/middleware/#reusing-starlette-components
+    https://fastapi.tiangolo.com/advanced/middleware/#advanced-middleware
+    """
+
+    def __init__(self, app: FastAPI):
+        self._app: FastAPI = app
+        self._profile_header_trigger: str = "x-profile-api-server"
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self._app(scope, receive, send)
+            return
+
+        profiler = Profiler(async_mode="enabled")
+        request: Request = Request(scope)
+        headers = dict(request.headers)
+        if self._profile_header_trigger in headers:
+            headers.pop(self._profile_header_trigger)
+            scope["headers"] = [
+                (k.encode("utf8"), v.encode("utf8")) for k, v in headers.items()
+            ]
+            profiler.start()
+
+        async def send_wrapper(message):
+            if profiler.is_running:
+                profiler.stop()
+            if profiler.last_session:
+                body: bytes = json.dumps(
+                    {"profile": profiler.output_text(unicode=True, color=True)}
+                ).encode("utf8")
+                if message["type"] == "http.response.start":
+                    message["headers"] = _generate_response_headers(body)
+                elif message["type"] == "http.response.body":
+                    message["body"] = body
+            await send(message)
+
+        await self._app(scope, receive, send_wrapper)

services/api-server/src/simcore_service_api_server/core/application.py

@@ -113,6 +113,10 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
             else None,
         ),
     )
+    if settings.API_SERVER_DEV_FEATURES_ENABLED:
+        from ._profiler_middleware import ApiServerProfilerMiddleware
+
+        app.add_middleware(ApiServerProfilerMiddleware)
 
     # routing