Add log proxy (#41)

* Creating a log proxy to the new log listener in ESR

Author: t-persson

.gitignore

@@ -21,6 +21,7 @@ __pycache__/*
 .pydevproject
 .settings
 .idea
+.python-version
 tags
 
 # Package files

Dockerfile

@@ -9,14 +9,14 @@ ARG TZ
 ENV TZ=$TZ
 
 COPY --from=build /src/dist/*.whl /tmp
+
 # hadolint ignore=DL3013
 # hadolint ignore=DL3008
-
 RUN apt-get update && \
     apt-get install -y gcc libc-dev tzdata --no-install-recommends && \
     pip install --no-cache-dir /tmp/*.whl && \
     apt-get purge -y --auto-remove gcc libc-dev && \
-	rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/*
 
 RUN groupadd -r etos && useradd -r -m -s /bin/false -g etos etos
 USER etos

requirements.txt

@@ -18,3 +18,6 @@ uvicorn~=0.22
 fastapi~=0.96.0
 aiohttp[speedups]~=3.8
 gql[requests]~=3.4
+httpx~=0.24
+kubernetes~=26.1
+sse-starlette~=1.6

setup.cfg

@@ -30,6 +30,9 @@ install_requires =
     fastapi~=0.96.0
     aiohttp[speedups]~=3.8
     gql[requests]~=3.4
+    httpx~=0.24
+    kubernetes~=26.1
+    sse-starlette~=1.6
 
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4
 python_requires = >=3.4

src/etos_api/main.py

@@ -53,3 +53,4 @@ async def redirect_head_to_root():
 APP.include_router(routers.etos.ROUTER)
 APP.include_router(routers.selftest.ROUTER)
 APP.include_router(routers.environment_provider.ROUTER)
+APP.include_router(routers.logs.ROUTER)

src/etos_api/routers/__init__.py

@@ -14,4 +14,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """ETOS API routers module."""
-from . import environment_provider, etos, selftest
+from . import environment_provider, etos, selftest, logs

src/etos_api/routers/logs/__init__.py

@@ -0,0 +1,17 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS API log handler."""
+from .router import ROUTER

src/etos_api/routers/logs/router.py

@@ -0,0 +1,76 @@
+# Copyright Axis Communications AB.
+#
+# For a full list of individual contributors, please see the commit history.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""ETOS API log handler."""
+import asyncio
+import logging
+from uuid import UUID
+from kubernetes import client, config
+from fastapi import APIRouter, HTTPException
+
+from sse_starlette.sse import EventSourceResponse
+from starlette.requests import Request
+import httpx
+
+LOGGER = logging.getLogger(__name__)
+ROUTER = APIRouter()
+
+try:
+    config.load_incluster_config()
+except config.ConfigException:
+    try:
+        config.load_config()
+    except config.ConfigException:
+        LOGGER.warning("Could not load a Kubernetes config")
+
+
+@ROUTER.get("/logs/{uuid}", tags=["logs"])
+async def get_logs(uuid: UUID, request: Request):
+    """Get logs from an ETOS pod and stream them back as server sent events."""
+    corev1 = client.CoreV1Api()
+    thread = corev1.list_namespaced_pod("etos-development", async_req=True)
+    pod_list = thread.get()
+
+    ip_addr = None
+    for pod in pod_list.items:
+        if pod.status.phase == "Running" and pod.metadata.name.startswith(
+            f"suite-runner-{str(uuid)}"
+        ):
+            ip_addr = pod.status.pod_ip
+    if ip_addr is None:
+        raise HTTPException(
+            status_code=404, detail=f"Suite runner with UUID={uuid} not found"
+        )
+
+    async def sse(url):
+        index = 0
+        while True:
+            if await request.is_disconnected():
+                break
+            try:
+                response = httpx.get(url)
+                lines = response.text.splitlines()
+                for message in lines[index:]:
+                    yield {"id": index + 1, "event": "message", "data": message}
+                    index += 1
+            except httpx.RemoteProtocolError:
+                LOGGER.exception("Failed to connect to pod %r", url)
+            except IndexError:
+                pass
+            await asyncio.sleep(1)
+
+    return EventSourceResponse(
+        sse(f"http://{ip_addr}:8000/log"),
+    )