Merge pull request #22 from MannLabs/improve_webapp_logs

feat: enhance logging for webapp and configure volume for log storage

Author: mschwoer

airflow_src/plugins/sensors/ssh_utils.py

@@ -14,7 +14,7 @@ def ssh_execute(
     command: str,
     ssh_hook: SSHHook | None = None,
     *,
-    max_tries: int = 10,
+    max_tries: int = 30,
 ) -> str:
     """Execute the given `command` via the `ssh_hook`.
 
@@ -41,7 +41,7 @@ def ssh_execute(
             logging.error(f"Execution of SSH command failed too often: {command=}")
             raise AirflowFailException("Execution of SSH command failed too often.")
 
-        sleep(30 * call_count)  # no sleep in the first iteration
+        sleep(60 * call_count)  # no sleep in the first iteration
         call_count += 1
 
         try:

airflow_src/tests/plugins/sensors/test_ssh_utils.py

@@ -65,11 +65,11 @@ def test_ssh_execute_too_many_tries(mock_sleep: MagicMock) -> None:
     # given
     ssh_hook = MagicMock()
     bad_return = (254, b"command output", b"")
-    ssh_hook.exec_ssh_client_command.side_effect = [bad_return] * 11
+    ssh_hook.exec_ssh_client_command.side_effect = [bad_return] * 31
 
     # when
     with pytest.raises(AirflowFailException):
         ssh_execute("my_command", ssh_hook)
 
-    assert mock_sleep.call_count == 10  # noqa: PLR2004
-    assert ssh_hook.exec_ssh_client_command.call_count == 10  # noqa: PLR2004
+    assert mock_sleep.call_count == 30  # noqa: PLR2004
+    assert ssh_hook.exec_ssh_client_command.call_count == 30  # noqa: PLR2004

docker-compose.yaml

@@ -319,8 +319,9 @@ services:
       retries: 5
       start_period: 30s
     restart: always
-  #    # Uncomment for local development so that changes to webapp are immediately active
-  #    volumes:
+    volumes:
+      - ./webapp_logs:/app/logs
+  # Uncomment for local development so that changes to webapp are immediately active
   #      - ./webapp:/app/webapp
   #      - ./shared:/app/webapp/shared
 

webapp/conftest.py

@@ -0,0 +1,5 @@
+"""Test configuration for webapp tests."""
+
+import os
+
+os.environ["IS_PYTEST_RUN"] = "1"  # Indicate that tests are running

webapp/service/utils.py

@@ -1,13 +1,12 @@
 """Utilities for the streamlit app."""
 
 import io
+import logging
 import os
 import traceback
-from datetime import datetime
 from pathlib import Path
 
 import plotly.graph_objects as go
-import pytz
 import streamlit as st
 from PIL import Image
 
@@ -47,20 +46,30 @@ class QueryParams:
 DEFAULT_MAX_AGE_OVERVIEW = 30  # days
 DEFAULT_MAX_AGE_STATUS = 90  # days
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[
+        logging.FileHandler("/app/logs/webapp.log")
+        if os.getenv("IS_PYTEST_RUN", "0") != "1"
+        else None,
+        logging.StreamHandler(),  # Keep console output for debugging
+    ],
+)
+logger = logging.getLogger(__name__)
+
 
 def _log(item_to_log: str | Exception, extra_msg: str = "") -> None:
     """Write a log message and show it if it's an exception."""
-    now = datetime.now(tz=pytz.UTC).strftime("%Y-%m-%d %H:%M:%S.%f")
-
     if isinstance(item_to_log, Exception):
         if extra_msg:
             st.write(extra_msg)
             st.write(item_to_log)
         msg = f"{extra_msg}{item_to_log}\n{traceback.format_exc()}"
+        logger.error(msg, exc_info=True)
     else:
-        msg = f"{now}: {item_to_log}"
-
-    os.write(1, f"{msg}\n".encode())
+        logger.info(item_to_log)
 
 
 def empty_to_none(value: str) -> str | None: