qontract-api: sentry/glitchtip support (#5404)

* qontract-api: sentry/glitchtip support

* tests

* openshift deployment config

Author: chassing

openshift/qontract-api.yaml

@@ -95,6 +95,11 @@ objects:
                   value: api
                 - name: QAPI_APP_PORT
                   value: "${QAPI_APP_PORT}"
+                - name: QAPI_SENTRY_DSN
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${QAPI_SENTRY_SECRET_NAME}
+                      key: dsn
               envFrom:
                 - secretRef:
                     name: qontract-api-secret
@@ -227,6 +232,11 @@ objects:
                   value: "${QAPI_WORKER_METRICS_PORT}"
                 - name: QAPI_WORKER_TEMP_DIR
                   value: /worker-temp-dir
+                - name: QAPI_SENTRY_DSN
+                  valueFrom:
+                    secretKeyRef:
+                      name: ${QAPI_SENTRY_SECRET_NAME}
+                      key: dsn
               envFrom:
                 - secretRef:
                     name: qontract-api-secret
@@ -329,6 +339,11 @@ parameters:
     value: "INFO"
     required: true
 
+  - name: QAPI_SENTRY_SECRET_NAME
+    description: Name of the secret containing the Sentry DSN
+    value: "app-interface-prod-dsn"
+    required: true
+
   ## Api Pod limits
   - name: QAPI_API_REPLICAS
     description: Web replicas

qontract_api/pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
     "python-jose[cryptography]==3.5.0",
     "qontract-utils",
     "redis==7.1.0",
+    "sentry-sdk==2.51.0",
     "structlog==25.5.0",
     "uvicorn[standard]==0.38.0",
 ]

qontract_api/qontract_api/config.py

@@ -2,7 +2,7 @@
 
 from typing import Any
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from pydantic_settings import (
     BaseSettings,
     JsonConfigSettingsSource,
@@ -269,6 +269,15 @@ def settings_customise_sources(
         default="slack_sdk,httpcore,github.Requester",
         description="Comma-separated list of logger names to exclude from logging",
     )
+    # Sentry/Glitchtip
+    sentry_dsn: str = Field(
+        default="",
+        description="Sentry DSN for error tracking (Glitchtip compatible)",
+    )
+    sentry_event_level: str = Field(
+        default="ERROR",
+        description="Minimum log level to send events to Sentry",
+    )
 
     # Cache Backend
     cache_backend: str = Field(
@@ -347,6 +356,18 @@ def settings_customise_sources(
         description="Secret backend configuration (Vault, AWS KMS, Google)",
     )
 
+    @field_validator("sentry_event_level", mode="after")
+    @classmethod
+    def validate_sentry_event_level(cls, value: str) -> str:
+        """Validate sentry_event_level is a valid logging level."""
+        valid_levels = {"ERROR", "CRITICAL"}
+        value = value.upper()
+        if value not in valid_levels:
+            raise ValueError(
+                f"sentry_event_level must be one of {valid_levels}, got {value}"
+            )
+        return value
+
     def get_celery_broker_url(self) -> str:
         """Get Celery broker URL, defaulting to cache_broker_url."""
         return self.celery_broker_url or self.cache_broker_url

qontract_api/qontract_api/logger.py

@@ -9,7 +9,9 @@
 
 import logging
 
+import sentry_sdk
 import structlog
+from sentry_sdk.integrations.logging import LoggingIntegration
 
 from qontract_api.config import settings
 
@@ -93,6 +95,32 @@ def setup_logging() -> structlog.typing.WrappedLogger:
         excluded_logger = logging.getLogger(logger_name)
         excluded_logger.setLevel(logging.WARNING)
 
+    # initialize Sentry/Glitchtip integration if DSN is provided
+    if settings.sentry_dsn:
+        match settings.sentry_event_level:
+            case "CRITICAL":
+                sentry_event_level = logging.CRITICAL
+            case "ERROR":
+                sentry_event_level = logging.ERROR
+            case _:
+                raise NotImplementedError(
+                    f"Unsupported sentry_event_level: {settings.sentry_event_level}"
+                )
+
+        sentry_sdk.init(
+            dsn=settings.sentry_dsn,
+            # Add data like request headers and IP for users, if applicable;
+            # see https://docs.sentry.io/platforms/python/data-management/data-collected/ for more info
+            # Sensitive headers are automatically filtered by Sentry.
+            send_default_pii=True,
+            # Enable logs to be sent to Sentry
+            enable_logs=True,
+            ignore_errors=[ConnectionError],
+            integrations=[
+                LoggingIntegration(event_level=sentry_event_level),
+            ],
+        )
+
     return structlog.get_logger("qontract_api")
 
 

qontract_api/tests/test_logger.py

@@ -5,6 +5,7 @@
 from io import StringIO
 from unittest.mock import patch
 
+import pytest
 import structlog
 from structlog.typing import Processor
 
@@ -26,6 +27,7 @@ def test_setup_logging_returns_qontract_api_logger() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = True
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         logger = setup_logging()
 
@@ -41,6 +43,7 @@ def test_setup_logging_with_json_format() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = True
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         # Clear existing handlers
         root_logger = logging.getLogger()
@@ -61,6 +64,7 @@ def test_setup_logging_with_standard_format() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = False
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         # Clear existing handlers
         root_logger = logging.getLogger()
@@ -81,6 +85,7 @@ def test_logger_with_json_output() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = True
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         # Setup logging with JSON format
         root_logger = logging.getLogger()
@@ -141,6 +146,7 @@ def test_logger_with_context_vars() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = True
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         # Setup logging with JSON format
         root_logger = logging.getLogger()
@@ -200,6 +206,7 @@ def test_logger_with_extra_fields() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = True
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         # Setup logging
         root_logger = logging.getLogger()
@@ -267,6 +274,7 @@ def test_logger_with_exception() -> None:
         mock_settings.log_level = "INFO"
         mock_settings.log_format_json = True
         mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
 
         # Setup logging
         root_logger = logging.getLogger()
@@ -324,3 +332,123 @@ def test_logger_with_exception() -> None:
         assert exc_info["exc_value"] == "Test error"
 
         root_logger.handlers.clear()
+
+
+def test_sentry_init_called_when_dsn_provided() -> None:
+    """Test that sentry_sdk.init is called when SENTRY_DSN is provided."""
+    with (
+        patch("qontract_api.logger.settings") as mock_settings,
+        patch("qontract_api.logger.sentry_sdk.init") as mock_sentry_init,
+    ):
+        mock_settings.log_level = "INFO"
+        mock_settings.log_format_json = True
+        mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = "https://example@sentry.io/123"
+        mock_settings.sentry_event_level = "ERROR"
+
+        root_logger = logging.getLogger()
+        root_logger.handlers.clear()
+
+        setup_logging()
+
+        mock_sentry_init.assert_called_once()
+        call_args = mock_sentry_init.call_args
+
+        assert call_args.kwargs["dsn"] == "https://example@sentry.io/123"
+        assert call_args.kwargs["send_default_pii"] is True
+        assert call_args.kwargs["enable_logs"] is True
+        assert call_args.kwargs["ignore_errors"] == [ConnectionError]
+
+        root_logger.handlers.clear()
+
+
+def test_sentry_init_not_called_when_dsn_empty() -> None:
+    """Test that sentry_sdk.init is not called when SENTRY_DSN is empty."""
+    with (
+        patch("qontract_api.logger.settings") as mock_settings,
+        patch("qontract_api.logger.sentry_sdk.init") as mock_sentry_init,
+    ):
+        mock_settings.log_level = "INFO"
+        mock_settings.log_format_json = True
+        mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = ""
+
+        root_logger = logging.getLogger()
+        root_logger.handlers.clear()
+
+        setup_logging()
+
+        mock_sentry_init.assert_not_called()
+
+        root_logger.handlers.clear()
+
+
+def test_sentry_init_with_error_level() -> None:
+    """Test sentry_sdk.init uses ERROR event level correctly."""
+    with (
+        patch("qontract_api.logger.settings") as mock_settings,
+        patch("qontract_api.logger.sentry_sdk.init") as mock_sentry_init,
+        patch("qontract_api.logger.LoggingIntegration") as mock_logging_integration,
+    ):
+        mock_settings.log_level = "INFO"
+        mock_settings.log_format_json = True
+        mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = "https://example@sentry.io/123"
+        mock_settings.sentry_event_level = "ERROR"
+
+        root_logger = logging.getLogger()
+        root_logger.handlers.clear()
+
+        setup_logging()
+
+        mock_sentry_init.assert_called_once()
+        mock_logging_integration.assert_called_once_with(event_level=logging.ERROR)
+
+        root_logger.handlers.clear()
+
+
+def test_sentry_init_with_critical_level() -> None:
+    """Test sentry_sdk.init uses CRITICAL event level correctly."""
+    with (
+        patch("qontract_api.logger.settings") as mock_settings,
+        patch("qontract_api.logger.sentry_sdk.init") as mock_sentry_init,
+        patch("qontract_api.logger.LoggingIntegration") as mock_logging_integration,
+    ):
+        mock_settings.log_level = "INFO"
+        mock_settings.log_format_json = True
+        mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = "https://example@sentry.io/123"
+        mock_settings.sentry_event_level = "CRITICAL"
+
+        root_logger = logging.getLogger()
+        root_logger.handlers.clear()
+
+        setup_logging()
+
+        mock_sentry_init.assert_called_once()
+        mock_logging_integration.assert_called_once_with(event_level=logging.CRITICAL)
+
+        root_logger.handlers.clear()
+
+
+def test_sentry_init_with_invalid_level_raises_error() -> None:
+    """Test that invalid SENTRY_EVENT_LEVEL raises NotImplementedError."""
+    with (
+        patch("qontract_api.logger.settings") as mock_settings,
+        patch("qontract_api.logger.sentry_sdk.init"),
+    ):
+        mock_settings.log_level = "INFO"
+        mock_settings.log_format_json = True
+        mock_settings.log_exclude_loggers = ""
+        mock_settings.sentry_dsn = "https://example@sentry.io/123"
+        mock_settings.sentry_event_level = "WARNING"
+
+        root_logger = logging.getLogger()
+        root_logger.handlers.clear()
+
+        with pytest.raises(
+            NotImplementedError, match="Unsupported sentry_event_level: WARNING"
+        ):
+            setup_logging()
+
+        root_logger.handlers.clear()