Add Sentry support (#153)

* Add Sentry support (#113)

* Backport production changes (#119)

* Catch timeout exception more carefully (#120)

* Fix Sentry capturing message (#123)

---------

Co-authored-by: Lev <94750963+Kwaizer@users.noreply.github.com>
Co-authored-by: Maxim <mpetuhov@almalinux.org>

Author: 3 people

alts/scheduler/app.py

@@ -6,6 +6,7 @@
 
 import aiohttp
 import logging
+import sentry_sdk
 import signal
 import urllib.parse
 from threading import Event
@@ -40,6 +41,16 @@
 graceful_terminate_event = Event()
 http_bearer_scheme = HTTPBearer()
 
+if CONFIG.sentry_dsn:
+    sentry_sdk.init(
+        dsn=CONFIG.sentry_dsn,
+        traces_sample_rate=CONFIG.sentry_traces_sample_rate,
+        profiles_sample_rate=CONFIG.sentry_profiles_sample_rate,
+        environment=CONFIG.sentry_environment,
+        ignore_errors=[
+            ConnectionResetError
+        ],
+    )
 
 def get_celery_task_result(task_id: str, timeout: int = 1) -> tuple:
     """

alts/shared/models.py

@@ -245,6 +245,11 @@ def __init__(self, **data):
             ):
                 setattr(self, field_name, field)
 
+    # Sentry params
+    sentry_dsn: str = ''
+    sentry_traces_sample_rate: float = 0.2
+    sentry_profiles_sample_rate: float = 0.2
+    sentry_environment: str = 'dev'
     # Whether to setup Celery SSL
     use_ssl: bool = False
     # SSL configuration section

alts/worker/app.py

@@ -1,5 +1,6 @@
 import itertools
 import logging
+import sentry_sdk
 
 from celery import Celery
 from kombu import Exchange, Queue
@@ -32,6 +33,17 @@
 celery_app.conf.task_default_exchange = 'default'
 celery_app.conf.task_default_routing_key = 'default'
 
+if CONFIG.sentry_dsn:
+    sentry_sdk.init(
+        dsn=CONFIG.sentry_dsn,
+        traces_sample_rate=CONFIG.sentry_traces_sample_rate,
+        profiles_sample_rate=CONFIG.sentry_profiles_sample_rate,
+        environment=CONFIG.sentry_environment,
+        ignore_errors=[
+            ConnectionResetError
+        ],
+    )
+
 if CONFIG.use_ssl:
     if not getattr(CONFIG, 'ssl_config'):
         raise ValueError('Empty SSL configuration section')

alts/worker/runners/base.py

@@ -4,8 +4,8 @@
 import os
 import random
 import re
-import signal
 import shutil
+import signal
 import tempfile
 import time
 import traceback
@@ -18,15 +18,16 @@
     Dict,
     List,
     Optional,
-    Union,
     Tuple,
     Type,
+    Union,
 )
 
 from billiard.exceptions import SoftTimeLimitExceeded
 from filelock import FileLock
 from mako.lookup import TemplateLookup
-from plumbum import local, ProcessExecutionError, ProcessTimedOut
+from plumbum import ProcessExecutionError, ProcessTimedOut, local
+from sentry_sdk import capture_message
 
 from alts.shared.constants import COMMAND_TIMEOUT_EXIT_CODE
 from alts.shared.exceptions import (
@@ -599,6 +600,7 @@ def run_ansible_command(
             stderr = f'Timeout occurred when running ansible command: "{formulated_cmd}"'
             exit_code = COMMAND_TIMEOUT_EXIT_CODE
             exception_happened = True
+            capture_message(stderr)
         except Exception as e:
             self._logger.error(
                 'Unknown error happened during %s execution: %s',
@@ -759,7 +761,7 @@ def initial_provision(self, verbose=False):
         )
         if exit_code == COMMAND_TIMEOUT_EXIT_CODE:
             return 1, '', f'Provision has timed out: {out}\n{err}'
-        elif exit_code != 0:
+        if exit_code != 0:
             return 1, '', f'Provision exited abnormally: {out}\n{err}'
         return exit_code, out, err
 

requirements/base.txt

@@ -12,5 +12,6 @@ pulpcore-client==3.61.0
 pydantic==2.9.2
 pyone==6.10.0
 requests==2.32.3
+sentry-sdk==2.22.0
 syncer==2.0.3
 tap.py==3.1