feat: celery

Add celery task:
- prune_inactive_app
- close_window

Author: Kl0ven

controller/__init__.py

@@ -0,0 +1,3 @@
+from controller.celery import app as celery_app
+
+__all__ = ("celery_app",)

controller/celery.py

@@ -0,0 +1,17 @@
+import os
+
+from celery import Celery
+
+# Set the default Django settings module for the 'celery' program.
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "controller.settings")
+
+app = Celery("controller")
+
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+# Load task modules from all registered Django apps.
+app.autodiscover_tasks()

controller/sentry/tasks.py

@@ -0,0 +1,26 @@
+from datetime import timedelta
+
+from celery import shared_task
+from celery.utils.log import get_task_logger
+from django.conf import settings
+from django.db.models import F
+from django.utils import timezone
+
+from controller.sentry.models import App
+
+LOGGER = get_task_logger(__name__)
+
+
+@shared_task()
+def prune_inactive_app() -> None:
+    last_seen = timezone.now() - timedelta(days=settings.APP_AUTO_PRUNE_MAX_AGE_DAY)
+    apps = App.objects.filter(last_seen__lt=last_seen)
+    if apps_count := apps.count():
+        LOGGER.info("Pruning %s apps", apps_count)
+        apps.delete()
+
+
+@shared_task()
+def close_window() -> None:
+    apps = App.objects.filter(active_window_end__lt=timezone.now())
+    apps.update(active_sample_rate=F("default_sample_rate"), active_window_end=None)

controller/sentry/views.py

@@ -23,9 +23,6 @@ def retrieve(self, request, *args, **kwargs):
         panic = cache.get(settings.PANIC_KEY)
         now = timezone.now()
         app.last_seen = now
-        if app.active_window_end and app.active_window_end < now:
-            app.active_sample_rate = app.default_sample_rate
-            app.active_window_end = None
         app.save()
         if panic:
             app.active_sample_rate = 0.0

controller/settings.py

@@ -12,6 +12,9 @@
 import os
 import sys
 from pathlib import Path
+from urllib.parse import quote
+
+from celery.schedules import crontab
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
@@ -27,6 +30,7 @@
 DEBUG = os.getenv("ENV", "production") != "production"
 TESTING = sys.argv[1:2] == ["test"] or os.getenv("TESTING")
 
+# Static and Media
 STATIC_URL = os.getenv("STATIC_URL", "/assets/static/")
 MEDIA_URL = os.getenv("MEDIA_URL", "/assets/media/")
 STATIC_ROOT = os.path.join(BASE_DIR, "assets/static")
@@ -42,7 +46,13 @@
     ALLOWED_HOSTS = ["*"]
 
 # Application definition
+# URLs
+ROOT_URLCONF = "controller.urls"
 
+# WSGI
+WSGI_APPLICATION = "controller.wsgi.application"
+
+# Application definition
 INSTALLED_APPS = [
     "admin_action_tools",
     "django.contrib.admin",
@@ -62,35 +72,10 @@
     "health_check.db",  # stock Django health checkers
     "health_check.cache",
     "health_check.storage",
+    "django_celery_results",
     "controller.sentry",
 ]
 
-AUTHENTICATION_BACKENDS = (
-    # "django.contrib.auth.backends.ModelBackend",
-    "controller.sentry.auth.ControllerOIDCAuthenticationBackend",
-)
-
-OIDC_RP_CLIENT_ID = os.getenv("OIDC_RP_CLIENT_ID")
-OIDC_RP_CLIENT_SECRET = os.getenv("OIDC_RP_CLIENT_SECRET")
-# "<URL of the OIDC OP authorization endpoint>"
-OIDC_OP_AUTHORIZATION_ENDPOINT = os.getenv("OIDC_OP_AUTHORIZATION_ENDPOINT")
-# "<URL of the OIDC OP token endpoint>"
-OIDC_OP_TOKEN_ENDPOINT = os.getenv("OIDC_OP_TOKEN_ENDPOINT")
-# "<URL of the OIDC OP userinfo endpoint>"
-OIDC_OP_USER_ENDPOINT = os.getenv("OIDC_OP_USER_ENDPOINT")
-# "<URL path to redirect to after login>"
-LOGIN_REDIRECT_URL = os.getenv("LOGIN_REDIRECT_URL")
-# "<URL path to redirect to after logout>"
-LOGOUT_REDIRECT_URL = os.getenv("LOGOUT_REDIRECT_URL")
-
-OIDC_RP_SIGN_ALGO = os.getenv("OIDC_RP_SIGN_ALGO", "RS256")
-
-OIDC_OP_JWKS_ENDPOINT = os.getenv("OIDC_OP_JWKS_ENDPOINT")
-
-
-DEVELOPER_GROUP = os.getenv("DEVELOPER_GROUP", "Developer")
-
-
 MIDDLEWARE = [
     "django.middleware.security.SecurityMiddleware",
     "django.contrib.sessions.middleware.SessionMiddleware",
@@ -101,8 +86,8 @@
     "django.middleware.clickjacking.XFrameOptionsMiddleware",
 ]
 
-ROOT_URLCONF = "controller.urls"
 
+# template
 TEMPLATES = [
     {
         "BACKEND": "django.template.backends.django.DjangoTemplates",
@@ -120,7 +105,28 @@
     },
 ]
 
-WSGI_APPLICATION = "controller.wsgi.application"
+# Authentication
+AUTHENTICATION_BACKENDS = (
+    # "django.contrib.auth.backends.ModelBackend",
+    "controller.sentry.auth.ControllerOIDCAuthenticationBackend",
+)
+
+OIDC_RP_CLIENT_ID = os.getenv("OIDC_RP_CLIENT_ID")
+OIDC_RP_CLIENT_SECRET = os.getenv("OIDC_RP_CLIENT_SECRET")
+# "<URL of the OIDC OP authorization endpoint>"
+OIDC_OP_AUTHORIZATION_ENDPOINT = os.getenv("OIDC_OP_AUTHORIZATION_ENDPOINT")
+# "<URL of the OIDC OP token endpoint>"
+OIDC_OP_TOKEN_ENDPOINT = os.getenv("OIDC_OP_TOKEN_ENDPOINT")
+# "<URL of the OIDC OP userinfo endpoint>"
+OIDC_OP_USER_ENDPOINT = os.getenv("OIDC_OP_USER_ENDPOINT")
+# "<URL path to redirect to after login>"
+LOGIN_REDIRECT_URL = os.getenv("LOGIN_REDIRECT_URL")
+# "<URL path to redirect to after logout>"
+LOGOUT_REDIRECT_URL = os.getenv("LOGOUT_REDIRECT_URL")
+
+OIDC_RP_SIGN_ALGO = os.getenv("OIDC_RP_SIGN_ALGO", "RS256")
+
+OIDC_OP_JWKS_ENDPOINT = os.getenv("OIDC_OP_JWKS_ENDPOINT")
 
 
 # Database
@@ -172,7 +178,11 @@
 
 DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
 
+
+# Reuse db connection
 CONN_MAX_AGE = None
+
+# CACHE
 APP_CACHE_TIMEOUT = 0
 
 if not TESTING:
@@ -190,22 +200,56 @@
     }
 
 
+# App config
 DEFAULT_SAMPLE_RATE = float(os.getenv("DEFAULT_SAMPLE_RATE", "0.1"))
+
 DEFAULT_WSGI_IGNORE_PATHS = os.getenv("DEFAULT_WSGI_IGNORE_PATHS", "/health,/healthz,/health/,/healthz/").split(",")
 
 
 DEFAULT_CELERY_IGNORE_TASKS = []
 
-
 CACHE_META_INVALIDATION = {
     "SERVER_NAME": os.getenv("CACHE_META_SERVER_NAME", "localhost"),
     "SERVER_PORT": int(os.getenv("CACHE_META_SERVER_PORT", "8000")),
     "HTTP_ACCEPT": os.getenv("CACHE_META_HTTP_ACCEPT", "*/*"),
 }
 
-
 MAX_BUMP_TIME_SEC = int(os.getenv("MAX_BUMP_TIME_SEC", "0"))
 if MAX_BUMP_TIME_SEC == 0:
     MAX_BUMP_TIME_SEC = 30 * 60  # 30 minutes
 
+# CACHE KEY for panic
 PANIC_KEY = "PANIC"
+
+DEVELOPER_GROUP = os.getenv("DEVELOPER_GROUP", "Developer")
+
+APP_AUTO_PRUNE = os.getenv("APP_AUTO_PRUNE", "true").lower() == "true"
+APP_AUTO_PRUNE_MAX_AGE_DAY = int(os.getenv("APP_AUTO_PRUNE_MAX_AGE_DAY", "5"))
+
+
+# Celery
+BROKER_USER = quote(os.environ.get("CELERY_BROKER_USER", "rabbitmq"))
+BROKER_PASSWORD = quote(os.environ.get("CELERY_BROKER_PASSWORD", "rabbitmq"))
+BROKER_HOST = os.environ.get("CELERY_BROKER_HOST", "localhost")
+BROKER_PORT = os.environ.get("CELERY_BROKER_PORT", "5672")
+BROKER_VHOST = quote(os.environ.get("CELERY_BROKER_VHOST", "/"))
+
+
+CELERY_ACCEPT_CONTENT = ["json"]
+CELERY_ACKS_LATE = True
+CELERY_PREFETCH_MULTIPLIER = 1
+CELERY_RESULT_BACKEND = "django-db"
+CELERY_BROKER_URL = f"amqp://{BROKER_USER}:{BROKER_PASSWORD}@{BROKER_HOST}:{BROKER_PORT}/{BROKER_VHOST}"
+
+CELERY_BEAT_SCHEDULE = {
+    "close-window": {
+        "task": "controller.sentry.tasks.close_window",
+        "schedule": crontab(),
+    }
+}
+
+if APP_AUTO_PRUNE:
+    CELERY_BEAT_SCHEDULE["prune-inactive"] = {
+        "task": "controller.sentry.tasks.prune_inactive_app",
+        "schedule": crontab(minute="0", hour="*"),
+    }