Set up custom metrics and dogstatsd (#729)

* Add some metrics to the tasks

* lint

* linter doesn't like datadog documentation

* Add metric to sync panoramas command

This ensures that statsd won't interrupt normal operation

* add stats to join form

* add metric to index for testing

* update endpoint

* lint

---------

Co-authored-by: Will Nilges <willard.nilges@gmail.com>

Author: WillNilges

pyproject.toml

@@ -45,6 +45,8 @@ dependencies = [
     "matplotlib==3.9.*",
     "django-ipware==7.0.1",
     "django-admin-site-search==1.1.*",
+    "datadog==0.50.*",
+
 ]
 
 [project.optional-dependencies]

src/meshapi/management/commands/sync_panoramas.py

@@ -1,6 +1,7 @@
 from argparse import ArgumentParser
 from typing import Any
 
+from datadog import statsd
 from django.core.management.base import BaseCommand
 
 from meshapi.util.panoramas import sync_github_panoramas
@@ -13,6 +14,7 @@ def add_arguments(self, parser: ArgumentParser) -> None:
         pass
 
     def handle(self, *args: Any, **options: Any) -> None:
+        statsd.increment("meshdb.commands.sync_panoramas", tags=[])
         print("Syncing panoramas from GitHub...")
         panoramas_saved, warnings = sync_github_panoramas()
         print(f"Saved {panoramas_saved} panoramas. Got {len(warnings)} warnings.")

src/meshapi/tasks.py

@@ -2,6 +2,7 @@
 import os
 
 from celery.schedules import crontab
+from datadog import statsd
 from django.core import management
 from flags.state import disable_flag, enable_flag
 
@@ -32,8 +33,11 @@ def run_database_backup() -> None:
         management.call_command("dbbackup")
     except Exception as e:
         logging.exception(e)
+        statsd.increment("meshdb.tasks.run_database_backup", tags=["status:failure"])
         raise e
 
+    statsd.increment("meshdb.tasks.run_database_backup", tags=["status:success"])
+
 
 @celery_app.task
 @skip_if_flag_disabled("TASK_ENABLED_RESET_DEV_DATABASE")
@@ -54,8 +58,11 @@ def reset_dev_database() -> None:
         disable_flag("MAINTENANCE_MODE")
     except Exception as e:
         logging.exception(e)
+        statsd.increment("meshdb.tasks.reset_dev_database", tags=["status:failure"])
         raise e
 
+    statsd.increment("meshdb.tasks.reset_dev_database", tags=["status:success"])
+
 
 @celery_app.task
 @skip_if_flag_disabled("TASK_ENABLED_UPDATE_PANORAMAS")
@@ -68,8 +75,11 @@ def run_update_panoramas() -> None:
     except Exception as e:
         # Make sure the failure gets logged.
         logging.exception(e)
+        statsd.increment("meshdb.tasks.run_update_panoramas", tags=["status:failure"])
         raise e
 
+    statsd.increment("meshdb.tasks.run_update_panoramas", tags=["status:success"])
+
 
 @celery_app.task
 @skip_if_flag_disabled("TASK_ENABLED_SYNC_WITH_UISP")
@@ -82,8 +92,11 @@ def run_update_from_uisp() -> None:
     except Exception as e:
         # Make sure the failure gets logged.
         logging.exception(e)
+        statsd.increment("meshdb.tasks.run_update_from_uisp", tags=["status:failure"])
         raise e
 
+    statsd.increment("meshdb.tasks.run_update_from_uisp", tags=["status:success"])
+
 
 jitter_minutes = 0 if MESHDB_ENVIRONMENT == "prod2" else 2
 

src/meshapi/views/forms.py

@@ -6,6 +6,7 @@
 from json.decoder import JSONDecodeError
 from typing import Optional
 
+from datadog import statsd
 from django.db import IntegrityError, transaction
 from django.db.models import Q
 from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view, inline_serializer
@@ -100,6 +101,7 @@ class Meta:
 @permission_classes([permissions.AllowAny])
 @advisory_lock("join_form_lock")
 def join_form(request: Request) -> Response:
+    statsd.increment("meshdb.join_form.request", tags=[])
     request_json = json.loads(request.body)
     try:
         r = JoinFormRequest(**request_json)
@@ -126,7 +128,9 @@ def join_form(request: Request) -> Response:
             logging.exception("Captcha validation failed")
             return Response({"detail": "Captcha verification failed"}, status=status.HTTP_401_UNAUTHORIZED)
 
-    return process_join_form(r, request)
+    response = process_join_form(r, request)
+    statsd.increment("meshdb.join_form.response", tags=[f"status:{response.status_code}"])
+    return response
 
 
 def process_join_form(r: JoinFormRequest, request: Optional[Request] = None) -> Response:

src/meshdb/wsgi.py

@@ -9,8 +9,13 @@
 
 import os
 
+# Initialize dogstatsd
+from datadog import initialize
 from django.core.wsgi import get_wsgi_application
 
+initialize(statsd_host="datadog-agent.datadog.svc.cluster.local", statsd_port=8125)
+
+
 os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings")
 
 application = get_wsgi_application()

src/meshweb/views/index.py

@@ -1,9 +1,11 @@
+from datadog import statsd
 from django.conf import settings
 from django.http import HttpRequest, HttpResponse
 from django.template import loader
 
 
 def index(request: HttpRequest) -> HttpResponse:
+    statsd.increment("meshdb.views.index", tags=[])
     template = loader.get_template("meshweb/index.html")
     links = {
         "Member Tools": [