Adding OTEL metric shipping to mitxonline (#2649)

Author: Ardiea

app.json

@@ -549,6 +549,30 @@
       "description": "The number of hours until an access token for the Open edX API expires",
       "required": false
     },
+    "OPENTELEMETRY_ENABLED": {
+       "description": "Enable collection and shipment of opentelemetry data",
+       "required": false
+    },
+    "OPENTELEMETRY_ENDPOINT": {
+      "description": "Endpoint for opentelemetry",
+      "required": false
+    },
+    "OPENTELEMETRY_EXPORT_TIMEOUT_MS": {
+      "description": "Timeout for opentelemetry export",
+      "required": false
+    },
+    "OPENTELEMETRY_INSECURE": {
+      "description": "Use insecure connection to opentelemetry",
+      "required": false
+    },
+    "OPENTELEMETRY_SERVICE_NAME": {
+      "description": "The name of the service to report to opentelemetry",
+      "required": false
+    },
+    "OPENTELEMETRY_TRACES_BATCH_SIZE": {
+      "description": "Batch size for traces",
+      "required": false
+    },
     "OPEN_EXCHANGE_RATES_APP_ID": {
       "description": "open exchange app id for fetching currency exchange rate",
       "required": false

main/apps.py

@@ -16,3 +16,7 @@ def ready(self):
 
         envs.validate()
         configure()
+
+        from main.telemetry import configure_opentelemetry
+
+        configure_opentelemetry()

main/settings.py

@@ -1351,3 +1351,35 @@
     "localhost",
     "mitxonline.odl.local",
 ]
+
+
+OPENTELEMETRY_ENABLED = get_bool(
+    name="OPENTELEMETRY_ENABLED",
+    default=False,
+    description="Enable collection and shipment of opentelemetry data",
+)
+OPENTELEMETRY_SERVICE_NAME = get_string(
+    name="OPENTELEMETRY_SERVICE_NAME",
+    default="mitxonline",
+    description="The name of the service to report to opentelemetry",
+)
+OPENTELEMETRY_INSECURE = get_bool(
+    name="OPENTELEMETRY_INSECURE",
+    default=True,
+    description="Use insecure connection to opentelemetry",
+)
+OPENTELEMETRY_ENDPOINT = get_string(
+    name="OPENTELEMETRY_ENDPOINT",
+    default=None,
+    description="Endpoint for opentelemetry",
+)
+OPENTELEMETRY_TRACES_BATCH_SIZE = get_int(
+    name="OPENTELEMETRY_TRACES_BATCH_SIZE",
+    default=512,
+    description="Batch size for traces",
+)
+OPENTELEMETRY_EXPORT_TIMEOUT_MS = get_int(
+    name="OPENTELEMETRY_EXPORT_TIMEOUT_MS",
+    default=5000,
+    description="Timeout for opentelemetry export",
+)

main/telemetry.py

@@ -0,0 +1,91 @@
+"""OpenTelemetry initialization and configuration for MITxOnline."""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from django.conf import settings
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
+from opentelemetry.instrumentation.celery import CeleryInstrumentor
+from opentelemetry.instrumentation.django import DjangoInstrumentor
+from opentelemetry.instrumentation.psycopg import PsycopgInstrumentor
+from opentelemetry.instrumentation.redis import RedisInstrumentor
+from opentelemetry.instrumentation.requests import RequestsInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
+
+log = logging.getLogger(__name__)
+
+
+def configure_opentelemetry() -> Optional[TracerProvider]:
+    """
+    Configure OpenTelemetry with appropriate instrumentations and exporters.
+    Returns the tracer provider if configured, None otherwise.
+    """
+    if not getattr(settings, "OPENTELEMETRY_ENABLED", False):
+        log.info("OpenTelemetry is disabled")
+        return None
+
+    log.info("Initializing OpenTelemetry")
+
+    # Create a resource with service info
+    resource = Resource.create(
+        {
+            "service.name": getattr(
+                settings, "OPENTELEMETRY_SERVICE_NAME", "mitxonline"
+            ),
+            "service.version": getattr(settings, "VERSION", "unknown"),
+            "deployment.environment": settings.ENVIRONMENT,
+        }
+    )
+
+    # Configure the tracer provider
+    tracer_provider = TracerProvider(resource=resource)
+    trace.set_tracer_provider(tracer_provider)
+
+    # Add console exporter for development/testing
+    if settings.DEBUG:
+        log.info("Adding console exporter for OpenTelemetry")
+        console_exporter = ConsoleSpanExporter()
+        tracer_provider.add_span_processor(BatchSpanProcessor(console_exporter))
+
+    # Add OTLP exporter if configured
+    otlp_endpoint = getattr(settings, "OPENTELEMETRY_ENDPOINT", None)
+    if otlp_endpoint:
+        log.info("Configuring OTLP exporter to endpoint: %s", otlp_endpoint)
+
+        headers = {}
+
+        try:
+            otlp_exporter = OTLPSpanExporter(
+                endpoint=otlp_endpoint,
+                headers=headers,
+                insecure=getattr(settings, "OPENTELEMETRY_INSECURE", True),
+            )
+
+            tracer_provider.add_span_processor(
+                BatchSpanProcessor(
+                    otlp_exporter,
+                    max_export_batch_size=getattr(
+                        settings, "OPENTELEMETRY_BATCH_SIZE", 512
+                    ),
+                    schedule_delay_millis=getattr(
+                        settings, "OPENTELEMETRY_EXPORT_TIMEOUT_MS", 5000
+                    ),
+                )
+            )
+        except Exception:
+            log.exception("Failed to configure OTLP exporter")
+
+    # Initialize instrumentations
+    DjangoInstrumentor().instrument()
+    PsycopgInstrumentor().instrument()
+    RedisInstrumentor().instrument()
+    CeleryInstrumentor().instrument()
+    RequestsInstrumentor().instrument()
+
+    log.info("OpenTelemetry initialized successfully")
+    return tracer_provider