Otel spike

Author: Ben Corlett

app/__init__.py

@@ -135,6 +135,8 @@
 from app.notify_client.upload_api_client import upload_api_client  # noqa
 from app.notify_client.user_api_client import user_api_client  # noqa
 from app.notify_session import NotifyAdminSessionInterface
+from app.otel.metrics import otel_metrics
+from app.otel.traces import otel_traces
 from app.s3_client.logo_client import logo_client
 from app.template_previews import template_preview_client  # noqa
 from app.url_converters import (
@@ -179,6 +181,9 @@ def create_app(application):
 
     init_app(application)
 
+    otel_metrics.init_app(application)
+    otel_traces.init_app(application)
+
     if "extensions" not in application.jinja_options:
         application.jinja_options["extensions"] = []
 

app/config.py

@@ -9,6 +9,10 @@ class Config:
     DANGEROUS_SALT = os.environ.get("DANGEROUS_SALT")
     ZENDESK_API_KEY = os.environ.get("ZENDESK_API_KEY")
 
+    OTEL_EXPORT_TYPE = os.getenv("OTEL_EXPORT_TYPE", "otlp")
+    OTEL_COLLECTOR_ENDPOINT = os.getenv("OTEL_COLLECTOR_ENDPOINT", "localhost:4317")
+    OTEL_INSTRUMENTATIONS = os.getenv("OTEL_INSTRUMENTATIONS", "wsgi,celery,flask,redis,sqlalchemy,requests")
+
     # if we're not on cloudfoundry, we can get to this app from localhost. but on cloudfoundry its different
     ADMIN_BASE_URL = os.environ.get("ADMIN_BASE_URL", "http://localhost:6012")
 
@@ -118,6 +122,7 @@ class Development(Config):
     S3_BUCKET_REPORT_REQUESTS_DOWNLOAD = "development-report-requests-download"
 
     LOGO_CDN_DOMAIN = "static-logos.notify.tools"
+    OTEL_EXPORT_TYPE = os.getenv("OTEL_EXPORT_TYPE", "none")
 
     ADMIN_CLIENT_SECRET = "dev-notify-secret-key"
     DANGEROUS_SALT = "dev-notify-salt"
@@ -155,6 +160,7 @@ class Test(Development):
 
     ASSET_DOMAIN = "static.example.com"
     ASSET_PATH = "https://static.example.com/"
+    OTEL_EXPORT_TYPE = os.getenv("OTEL_EXPORT_TYPE", "none")
 
 
 class CloudFoundryConfig(Config):

app/main/views/templates.py

@@ -22,6 +22,9 @@
 from notifications_utils.pdf import pdf_page_count
 from notifications_utils.s3 import s3download
 from notifications_utils.template import Template
+from opentelemetry import trace
+from opentelemetry.baggage import set_baggage
+from opentelemetry.context import attach, detach
 from pypdf.errors import PdfReadError
 from requests import RequestException
 
@@ -719,69 +722,80 @@ def abort_for_unauthorised_bilingual_letters_or_invalid_options(language: str |
 def edit_service_template(service_id, template_id, language=None):
     template = current_service.get_template_with_user_permission_or_403(template_id, current_user)
 
-    if template.template_type not in current_service.available_template_types:
-        return redirect(
-            url_for(
-                ".action_blocked",
-                service_id=service_id,
-                notification_type=template.template_type,
-                return_to="view_template",
-                template_id=template.id,
-            )
-        )
+    ctx = set_baggage("template_id", str(template_id))
+    token = attach(ctx)
 
-    abort_for_unauthorised_bilingual_letters_or_invalid_options(language, template)
+    with trace.get_tracer(__name__).start_as_current_span("edit_service_template") as span:
+        try:
+            if template.template_type not in current_service.available_template_types:
+                return redirect(
+                    url_for(
+                        ".action_blocked",
+                        service_id=service_id,
+                        notification_type=template.template_type,
+                        return_to="view_template",
+                        template_id=template.id,
+                    )
+                )
 
-    form = get_template_form(template.template_type, language=language)(**template._template)
+            abort_for_unauthorised_bilingual_letters_or_invalid_options(language, template)
 
-    if form.validate_on_submit():
-        new_template = get_template(
-            template._template | form.new_template_data,
-            current_service,
-        )
-        template_change = template.compare_to(new_template)
+            form = get_template_form(template.template_type, language=language)(**template._template)
 
-        if template_change.placeholders_added and not request.form.get("confirm") and current_service.api_keys:
-            return render_template(
-                "views/templates/breaking-change.html",
-                template_change=template_change,
-                new_template=new_template,
-                form=form,
-            )
-        try:
-            service_api_client.update_service_template(
-                service_id=service_id,
-                template_id=template_id,
-                **form.new_template_data,
-            )
-        except HTTPError as e:
-            if e.status_code == 400:
-                if "content" in e.message and any("character count greater than" in x for x in e.message["content"]):
-                    form.template_content.errors.extend(e.message["content"])
-                elif "content" in e.message and any(x == QR_CODE_TOO_LONG for x in e.message["content"]):
-                    form.template_content.errors.append(
-                        "Cannot create a usable QR code - the link you entered is too long"
+            if form.validate_on_submit():
+                new_template = get_template(
+                    template._template | form.new_template_data,
+                    current_service,
+                )
+                template_change = template.compare_to(new_template)
+
+                span.add_event("This is an example span event")
+
+                if template_change.placeholders_added and not request.form.get("confirm") and current_service.api_keys:
+                    return render_template(
+                        "views/templates/breaking-change.html",
+                        template_change=template_change,
+                        new_template=new_template,
+                        form=form,
                     )
+                try:
+                    service_api_client.update_service_template(
+                        service_id=service_id,
+                        template_id=template_id,
+                        **form.new_template_data,
+                    )
+                except HTTPError as e:
+                    if e.status_code == 400:
+                        if "content" in e.message and any(
+                            "character count greater than" in x for x in e.message["content"]
+                        ):
+                            form.template_content.errors.extend(e.message["content"])
+                        elif "content" in e.message and any(x == QR_CODE_TOO_LONG for x in e.message["content"]):
+                            form.template_content.errors.append(
+                                "Cannot create a usable QR code - the link you entered is too long"
+                            )
+                        else:
+                            raise e
+                    else:
+                        raise e
                 else:
-                    raise e
-            else:
-                raise e
-        else:
-            editing_english_content_in_bilingual_letter = (
-                template.template_type == "letter" and template.welsh_page_count and language != "welsh"
-            )
-            return redirect(
-                url_for(
-                    "main.view_template",
-                    service_id=service_id,
-                    template_id=template_id,
-                    **(
-                        {"_anchor": "first-page-of-english-in-bilingual-letter"}
-                        if editing_english_content_in_bilingual_letter
-                        else {}
-                    ),
-                )
-            )
+                    editing_english_content_in_bilingual_letter = (
+                        template.template_type == "letter" and template.welsh_page_count and language != "welsh"
+                    )
+                    return redirect(
+                        url_for(
+                            "main.view_template",
+                            service_id=service_id,
+                            template_id=template_id,
+                            **(
+                                {"_anchor": "first-page-of-english-in-bilingual-letter"}
+                                if editing_english_content_in_bilingual_letter
+                                else {}
+                            ),
+                        )
+                    )
+        finally:
+            detach(token)
 
     return render_template(
         f"views/edit-{template.template_type}-template.html",

app/otel/decorators.py

@@ -0,0 +1,70 @@
+import functools
+import time
+
+from app.otel.metrics import otel_metrics
+
+
+def otel(counter_name=None, histogram_name=None, attributes=None):
+    if attributes is None:
+        attributes = {}
+
+    def time_function(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            start_time = time.monotonic()
+            c_name = counter_name or func.__name__
+            h_name = histogram_name or f"{func.__name__}_time"
+
+            # Create counter if it doesn't exist
+            if not hasattr(otel_metrics, c_name):
+                setattr(
+                    otel_metrics,
+                    c_name,
+                    otel_metrics.meter.create_counter(c_name, description=f"Calls to the {func.__name__} task"),
+                )
+            counter = getattr(otel_metrics, c_name)
+
+            # Create histogram if it doesn't exist
+            if not hasattr(otel_metrics, h_name):
+                setattr(
+                    otel_metrics,
+                    h_name,
+                    otel_metrics.meter.create_histogram(
+                        h_name,
+                        description=f"time taken to execute {func.__name__} function",
+                        explicit_bucket_boundaries_advisory=getattr(otel_metrics, "default_histogram_bucket", None),
+                    ),
+                )
+            histogram = getattr(otel_metrics, h_name)
+
+            try:
+                result = func(*args, **kwargs)
+                elapsed_time = time.monotonic() - start_time
+
+                counter.add(
+                    amount=1,
+                    attributes={**attributes, "function_name": func.__name__, "status": "success"},
+                )
+
+                histogram.record(
+                    amount=elapsed_time,
+                    attributes={**attributes, "function_name": func.__name__, "status": "success"},
+                )
+
+            except Exception as e:
+                elapsed_time = time.monotonic() - start_time
+                counter.add(
+                    amount=1,
+                    attributes={**attributes, "function_name": func.__name__, "status": "error"},
+                )
+                histogram.record(
+                    amount=elapsed_time,
+                    attributes={**attributes, "function_name": func.__name__, "status": "error"},
+                )
+                raise e
+            else:
+                return result
+
+        return wrapper
+
+    return time_function

app/otel/metrics.py

@@ -0,0 +1,69 @@
+from opentelemetry import metrics
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import (
+    ConsoleMetricExporter,
+    PeriodicExportingMetricReader,
+)
+from opentelemetry.sdk.resources import Resource
+
+
+class Metrics:
+    def __init__(self):
+        self.meter = None
+        self.default_histogram_bucket = [
+            0.005,
+            0.01,
+            0.025,
+            0.05,
+            0.075,
+            0.1,
+            0.25,
+            0.5,
+            0.75,
+            1.0,
+            2.5,
+            5.0,
+            7.5,
+            10.0,
+            float("inf"),
+        ]
+
+    def init_app(self, app):
+        export_mode = app.config.get("OTEL_EXPORT_TYPE", "none").lower()
+        metric_readers = []
+
+        if export_mode == "console":
+            app.logger.info("OpenTelemetry metrics will be exported to console")
+            metric_readers.append(PeriodicExportingMetricReader(ConsoleMetricExporter()))
+        elif export_mode == "otlp":
+            endpoint = app.config.get("OTEL_COLLECTOR_ENDPOINT", "localhost:4317")
+            app.logger.info("OpenTelemetry metrics will be exported to OTLP collector at %s", endpoint)
+            otlp_exporter = OTLPMetricExporter(endpoint=endpoint, insecure=True)
+            # Metrics will be exported every 60 seconds with a 30 seconds timeout by default.
+            # The following environments variables can be used to change this:
+            # OTEL_METRIC_EXPORT_INTERVAL
+            # OTEL_METRIC_EXPORT_TIMEOUT
+            metric_readers.append(PeriodicExportingMetricReader(otlp_exporter))
+
+        resource = Resource.create({"service.name": "notifications-api"})
+        provider = MeterProvider(metric_readers=metric_readers, resource=resource)
+        metrics.set_meter_provider(provider)
+        self.meter = metrics.get_meter(__name__)
+
+        self.create_counters()
+        self.create_histograms()
+        self.create_gauges()
+
+    def create_counters(self):
+        pass
+
+    def create_histograms(self):
+        pass
+
+    def create_gauges(self):
+        pass
+
+
+# Initialize the metrics instance singleton
+otel_metrics = Metrics()