WIP: Replace StatsD instrumentation with OpenTelemetry over OTLP

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-# This file was automatically copied from notifications-utils@111.0.0
+# This file was automatically copied from notifications-utils@112.0.0
 
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks

app/clients/sms/__init__.py

@@ -45,48 +45,32 @@ def __init__(self, current_app, statsd_client):
                     **adapter.poolmanager.connection_pool_kw,
                 }
 
-    def record_outcome(self, success):
-        if success:
-            self.current_app.logger.info(
-                "Provider request for %s succeeded",
-                self.name,
-                extra={
-                    "provider_name": self.name,
-                },
-            )
-            self.statsd_client.incr(f"clients.{self.name}.success")
-        else:
-            self.statsd_client.incr(f"clients.{self.name}.error")
-            self.current_app.logger.warning(
-                "Provider request for %s failed",
-                self.name,
-                extra={
-                    "provider_name": self.name,
-                },
-            )
-
     def send_sms(self, to, content, reference, international, sender):
         start_time = monotonic()
+        status = "succeeded"
 
         try:
             response = self.try_send_sms(to, content, reference, international, sender)
-            self.record_outcome(True)
+            self.statsd_client.incr(f"clients.{self.name}.success")
         except SmsClientResponseException as e:
-            self.record_outcome(False)
+            status = "failed"
+            self.statsd_client.incr(f"clients.{self.name}.error")
             raise e
         finally:
             elapsed_time = monotonic() - start_time
             self.statsd_client.timing(f"clients.{self.name}.request-time", elapsed_time)
             self.current_app.logger.info(
-                "%s request for %s finished in %.4g",
+                "Provider %s request for %s %s in %.4g",
                 self.name,
                 reference,
+                status,
                 elapsed_time,
                 extra={
                     "provider_name": self.name,
                     # for SMS, we happen to use the notification id as the "reference" for the provider
                     "notification_id": reference,
                     "duration": elapsed_time,
+                    "status": status,
                 },
             )
 

entrypoint.sh

@@ -5,7 +5,7 @@ export PROMETHEUS_MULTIPROC_DIR="/tmp"
 CONCURRENCY=${CONCURRENCY:-4}
 
 # Define a common command prefix
-WORKER_CMD="celery --quiet -A run_celery.notify_celery worker --logfile=/dev/null --concurrency=$CONCURRENCY"
+WORKER_CMD="opentelemetry-instrument celery --quiet -A run_celery.notify_celery worker --logfile=/dev/null --concurrency=$CONCURRENCY"
 COMMON_CMD="$WORKER_CMD -Q"
 
 case "$1" in

requirements.in

@@ -25,8 +25,10 @@ psutil>=6.0.0,<7.0.0
 notifications-python-client==10.0.1
 
 # Run `make bump-utils` to update to the latest version
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@111.0.0
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@jar-otel-semconv
 
 git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72
 
 sentry-sdk[flask,celery,sqlalchemy]~=1.45
+
+opentelemetry-distro[otlp]

requirements.txt

@@ -94,10 +94,16 @@ fqdn==1.5.1
     # via jsonschema
 gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72
     # via -r requirements.in
+googleapis-common-protos==1.73.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 govuk-bank-holidays==0.19
     # via notifications-utils
 greenlet==3.3.2
     # via eventlet
+grpcio==1.78.0
+    # via opentelemetry-exporter-otlp-proto-grpc
 gunicorn==25.1.0
     # via
     #   -r requirements.in
@@ -152,23 +158,63 @@ mistune==0.8.4
     # via notifications-utils
 notifications-python-client==10.0.1
     # via -r requirements.in
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@900cb380b9f92b23886586207f44c99893d2b56b
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@70e245b5bdd9234ddd8f1f1adbee3401ed1909cf
     # via -r requirements.in
 opentelemetry-api==1.40.0
-    # via notifications-utils
+    # via
+    #   notifications-utils
+    #   opentelemetry-distro
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-instrumentation
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-distro==0.61b0
+    # via -r requirements.in
+opentelemetry-exporter-otlp==1.40.0
+    # via opentelemetry-distro
+opentelemetry-exporter-otlp-proto-common==1.40.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.40.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.40.0
+    # via opentelemetry-exporter-otlp
+opentelemetry-instrumentation==0.61b0
+    # via opentelemetry-distro
+opentelemetry-proto==1.40.0
+    # via
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.40.0
+    # via
+    #   opentelemetry-distro
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-semantic-conventions==0.61b0
+    # via
+    #   opentelemetry-instrumentation
+    #   opentelemetry-sdk
 ordered-set==4.1.0
     # via notifications-utils
 packaging==26.0
     # via
     #   gunicorn
     #   kombu
     #   marshmallow
+    #   opentelemetry-instrumentation
 phonenumbers==9.0.25
     # via notifications-utils
 prometheus-client==0.24.1
     # via gds-metrics
 prompt-toolkit==3.0.52
     # via click-repl
+protobuf==6.33.5
+    # via
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 psutil==6.1.1
     # via -r requirements.in
 psycopg2-binary==2.9.10
@@ -206,6 +252,7 @@ requests==2.32.5
     #   govuk-bank-holidays
     #   notifications-python-client
     #   notifications-utils
+    #   opentelemetry-exporter-otlp-proto-http
 rfc3339-validator==0.1.4
     # via jsonschema
 rfc3987==1.3.8
@@ -238,7 +285,12 @@ statsd==4.0.1
 typing-extensions==4.15.0
     # via
     #   alembic
+    #   grpcio
     #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
     #   sqlalchemy
 tzdata==2025.3
     # via
@@ -268,5 +320,7 @@ werkzeug==3.1.6
     # via flask
 wheel==0.44.0
     # via click-datetime
+wrapt==1.17.3
+    # via opentelemetry-instrumentation
 zipp==3.23.0
     # via importlib-metadata

requirements_for_test.txt

@@ -143,6 +143,11 @@ freezegun==1.5.5
     # via -r requirements_for_test_common.in
 gds-metrics @ git+https://github.com/alphagov/gds_metrics_python.git@6f1840a57b6fb1ee40b7e84f2f18ec229de8aa72
     # via -r requirements.txt
+googleapis-common-protos==1.73.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
 govuk-bank-holidays==0.19
     # via
     #   -r requirements.txt
@@ -151,6 +156,10 @@ greenlet==3.3.2
     # via
     #   -r requirements.txt
     #   eventlet
+grpcio==1.78.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
 gunicorn==25.1.0
     # via
     #   -r requirements.txt
@@ -228,12 +237,56 @@ mypy-extensions==1.1.0
     # via mypy
 notifications-python-client==10.0.1
     # via -r requirements.txt
-notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@900cb380b9f92b23886586207f44c99893d2b56b
+notifications-utils @ git+https://github.com/alphagov/notifications-utils.git@70e245b5bdd9234ddd8f1f1adbee3401ed1909cf
     # via -r requirements.txt
 opentelemetry-api==1.40.0
     # via
     #   -r requirements.txt
     #   notifications-utils
+    #   opentelemetry-distro
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-instrumentation
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
+opentelemetry-distro==0.61b0
+    # via -r requirements.txt
+opentelemetry-exporter-otlp==1.40.0
+    # via -r requirements.txt
+opentelemetry-exporter-otlp-proto-common==1.40.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-exporter-otlp-proto-grpc==1.40.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-exporter-otlp
+opentelemetry-exporter-otlp-proto-http==1.40.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-exporter-otlp
+opentelemetry-instrumentation==0.61b0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-distro
+opentelemetry-proto==1.40.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-exporter-otlp-proto-common
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-sdk==1.40.0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-distro
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+opentelemetry-semantic-conventions==0.61b0
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-instrumentation
+    #   opentelemetry-sdk
 ordered-set==4.1.0
     # via
     #   -r requirements.txt
@@ -244,6 +297,7 @@ packaging==26.0
     #   gunicorn
     #   kombu
     #   marshmallow
+    #   opentelemetry-instrumentation
     #   pytest
 pathspec==1.0.4
     # via mypy
@@ -261,6 +315,11 @@ prompt-toolkit==3.0.52
     # via
     #   -r requirements.txt
     #   click-repl
+protobuf==6.33.5
+    # via
+    #   -r requirements.txt
+    #   googleapis-common-protos
+    #   opentelemetry-proto
 psutil==6.1.1
     # via -r requirements.txt
 psycopg2-binary==2.9.10
@@ -334,6 +393,7 @@ requests==2.32.5
     #   moto
     #   notifications-python-client
     #   notifications-utils
+    #   opentelemetry-exporter-otlp-proto-http
     #   requests-mock
     #   responses
 requests-mock==1.12.1
@@ -399,8 +459,13 @@ typing-extensions==4.15.0
     #   -r requirements.txt
     #   alembic
     #   beautifulsoup4
+    #   grpcio
     #   mypy
     #   opentelemetry-api
+    #   opentelemetry-exporter-otlp-proto-grpc
+    #   opentelemetry-exporter-otlp-proto-http
+    #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
     #   sqlalchemy
 tzdata==2025.3
     # via
@@ -443,6 +508,10 @@ wheel==0.44.0
     # via
     #   -r requirements.txt
     #   click-datetime
+wrapt==1.17.3
+    # via
+    #   -r requirements.txt
+    #   opentelemetry-instrumentation
 xmltodict==1.0.4
     # via moto
 zipp==3.23.0

requirements_for_test_common.in

@@ -1,4 +1,4 @@
-# This file was automatically copied from notifications-utils@111.0.0
+# This file was automatically copied from notifications-utils@112.0.0
 
 beautifulsoup4
 pytest

ruff.toml

@@ -1,4 +1,4 @@
-# This file was automatically copied from notifications-utils@111.0.0
+# This file was automatically copied from notifications-utils@112.0.0
 
 extend-exclude = [
     "migrations/versions/",