PB-1954: improve OTEL instrumentation and add local testing env

Author: hbollon

.env.local

@@ -1,4 +1,10 @@
 ALLOWED_DOMAINS=.*localhost,.*\.admin\.ch,.*\.bgdi\.ch
 
 # OTEL
-OTEL_SDK_DISABLED=true
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-qrcode
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.env.test

@@ -1,4 +1,10 @@
 ALLOWED_DOMAINS=some_random_domain,.*\.geo\.admin\.ch,http://localhost
 
 # OTEL
-OTEL_SDK_DISABLED=true
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-qrcode
+OTEL_PYTHON_EXCLUDED_URLS="checker"

Makefile

@@ -148,7 +148,7 @@ serve: clean_logs $(LOGS_DIR)
 
 .PHONY: gunicornserve
 gunicornserve: clean_logs $(LOGS_DIR)
-	OTEL_SDK_DISABLED=true SCRIPT_NAME=$(ROUTE_PREFIX) ENV_FILE=$(ENV_FILE) LOGS_DIR=$(LOGS_DIR) $(PYTHON) wsgi.py
+	SCRIPT_NAME=$(ROUTE_PREFIX) ENV_FILE=$(ENV_FILE) LOGS_DIR=$(LOGS_DIR) $(PYTHON) wsgi.py
 
 
 # Docker related functions.

Pipfile

@@ -10,14 +10,14 @@ gunicorn = "~=23.0.0"
 Flask = "~=3.0.3"
 Pillow = "~=10.4.0"
 qrcode = "~=7.4"
-logging-utilities = "~=4.4.1"
+logging-utilities = "~=5.3.0"
 python-dotenv = "~=1.0.1"
 
 # OpenTelemetry packages
 opentelemetry-sdk = "*"
 opentelemetry-exporter-otlp = "*"
 opentelemetry-instrumentation-flask = "*"
-opentelemetry-instrumentation-requests = "*"
+opentelemetry-instrumentation-logging = "*"
 
 [dev-packages]
 yapf = "*"

Pipfile.lock

@@ -1,25 +1,41 @@
-import os
+from os import getenv
 
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
     OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
 from opentelemetry.sdk.resources import Resource
 from opentelemetry.sdk.trace import TracerProvider
 from opentelemetry.sdk.trace.export import BatchSpanProcessor
 
 from app.helpers.utils import strtobool
 
 
-def setup_trace_provider(worker_pid):
-    trace.set_tracer_provider(TracerProvider(resource=Resource.create()))
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
 
-    # Since we created a new tracer, the default span processor is gone. We need to
-    # create a new one using the default OTEL env variables and ad it to the tracer.
-    span_processor = BatchSpanProcessor(
-        OTLPSpanExporter(
-            endpoint=os.getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
-            headers=os.getenv('OTEL_EXPORTER_OTLP_HEADERS'),
-            insecure=strtobool(os.getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+            )
         )
-    )
-    trace.get_tracer_provider().add_span_processor(span_processor)
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

app/helpers/otel.py

@@ -0,0 +1,14 @@
+version: "2"
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    volumes:
+      - ./otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"

docker-compose-otel.yml

@@ -0,0 +1,21 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+processors:
+  batch:
+
+exporters:
+  debug:
+    verbosity: detailed
+  zipkin:
+    endpoint: http://zipkin:9411/api/v2/spans
+
+service:
+  pipelines:
+    traces:
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [debug, zipkin]

otel-local-config.yaml

@@ -8,7 +8,7 @@
     load the ssl module.
 """
 
-# pylint: disable=wrong-import-position,wrong-import-order
+# pylint: disable=wrong-import-position,wrong-import-order,ungrouped-imports
 import gevent.monkey
 
 gevent.monkey.patch_all()
@@ -19,7 +19,9 @@
 # e.g. the flask instrumentation has no effect. See:
 # https://github.com/open-telemetry/opentelemetry.io/blob/main/content/en/docs/zero-code/python/troubleshooting.md#use-programmatic-auto-instrumentation
 
-from opentelemetry.instrumentation.auto_instrumentation import initialize
+from app.helpers.otel import initialize
+from app.helpers.otel import initialize_flask
+from app.helpers.otel import setup_trace_provider
 
 initialize()
 
@@ -28,18 +30,17 @@
 from gunicorn.app.base import BaseApplication
 
 from app.app import app as application
-from app.helpers import otel
 from app.helpers.utils import get_logging_cfg
-from app.helpers.utils import strtobool
 from app.settings import GUNICORN_KEEPALIVE
 
+initialize_flask(application)
+
 
 def post_fork(server, worker):
     server.log.info("Worker spawned (pid: %s)", worker.pid)
 
     # Setup OTEL providers for this worker
-    if not strtobool(os.getenv("OTEL_SDK_DISABLED", "false")):
-        otel.setup_trace_provider(worker.pid)
+    setup_trace_provider()
 
 
 class StandaloneApplication(BaseApplication):  # pylint: disable=abstract-method