Merge pull request #70 from geoadmin/PB-1954-add-otel-instrumentation

PB-1954: implement otel instrumentation

Author: hbollon

.env.local

@@ -1 +1,10 @@
 ALLOWED_DOMAINS=.*localhost,.*\.admin\.ch,.*\.bgdi\.ch
+
+# OTEL
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-qrcode
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.env.test

@@ -1 +1,10 @@
-ALLOWED_DOMAINS=some_random_domain,.*\.geo\.admin\.ch,http://localhost
+ALLOWED_DOMAINS=some_random_domain,.*\.geo\.admin\.ch,http://localhost
+
+# OTEL
+OTEL_SDK_DISABLED=false
+OTEL_ENABLE_FLASK=true
+OTEL_ENABLE_LOGGING=true
+OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317
+OTEL_EXPORTER_OTLP_INSECURE=true
+OTEL_RESOURCE_ATTRIBUTES=service.name=service-qrcode
+OTEL_PYTHON_EXCLUDED_URLS="checker"

.pylintrc

@@ -36,10 +36,6 @@ load-plugins=pylint_flask
 # Pickle collected data for later comparisons.
 persistent=yes
 
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages.
-suggestion-mode=yes
-
 # Allow loading of arbitrary C extensions. Extensions are imported into the
 # active Python interpreter and may run arbitrary code.
 unsafe-load-any-extension=no
@@ -62,6 +58,7 @@ confidence=
 # --disable=W".
 disable=missing-docstring,
     missing-module-docstring,
+    unused-argument,
     useless-object-inheritance,
 
 

Dockerfile

@@ -1,5 +1,5 @@
 FROM python:3.12-slim-bullseye
-ENV HTTP_PORT 8080
+ENV HTTP_PORT=8080
 RUN groupadd -r geoadmin && useradd -r -s /bin/false -g geoadmin geoadmin
 
 

Makefile

@@ -67,6 +67,7 @@ help:
 	@echo -e " \033[1mSetup TARGETS\033[0m "
 	@echo "- setup              Create the python virtual environment with developper tools and activate it"
 	@echo "- ci                 Create the python virtual environment and install requirements based on the Pipfile.lock"
+	@echo "- otelrequirements   Get a list of available otel instrumentation libraries to add to the pipfile of this project"
 	@echo -e " \033[1mFORMATING, LINTING AND TESTING TOOLS TARGETS\033[0m "
 	@echo "- format             Format the python source code"
 	@echo "- lint               Lint the python source code"
@@ -99,6 +100,9 @@ ci: $(REQUIREMENTS)
 	# Create virtual env with all packages for development using the Pipfile.lock
 	pipenv sync --dev
 
+.PHONY: otelrequirements
+otelrequirements:
+	edot-bootstrap --action=requirements
 
 # linting target, calls upon yapf to make sure your code is easier to read and respects some conventions.
 

Pipfile

@@ -10,9 +10,15 @@ gunicorn = "~=23.0.0"
 Flask = "~=3.0.3"
 Pillow = "~=10.4.0"
 qrcode = "~=7.4"
-logging-utilities = "~=4.4.1"
+logging-utilities = "~=5.3.0"
 python-dotenv = "~=1.0.1"
 
+# OpenTelemetry packages
+opentelemetry-sdk = "*"
+opentelemetry-exporter-otlp = "*"
+opentelemetry-instrumentation-flask = "*"
+opentelemetry-instrumentation-logging = "*"
+
 [dev-packages]
 yapf = "*"
 nose2 = "*"

Pipfile.lock

@@ -0,0 +1,41 @@
+from os import getenv
+
+from opentelemetry import trace
+from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import \
+    OTLPSpanExporter
+from opentelemetry.instrumentation.flask import FlaskInstrumentor
+from opentelemetry.instrumentation.logging import LoggingInstrumentor
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+from app.helpers.utils import strtobool
+
+
+def initialize() -> None:
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_LOGGING", "false")):
+            LoggingInstrumentor().instrument()
+
+
+def initialize_flask(app):
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        if strtobool(getenv("OTEL_ENABLE_FLASK", "false")):
+            FlaskInstrumentor().instrument_app(app)
+
+
+def setup_trace_provider():
+    if not strtobool(getenv("OTEL_SDK_DISABLED", "false")):
+        # Since we created a new tracer, the default span processor is gone. We need to
+        # create a new one using the default OTEL env variables and ad it to the tracer.
+        span_processor = BatchSpanProcessor(
+            OTLPSpanExporter(
+                endpoint=getenv('OTEL_EXPORTER_OTLP_ENDPOINT', "http://localhost:4317"),
+                headers=getenv('OTEL_EXPORTER_OTLP_HEADERS'),
+                insecure=strtobool(getenv('OTEL_EXPORTER_OTLP_INSECURE', "false"))
+            )
+        )
+
+        provider = TracerProvider(resource=Resource.create())
+        provider.add_span_processor(span_processor)
+        trace.set_tracer_provider(provider)

app/helpers/otel.py

@@ -30,3 +30,18 @@ def get_logging_cfg():
 def init_logging():
     config = get_logging_cfg()
     logging.config.dictConfig(config)
+
+
+def strtobool(val):
+    """Convert a string representation of truth to True or False.
+
+    True values are 'y', 'yes', 't', 'true', 'on', and '1';
+    False values are 'n', 'no', 'f', 'false', 'off', and '0'.
+    Raises ValueError if 'val' is anything else.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    raise ValueError(f"invalid truth value: {val}")

app/helpers/utils.py

@@ -0,0 +1,14 @@
+version: "2"
+services:
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    command: --config otel-local-config.yaml
+    volumes:
+      - ./otel-local-config.yaml:/otel-local-config.yaml
+    ports:
+      - "4317:4317"
+
+  zipkin:
+    image: openzipkin/zipkin:latest
+    ports:
+      - "9411:9411"