[Python/EC2] Add pipeline exporter for custom metrics testing (#473)

Author: Miqueasher

sample-apps/python/django_frontend_service/frontend_service_app/views.py

@@ -13,15 +13,47 @@
 from django.http import HttpResponse, JsonResponse
 from opentelemetry import trace, metrics
 from opentelemetry.trace.span import format_trace_id
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter
+from opentelemetry.semconv.resource import ResourceAttributes
 
 logger = logging.getLogger(__name__)
 
+# Custom export pipeline - runs alongside existing CWAgent & ADOT setup
+pipeline_resource = Resource.create({
+    "service.name": os.environ['SERVICE_NAME'],
+    "deployment.environment.name": os.environ['DEPLOYMENT_ENVIRONMENT_NAME']
+})
+
+pipeline_metric_exporter = OTLPMetricExporter(
+    endpoint="localhost:4317"
+)
+
+pipeline_metric_reader = PeriodicExportingMetricReader(
+    exporter=pipeline_metric_exporter,
+    export_interval_millis=1000
+)
+
+pipeline_meter_provider = MeterProvider(
+    resource=pipeline_resource,
+    metric_readers=[pipeline_metric_reader]
+)
+
+pipeline_meter = pipeline_meter_provider.get_meter("myMeter")
+
+
 #python equivalent of Meter meter = GlobalOpenTelemetry.getMeter("myMeter"); for custom metrics
 meter = metrics.get_meter("myMeter")
 agent_based_counter = meter.create_counter("agent_based_counter", unit="1", description="agent export counter")
 agent_based_histogram = meter.create_histogram("agent_based_histogram", description="agent export histogram")
 agent_based_gauge = meter.create_up_down_counter("agent_based_gauge", unit="1", description="agent export gauge")
 
+custom_pipeline_counter = pipeline_meter.create_counter("custom_pipeline_counter", unit="1", description="pipeline export counter")
+custom_pipeline_histogram = pipeline_meter.create_histogram("custom_pipeline_histogram", description="pipeline export histogram")
+custom_pipeline_gauge = pipeline_meter.create_up_down_counter("custom_pipeline_gauge", unit="1", description="pipeline export gauge")
+
 
 should_send_local_root_client_call = False
 lock = threading.Lock()
@@ -63,6 +95,10 @@ def aws_sdk_call(request):
     agent_based_histogram.record(random.randint(100, 1000), {"Operation": "histogram"})
     agent_based_gauge.add(random.randint(-10, 10), {"Operation": "gauge"})
 
+    custom_pipeline_counter.add(1, {"Operation": "pipeline_counter"})
+    custom_pipeline_histogram.record(random.randint(100, 1000), {"Operation": "pipeline_histogram"})
+    custom_pipeline_gauge.add(random.randint(-10, 10), {"Operation": "pipeline_gauge"})
+
     bucket_name = "e2e-test-bucket-name"
 
     # Add a unique test ID to bucketname to associate buckets to specific test runs

terraform/python/ec2/default/main.tf

@@ -179,8 +179,10 @@ resource "null_resource" "main_service_setup" {
       export OTEL_EXPORTER_OTLP_METRICS_PROTOCOL=grpc
       export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=localhost:4317
       export OTEL_EXPORTER_OTLP_METRICS_INSECURE=true
+      export SERVICE_NAME='python-sample-application-${var.test_id}'
+      export DEPLOYMENT_ENVIRONMENT_NAME='ec2:default'
       export OTEL_TRACES_SAMPLER=always_on
-      export OTEL_RESOURCE_ATTRIBUTES="service.name=python-sample-application-${var.test_id},deployment.environment.name=ec2:default"
+      export OTEL_RESOURCE_ATTRIBUTES="service.name=$${SERVICE_NAME},deployment.environment.name=$${DEPLOYMENT_ENVIRONMENT_NAME}"
       export AWS_REGION='${var.aws_region}'
       python${var.language_version} manage.py migrate
       nohup opentelemetry-instrument python${var.language_version} manage.py runserver 0.0.0.0:8000 --noreload &

validator/src/main/resources/expected-data-template/python/ec2/default/aws-otel-custom-metrics.mustache

@@ -156,4 +156,72 @@
       value: ANY_VALUE
     -
       name: cloud.platform
-      value: aws_ec2
+      value: aws_ec2
+
+# Export pipeline metrics
+-
+  metricName: custom_pipeline_counter
+  namespace: {{metricNamespace}}
+  dimensions:
+    -
+      name: deployment.environment.name
+      value: ec2:default
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: Operation
+      value: pipeline_counter
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE
+-
+  metricName: custom_pipeline_histogram
+  namespace: {{metricNamespace}}
+  dimensions:
+    -
+      name: deployment.environment.name
+      value: ec2:default
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: Operation
+      value: pipeline_histogram
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE
+-
+  metricName: custom_pipeline_gauge
+  namespace: {{metricNamespace}}
+  dimensions:
+    -
+      name: deployment.environment.name
+      value: ec2:default
+    -
+      name: service.name
+      value: {{serviceName}}
+    -
+      name: Operation
+      value: pipeline_gauge
+    -
+      name: telemetry.sdk.name
+      value: opentelemetry
+    -
+      name: telemetry.sdk.language
+      value: python
+    -
+      name: telemetry.sdk.version
+      value: ANY_VALUE