Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,16 @@
import requests
import schedule
from django.http import HttpResponse, JsonResponse
from opentelemetry import trace
from opentelemetry import trace, metrics
from opentelemetry.trace.span import format_trace_id

logger = logging.getLogger(__name__)

# Initialize OTEL metrics for span metrics
meter = metrics.get_meter(__name__)
request_counter = meter.create_counter("custom_requests_total", description="Total requests")
response_time_histogram = meter.create_histogram("custom_response_time", description="Response time")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

response_time_histogram unused?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

histogram code commented out for easy implementation after initial test validation.


Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

IMO we should create two meters - one using metrics.get_meter(__name__) (Agent-based export) like you've done here, and another using Custom export pipeline from public docs

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

2nd meter has been created.

should_send_local_root_client_call = False
lock = threading.Lock()
def run_local_root_client_call_recurring_service():
Expand Down Expand Up @@ -47,9 +52,18 @@ def run_scheduler():
run_local_root_client_call_recurring_service()

def healthcheck(request):
# Setup Span Attributes And Initialize Counter To Recieve Custom Metrics
span = trace.get_current_span()
span.set_attribute("operation.type", "healthcheck")
request_counter.add(1, {"operation.type": "healthcheck"})
return HttpResponse("healthcheck")

def aws_sdk_call(request):
# Setup Span Attributes And Initialize Counter To Recieve Custom Metrics
span = trace.get_current_span()
span.set_attribute("operation.type", "aws_sdk_call")
request_counter.add(1, {"operation.type": "aws_sdk_call"})

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TBH I don't think we need custom metrics on every API. Probably one is fine.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed from all API's except sdk call.

bucket_name = "e2e-test-bucket-name"

# Add a unique test ID to bucketname to associate buckets to specific test runs
Expand All @@ -70,6 +84,11 @@ def aws_sdk_call(request):
return get_xray_trace_id()

def http_call(request):
# Setup Span attributes to recieve Custom Metrics
span = trace.get_current_span()
span.set_attribute("operation.type", "http_call")
request_counter.add(1, {"operation.type": "http_call"})

url = "https://www.amazon.com"
try:
response = requests.get(url)
Expand All @@ -80,6 +99,11 @@ def http_call(request):
return get_xray_trace_id()

def downstream_service(request):
# Setup Span attributes to recieve Custom Metrics
span = trace.get_current_span()
span.set_attribute("operation.type", "downstream_service")
request_counter.add(1, {"operation.type": "downstream_service"})

ip = request.GET.get('ip', '')
ip = ip.replace("/", "")
url = f"http://{ip}:8001/healthcheck"
Expand All @@ -94,6 +118,10 @@ def downstream_service(request):
return get_xray_trace_id()

def async_service(request):
# Setup Span Attributes And Initialize Counter To Recieve Custom Metrics
span = trace.get_current_span()
span.set_attribute("operation.type", "async_service")
request_counter.add(1, {"operation.type": "async_service"})
global should_send_local_root_client_call
# Log the request
logger.info("Client-call received")
Expand All @@ -111,6 +139,11 @@ def get_xray_trace_id():
return JsonResponse({"traceId": xray_trace_id})

def mysql(request):
# Setup Span attributes to recieve Custom Metrics
span = trace.get_current_span()
span.set_attribute("operation.type", "mysql")
request_counter.add(1, {"operation.type": "mysql"})

logger.info("mysql received")

encoded_password = os.environ["RDS_MYSQL_CLUSTER_PASSWORD"]
Expand Down
22 changes: 22 additions & 0 deletions terraform/python/ec2/default/amazon-cloudwatch-custom-agent.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
{
"traces": {
"traces_collected": {
"application_signals": {}
}
},
"logs": {
"metrics_collected": {
"application_signals": {}
}
},
"metrics": {
"namespace": "CWAgent",
"metrics_collected": {
"application_signals": {},
"otlp": {
"grpc_endpoint": "0.0.0.0:4317",
"http_endpoint": "0.0.0.0:4318"
}
}
}
}
13 changes: 13 additions & 0 deletions terraform/python/ec2/default/variables.tf
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are we setting OTEL_RESOURCE_ATTRIBUTES="service.name=$YOUR_SVC_NAME,deployment.environment.name=$YOUR_ENV_NAME" anywhere?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes it is being set in 'main.tf'

Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,17 @@ variable "language_version" {

variable "cpu_architecture" {
default = "x86_64"
}
#Adding Custom Metrics Variables

variable "custom_metrics_enabled" {
description = "Enable custom OTEL metrics in the sample application"
type = bool
default = false
}

variable "custom_metrics_config" {
description = "JSON configuration for custom metrics"
type = string
default = "{}"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

amazon-cloudwatch-custom-agent.json?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It replaces the empty curly brackets as the default now.

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
# OpenTelemetry Custom Metrics Validation Templates
-
metricName: custom_requests_total
namespace: {{customMetricNamespace}}
dimensions:
-
name: operation.type
value: healthcheck
-
name: Service
value: {{serviceName}}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should also see deployment.environment.name

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It has been added.


-
metricName: custom_requests_total
namespace: {{customMetricNamespace}}
dimensions:
-
name: operation.type
value: aws_sdk_call
-
name: Service
value: {{serviceName}}

-
metricName: custom_requests_total
namespace: {{customMetricNamespace}}
dimensions:
-
name: operation.type
value: http_call
-
name: Service
value: {{serviceName}}

-
metricName: custom_requests_total
namespace: {{customMetricNamespace}}
dimensions:
-
name: operation.type
value: downstream_service
-
name: Service
value: {{serviceName}}

-
metricName: custom_requests_total
namespace: {{customMetricNamespace}}
dimensions:
-
name: operation.type
value: async_service
-
name: Service
value: {{serviceName}}

-
metricName: custom_requests_total
namespace: {{customMetricNamespace}}
dimensions:
-
name: operation.type
value: mysql
-
name: Service
value: {{serviceName}}

# Code for adding in Custom Histogram Metrics
# -
# metricName: custom_response_time
# namespace: {{customMetricNamespace}}
# dimensions:
# -
# name: operation.type
# value: aws_sdk_call
# -
# name: Service
# value: {{serviceName}}
Loading