Skip to content

Commit 50e34e8

Browse files
dtcaciuc-ggsdtcaciuc-ggs
authored
[Logs forwarder] Parse enhanced metrics from Lambda telemetry JSON logs. (DataDog#859)
* Parse enhanced metrics from Lambda telemetry JSON logs. These get emitted instead of regular REPORT logs if log format is set to JSON. * Address review comments. * Less try/catch * Organize constants * Remove unnecessary defensive JSON check
1 parent 0799b46 commit 50e34e8

6 files changed

+393
-1
lines changed

‎aws/logs_monitoring/enhanced_lambda_metrics.py‎

Lines changed: 87 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# under the Apache License Version 2.0.
33
# This product includes software developed at Datadog (https://www.datadoghq.com/).
44
# Copyright 2021 Datadog, Inc.
5+
import json
56
import os
67
import logging
78
import re
@@ -60,6 +61,19 @@
6061
INIT_DURATION_METRIC_NAME,
6162
]
6263

64+
# Keys that appear in Lambda telemetry records emitted when JSON logs are enabled
65+
MEMORY_ALLOCATED_RECORD_KEY = "memorySizeMB"
66+
INIT_DURATION_RECORD_KEY = "initDurationMs"
67+
DURATION_RECORD_KEY = "durationMs"
68+
BILLED_DURATION_RECORD_KEY = "billedDurationMs"
69+
MAX_MEMORY_USED_RECORD_KEY = "maxMemoryUsedMB"
70+
RUNTIME_METRICS_BY_RECORD_KEY = {
71+
# Except INIT_DURATION_RECORD_KEY which is handled separately
72+
DURATION_RECORD_KEY: DURATION_METRIC_NAME,
73+
BILLED_DURATION_RECORD_KEY: BILLED_DURATION_METRIC_NAME,
74+
MAX_MEMORY_USED_RECORD_KEY: MAX_MEMORY_USED_METRIC_NAME,
75+
}
76+
6377
# Multiply the duration metrics by 1/1000 to convert ms to seconds
6478
METRIC_ADJUSTMENT_FACTORS = {
6579
DURATION_METRIC_NAME: 0.001,
@@ -202,8 +216,12 @@ def generate_enhanced_lambda_metrics(log, tags_cache):
202216
if not is_lambda_log:
203217
return []
204218

219+
# Check if its Lambda lifecycle log that is emitted if log format is set to JSON
220+
parsed_metrics = parse_metrics_from_json_report_log(log_message)
221+
205222
# Check if this is a REPORT log
206-
parsed_metrics = parse_metrics_from_report_log(log_message)
223+
if not parsed_metrics:
224+
parsed_metrics = parse_metrics_from_report_log(log_message)
207225

208226
# Check if this is a timeout
209227
if not parsed_metrics:
@@ -254,6 +272,74 @@ def parse_lambda_tags_from_arn(arn):
254272
]
255273

256274

275+
def parse_metrics_from_json_report_log(log_message):
276+
try:
277+
body = json.loads(log_message)
278+
except json.JSONDecodeError:
279+
return []
280+
281+
stage = body.get("type", "")
282+
record = body.get("record", {})
283+
record_metrics = record.get("metrics", {})
284+
285+
if stage != "platform.report" or not record_metrics:
286+
return []
287+
288+
metrics = []
289+
290+
for record_key, metric_name in RUNTIME_METRICS_BY_RECORD_KEY.items():
291+
metric_point_value = record_metrics[record_key]
292+
293+
if metric_name in METRIC_ADJUSTMENT_FACTORS:
294+
metric_point_value *= METRIC_ADJUSTMENT_FACTORS[metric_name]
295+
296+
metrics.append(
297+
DatadogMetricPoint(
298+
f"{ENHANCED_METRICS_NAMESPACE_PREFIX}.{metric_name}",
299+
metric_point_value,
300+
)
301+
)
302+
303+
tags = [
304+
f"{MEMORY_ALLOCATED_FIELD_NAME}:{record_metrics[MEMORY_ALLOCATED_RECORD_KEY]}"
305+
]
306+
307+
init_duration = record_metrics.get(INIT_DURATION_RECORD_KEY)
308+
if init_duration:
309+
tags.append("cold_start:true")
310+
metrics.append(
311+
DatadogMetricPoint(
312+
f"{ENHANCED_METRICS_NAMESPACE_PREFIX}.{INIT_DURATION_METRIC_NAME}",
313+
init_duration * METRIC_ADJUSTMENT_FACTORS[INIT_DURATION_METRIC_NAME],
314+
)
315+
)
316+
else:
317+
tags.append("cold_start:false")
318+
319+
metrics.append(
320+
DatadogMetricPoint(
321+
f"{ENHANCED_METRICS_NAMESPACE_PREFIX}.{ESTIMATED_COST_METRIC_NAME}",
322+
calculate_estimated_cost(
323+
record_metrics[BILLED_DURATION_RECORD_KEY],
324+
record_metrics[MEMORY_ALLOCATED_RECORD_KEY],
325+
),
326+
)
327+
)
328+
329+
if record.get("status") == "timeout":
330+
metrics.append(
331+
DatadogMetricPoint(
332+
f"{ENHANCED_METRICS_NAMESPACE_PREFIX}.{TIMEOUTS_METRIC_NAME}",
333+
1.0,
334+
)
335+
)
336+
337+
for metric in metrics:
338+
metric.add_tags(tags)
339+
340+
return metrics
341+
342+
257343
def parse_metrics_from_report_log(report_log_line):
258344
"""Parses and returns metrics from the REPORT Lambda log
259345

‎aws/logs_monitoring/tests/approved_files/TestEnhancedLambdaMetrics.test_generate_enhanced_lambda_metrics_json.approved.json‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
[
2+
{
3+
"name": "aws.lambda.enhanced.duration",
4+
"tags": [
5+
"memorysize:128",
6+
"cold_start:false",
7+
"region:us-east-1",
8+
"account_id:172597598159",
9+
"aws_account:172597598159",
10+
"functionname:post-coupon-prod-us"
11+
],
12+
"timestamp": 10000,
13+
"value": 3.47065
14+
},
15+
{
16+
"name": "aws.lambda.enhanced.billed_duration",
17+
"tags": [
18+
"memorysize:128",
19+
"cold_start:false",
20+
"region:us-east-1",
21+
"account_id:172597598159",
22+
"aws_account:172597598159",
23+
"functionname:post-coupon-prod-us"
24+
],
25+
"timestamp": 10000,
26+
"value": 3.5
27+
},
28+
{
29+
"name": "aws.lambda.enhanced.max_memory_used",
30+
"tags": [
31+
"memorysize:128",
32+
"cold_start:false",
33+
"region:us-east-1",
34+
"account_id:172597598159",
35+
"aws_account:172597598159",
36+
"functionname:post-coupon-prod-us"
37+
],
38+
"timestamp": 10000,
39+
"value": 89
40+
},
41+
{
42+
"name": "aws.lambda.enhanced.estimated_cost",
43+
"tags": [
44+
"memorysize:128",
45+
"cold_start:false",
46+
"region:us-east-1",
47+
"account_id:172597598159",
48+
"aws_account:172597598159",
49+
"functionname:post-coupon-prod-us"
50+
],
51+
"timestamp": 10000,
52+
"value": 7.49168125e-06
53+
}
54+
]

‎aws/logs_monitoring/tests/approved_files/TestEnhancedLambdaMetrics.test_parse_metrics_from_cold_start_json_report_log.approved.json‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
[
2+
{
3+
"name": "aws.lambda.enhanced.duration",
4+
"tags": [
5+
"memorysize:128",
6+
"cold_start:true"
7+
],
8+
"timestamp": null,
9+
"value": 0.0008100000000000001
10+
},
11+
{
12+
"name": "aws.lambda.enhanced.billed_duration",
13+
"tags": [
14+
"memorysize:128",
15+
"cold_start:true"
16+
],
17+
"timestamp": null,
18+
"value": 0.1
19+
},
20+
{
21+
"name": "aws.lambda.enhanced.max_memory_used",
22+
"tags": [
23+
"memorysize:128",
24+
"cold_start:true"
25+
],
26+
"timestamp": null,
27+
"value": 90
28+
},
29+
{
30+
"name": "aws.lambda.enhanced.init_duration",
31+
"tags": [
32+
"memorysize:128",
33+
"cold_start:true"
34+
],
35+
"timestamp": null,
36+
"value": 1.234
37+
},
38+
{
39+
"name": "aws.lambda.enhanced.estimated_cost",
40+
"tags": [
41+
"memorysize:128",
42+
"cold_start:true"
43+
],
44+
"timestamp": null,
45+
"value": 4.0833375e-07
46+
}
47+
]

‎aws/logs_monitoring/tests/approved_files/TestEnhancedLambdaMetrics.test_parse_metrics_from_json_report_log.approved.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
[
2+
{
3+
"name": "aws.lambda.enhanced.duration",
4+
"tags": [
5+
"memorysize:128",
6+
"cold_start:false"
7+
],
8+
"timestamp": null,
9+
"value": 0.00062
10+
},
11+
{
12+
"name": "aws.lambda.enhanced.billed_duration",
13+
"tags": [
14+
"memorysize:128",
15+
"cold_start:false"
16+
],
17+
"timestamp": null,
18+
"value": 0.1
19+
},
20+
{
21+
"name": "aws.lambda.enhanced.max_memory_used",
22+
"tags": [
23+
"memorysize:128",
24+
"cold_start:false"
25+
],
26+
"timestamp": null,
27+
"value": 51
28+
},
29+
{
30+
"name": "aws.lambda.enhanced.estimated_cost",
31+
"tags": [
32+
"memorysize:128",
33+
"cold_start:false"
34+
],
35+
"timestamp": null,
36+
"value": 4.0833375e-07
37+
}
38+
]

‎aws/logs_monitoring/tests/approved_files/TestEnhancedLambdaMetrics.test_parse_metrics_from_timeout_json_report_log.approved.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
[
2+
{
3+
"name": "aws.lambda.enhanced.duration",
4+
"tags": [
5+
"memorysize:128",
6+
"cold_start:true"
7+
],
8+
"timestamp": null,
9+
"value": 30.0
10+
},
11+
{
12+
"name": "aws.lambda.enhanced.billed_duration",
13+
"tags": [
14+
"memorysize:128",
15+
"cold_start:true"
16+
],
17+
"timestamp": null,
18+
"value": 30.0
19+
},
20+
{
21+
"name": "aws.lambda.enhanced.max_memory_used",
22+
"tags": [
23+
"memorysize:128",
24+
"cold_start:true"
25+
],
26+
"timestamp": null,
27+
"value": 74
28+
},
29+
{
30+
"name": "aws.lambda.enhanced.init_duration",
31+
"tags": [
32+
"memorysize:128",
33+
"cold_start:true"
34+
],
35+
"timestamp": null,
36+
"value": 0.985413
37+
},
38+
{
39+
"name": "aws.lambda.enhanced.estimated_cost",
40+
"tags": [
41+
"memorysize:128",
42+
"cold_start:true"
43+
],
44+
"timestamp": null,
45+
"value": 6.270012500000001e-05
46+
},
47+
{
48+
"name": "aws.lambda.enhanced.timeouts",
49+
"tags": [
50+
"memorysize:128",
51+
"cold_start:true"
52+
],
53+
"timestamp": null,
54+
"value": 1.0
55+
}
56+
]

0 commit comments

Comments
 (0)