Skip to content

Commit 8c289a4

Browse files
duncanistaduncanista
committed
fix issues with black
1 parent 87631e9 commit 8c289a4

File tree

2 files changed

+138
-38
lines changed

2 files changed

+138
-38
lines changed

‎aws/logs_monitoring/enhanced_lambda_metrics.py‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,13 +34,17 @@
3434
r"REPORT\s+"
3535
+ r"RequestId:\s+(?P<{}>[\w-]+)\s+".format(REQUEST_ID_FIELD_NAME)
3636
+ r"Duration:\s+(?P<{}>[\d\.]+)\s+ms\s+".format(DURATION_METRIC_NAME)
37-
+ r"Billed\s+Duration:\s+(?P<{}>[\d\.]+)\s+ms\s+".format(BILLED_DURATION_METRIC_NAME)
37+
+ r"Billed\s+Duration:\s+(?P<{}>[\d\.]+)\s+ms\s+".format(
38+
BILLED_DURATION_METRIC_NAME
39+
)
3840
+ r"Memory\s+Size:\s+(?P<{}>\d+)\s+MB\s+".format(MEMORY_ALLOCATED_FIELD_NAME)
3941
+ r"Max\s+Memory\s+Used:\s+(?P<{}>\d+)\s+MB".format(MAX_MEMORY_USED_METRIC_NAME)
4042
+ r"(\s+Init\s+Duration:\s+(?P<{}>[\d\.]+)\s+ms)?".format(INIT_DURATION_METRIC_NAME)
4143
)
4244

43-
TIMED_OUT_REGEX = re.compile(r"Task\stimed\sout\safter\s+(?P<{}>[\d\.]+)\s+seconds".format(TIMEOUTS_METRIC_NAME))
45+
TIMED_OUT_REGEX = re.compile(
46+
r"Task\stimed\sout\safter\s+(?P<{}>[\d\.]+)\s+seconds".format(TIMEOUTS_METRIC_NAME)
47+
)
4448

4549
OUT_OF_MEMORY_ERROR_STRINGS = [
4650
"fatal error: runtime: out of memory", # Go
@@ -129,13 +133,19 @@ def submit_to_dd(self):
129133
if not timestamp:
130134
timestamp = time()
131135

132-
logger.debug("Submitting metric {} {} {}".format(self.name, self.value, self.tags))
133-
lambda_stats.distribution(self.name, self.value, timestamp=timestamp, tags=self.tags)
136+
logger.debug(
137+
"Submitting metric {} {} {}".format(self.name, self.value, self.tags)
138+
)
139+
lambda_stats.distribution(
140+
self.name, self.value, timestamp=timestamp, tags=self.tags
141+
)
134142

135143

136144
def get_last_modified_time(s3_file):
137145
last_modified_str = s3_file["ResponseMetadata"]["HTTPHeaders"]["last-modified"]
138-
last_modified_date = datetime.datetime.strptime(last_modified_str, "%a, %d %b %Y %H:%M:%S %Z")
146+
last_modified_date = datetime.datetime.strptime(
147+
last_modified_str, "%a, %d %b %Y %H:%M:%S %Z"
148+
)
139149
last_modified_unix_time = int(last_modified_date.strftime("%s"))
140150
return last_modified_unix_time
141151

@@ -153,7 +163,9 @@ def parse_and_submit_enhanced_metrics(logs, cache_layer):
153163

154164
for log in logs:
155165
try:
156-
enhanced_metrics = generate_enhanced_lambda_metrics(log, cache_layer.get_lambda_tags_cache())
166+
enhanced_metrics = generate_enhanced_lambda_metrics(
167+
log, cache_layer.get_lambda_tags_cache()
168+
)
157169
for enhanced_metric in enhanced_metrics:
158170
enhanced_metric.submit_to_dd()
159171
except Exception as e:
@@ -178,13 +190,13 @@ def generate_enhanced_lambda_metrics(log, tags_cache):
178190
"owner": "172597598159"
179191
},
180192
"function_version": "$LATEST",
181-
"invoked_function_arn": "arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
193+
"invoked_function_arn": "arn:aws:lambda:us-east-1:..."
182194
},
183195
"lambda": {
184196
"arn": "arn:aws:lambda:us-east-1:172597598159:function:function_log_generator"
185197
},
186198
"ddsourcecategory": "aws",
187-
"ddtags": "env:demo,python_version:3.6,role:lambda,forwardername:collect_logs_datadog_demo,memorysize:128,forwarder_version:2.0.0,functionname:function_log_generator,env:none",
199+
"ddtags": "env:demo,python_version:3.6,...",
188200
"ddsource": "lambda",
189201
"service": "function_log_generator",
190202
"host": "arn:aws:lambda:us-east-1:172597598159:function:function_log_generator"
@@ -444,7 +456,9 @@ def create_out_of_memory_enhanced_metric(log_line):
444456
DatadogMetricPoint[]
445457
"""
446458

447-
contains_out_of_memory_error = any(s in log_line for s in OUT_OF_MEMORY_ERROR_STRINGS)
459+
contains_out_of_memory_error = any(
460+
s in log_line for s in OUT_OF_MEMORY_ERROR_STRINGS
461+
)
448462

449463
if not contains_out_of_memory_error:
450464
return []

‎aws/logs_monitoring/tests/test_enhanced_lambda_metrics.py‎

Lines changed: 115 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -96,16 +96,28 @@ class TestEnhancedLambdaMetrics(unittest.TestCase):
9696
"durationMs": 2.524,
9797
},
9898
"spans": [
99-
{"name": "responseLatency", "start": "2026-01-08T18:22:35.342Z", "durationMs": 0.642},
100-
{"name": "responseDuration", "start": "2026-01-08T18:22:35.343Z", "durationMs": 0.075},
99+
{
100+
"name": "responseLatency",
101+
"start": "2026-01-08T18:22:35.342Z",
102+
"durationMs": 0.642,
103+
},
104+
{
105+
"name": "responseDuration",
106+
"start": "2026-01-08T18:22:35.343Z",
107+
"durationMs": 0.075,
108+
},
101109
],
102110
"status": "success",
103111
},
104112
}
105113
)
106114

107115
def test_parse_lambda_tags_from_arn(self):
108-
verify_as_json(parse_lambda_tags_from_arn("arn:aws:lambda:us-east-1:1234597598159:function:swf-hello-test"))
116+
verify_as_json(
117+
parse_lambda_tags_from_arn(
118+
"arn:aws:lambda:us-east-1:1234597598159:function:swf-hello-test"
119+
)
120+
)
109121

110122
def test_parse_metrics_from_report_log(self):
111123
parsed_metrics = parse_metrics_from_report_log(self.malformed_report)
@@ -144,7 +156,9 @@ def test_parse_metrics_from_timeout_json_report_log(self):
144156

145157
def test_parse_metrics_from_partial_metrics_json_report_log(self):
146158
"""Test that JSON report logs with partial/incomplete metrics don't raise KeyError"""
147-
parsed_metrics = parse_metrics_from_json_report_log(self.managed_instances_metrics_json_report)
159+
parsed_metrics = parse_metrics_from_json_report_log(
160+
self.managed_instances_metrics_json_report
161+
)
148162
# Should only return metrics that are present (duration in this case)
149163
# Should not raise KeyError for missing billedDurationMs, maxMemoryUsedMB, memorySizeMB
150164
assert len(parsed_metrics) == 1 # Only duration metric
@@ -157,19 +171,31 @@ def test_parse_metrics_from_partial_metrics_json_report_log(self):
157171

158172
def test_create_out_of_memory_enhanced_metric(self):
159173
go_out_of_memory_error = "fatal error: runtime: out of memory"
160-
self.assertEqual(len(create_out_of_memory_enhanced_metric(go_out_of_memory_error)), 1)
174+
self.assertEqual(
175+
len(create_out_of_memory_enhanced_metric(go_out_of_memory_error)), 1
176+
)
161177

162-
java_out_of_memory_error = "Requested array size exceeds VM limit: java.lang.OutOfMemoryError"
163-
self.assertEqual(len(create_out_of_memory_enhanced_metric(java_out_of_memory_error)), 1)
178+
java_out_of_memory_error = (
179+
"Requested array size exceeds VM limit: java.lang.OutOfMemoryError"
180+
)
181+
self.assertEqual(
182+
len(create_out_of_memory_enhanced_metric(java_out_of_memory_error)), 1
183+
)
164184

165185
node_out_of_memory_error = "FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory"
166-
self.assertEqual(len(create_out_of_memory_enhanced_metric(node_out_of_memory_error)), 1)
186+
self.assertEqual(
187+
len(create_out_of_memory_enhanced_metric(node_out_of_memory_error)), 1
188+
)
167189

168190
python_out_of_memory_error = "fatal error: runtime: out of memory"
169-
self.assertEqual(len(create_out_of_memory_enhanced_metric(python_out_of_memory_error)), 1)
191+
self.assertEqual(
192+
len(create_out_of_memory_enhanced_metric(python_out_of_memory_error)), 1
193+
)
170194

171195
ruby_out_of_memory_error = "failed to allocate memory (NoMemoryError)"
172-
self.assertEqual(len(create_out_of_memory_enhanced_metric(ruby_out_of_memory_error)), 1)
196+
self.assertEqual(
197+
len(create_out_of_memory_enhanced_metric(ruby_out_of_memory_error)), 1
198+
)
173199

174200
success_message = "Success!"
175201
self.assertEqual(len(create_out_of_memory_enhanced_metric(success_message)), 0)
@@ -204,9 +230,15 @@ def test_generate_enhanced_lambda_metrics_json(
204230
"owner": "172597598159",
205231
},
206232
"function_version": "$LATEST",
207-
"invoked_function_arn": ("arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"),
233+
"invoked_function_arn": (
234+
"arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
235+
),
236+
},
237+
"lambda": {
238+
"arn": (
239+
"arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us"
240+
)
208241
},
209-
"lambda": {"arn": ("arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us")},
210242
"timestamp": 10000,
211243
}
212244

@@ -219,7 +251,11 @@ def test_generate_enhanced_lambda_metrics(self):
219251

220252
logs_input = {
221253
"message": (
222-
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\tDuration: 3470.65 ms\tBilled Duration: 3500 ms\tMemory Size: 128 MB\tMax Memory Used: 89 MB\t\nXRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\tSegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
254+
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\t"
255+
"Duration: 3470.65 ms\tBilled Duration: 3500 ms\t"
256+
"Memory Size: 128 MB\tMax Memory Used: 89 MB\t\n"
257+
"XRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\t"
258+
"SegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
223259
),
224260
"aws": {
225261
"awslogs": {
@@ -228,9 +264,15 @@ def test_generate_enhanced_lambda_metrics(self):
228264
"owner": "172597598159",
229265
},
230266
"function_version": "$LATEST",
231-
"invoked_function_arn": ("arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"),
267+
"invoked_function_arn": (
268+
"arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
269+
),
270+
},
271+
"lambda": {
272+
"arn": (
273+
"arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us"
274+
)
232275
},
233-
"lambda": {"arn": ("arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us")},
234276
"timestamp": 10000,
235277
}
236278

@@ -241,11 +283,17 @@ def test_generate_enhanced_lambda_metrics_with_tags(
241283
self,
242284
):
243285
tags_cache = LambdaTagsCache("")
244-
tags_cache.get = MagicMock(return_value=["team:metrics", "monitor:datadog", "env:prod", "creator:swf"])
286+
tags_cache.get = MagicMock(
287+
return_value=["team:metrics", "monitor:datadog", "env:prod", "creator:swf"]
288+
)
245289

246290
logs_input = {
247291
"message": (
248-
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\tDuration: 3470.65 ms\tBilled Duration: 3500 ms\tMemory Size: 128 MB\tMax Memory Used: 89 MB\t\nXRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\tSegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
292+
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\t"
293+
"Duration: 3470.65 ms\tBilled Duration: 3500 ms\t"
294+
"Memory Size: 128 MB\tMax Memory Used: 89 MB\t\n"
295+
"XRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\t"
296+
"SegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
249297
),
250298
"aws": {
251299
"awslogs": {
@@ -254,9 +302,15 @@ def test_generate_enhanced_lambda_metrics_with_tags(
254302
"owner": "172597598159",
255303
},
256304
"function_version": "$LATEST",
257-
"invoked_function_arn": ("arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"),
305+
"invoked_function_arn": (
306+
"arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
307+
),
308+
},
309+
"lambda": {
310+
"arn": (
311+
"arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us"
312+
)
258313
},
259-
"lambda": {"arn": ("arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us")},
260314
"timestamp": 10000,
261315
}
262316

@@ -269,7 +323,11 @@ def test_generate_enhanced_lambda_metrics_once_with_missing_arn(self):
269323

270324
logs_input = {
271325
"message": (
272-
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\tDuration: 3470.65 ms\tBilled Duration: 3500 ms\tMemory Size: 128 MB\tMax Memory Used: 89 MB\t\nXRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\tSegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
326+
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\t"
327+
"Duration: 3470.65 ms\tBilled Duration: 3500 ms\t"
328+
"Memory Size: 128 MB\tMax Memory Used: 89 MB\t\n"
329+
"XRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\t"
330+
"SegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
273331
),
274332
"aws": {
275333
"awslogs": {
@@ -278,9 +336,15 @@ def test_generate_enhanced_lambda_metrics_once_with_missing_arn(self):
278336
"owner": "172597598159",
279337
},
280338
"function_version": "$LATEST",
281-
"invoked_function_arn": ("arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"),
339+
"invoked_function_arn": (
340+
"arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
341+
),
342+
},
343+
"lambda": {
344+
"arn": (
345+
"arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us"
346+
)
282347
},
283-
"lambda": {"arn": ("arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us")},
284348
"timestamp": 10000,
285349
}
286350

@@ -306,7 +370,11 @@ def test_generate_enhanced_lambda_metrics_refresh_s3_cache(self, mock1, mock2):
306370

307371
logs_input = {
308372
"message": (
309-
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\tDuration: 3470.65 ms\tBilled Duration: 3500 ms\tMemory Size: 128 MB\tMax Memory Used: 89 MB\t\nXRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\tSegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
373+
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\t"
374+
"Duration: 3470.65 ms\tBilled Duration: 3500 ms\t"
375+
"Memory Size: 128 MB\tMax Memory Used: 89 MB\t\n"
376+
"XRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\t"
377+
"SegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
310378
),
311379
"aws": {
312380
"awslogs": {
@@ -315,9 +383,15 @@ def test_generate_enhanced_lambda_metrics_refresh_s3_cache(self, mock1, mock2):
315383
"owner": "172597598159",
316384
},
317385
"function_version": "$LATEST",
318-
"invoked_function_arn": ("arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"),
386+
"invoked_function_arn": (
387+
"arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
388+
),
389+
},
390+
"lambda": {
391+
"arn": (
392+
"arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us"
393+
)
319394
},
320-
"lambda": {"arn": ("arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us")},
321395
"timestamp": 10000,
322396
}
323397

@@ -358,7 +432,11 @@ def test_generate_enhanced_lambda_metrics_client_error(
358432

359433
logs_input = {
360434
"message": (
361-
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\tDuration: 3470.65 ms\tBilled Duration: 3500 ms\tMemory Size: 128 MB\tMax Memory Used: 89 MB\t\nXRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\tSegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
435+
"REPORT RequestId: fe1467d6-1458-4e20-8e40-9aaa4be7a0f4\t"
436+
"Duration: 3470.65 ms\tBilled Duration: 3500 ms\t"
437+
"Memory Size: 128 MB\tMax Memory Used: 89 MB\t\n"
438+
"XRAY TraceId: 1-5d8bba5a-dc2932496a65bab91d2d42d4\t"
439+
"SegmentId: 5ff79d2a06b82ad6\tSampled: true\t\n"
362440
),
363441
"aws": {
364442
"awslogs": {
@@ -367,9 +445,15 @@ def test_generate_enhanced_lambda_metrics_client_error(
367445
"owner": "172597598159",
368446
},
369447
"function_version": "$LATEST",
370-
"invoked_function_arn": ("arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"),
448+
"invoked_function_arn": (
449+
"arn:aws:lambda:us-east-1:172597598159:function:collect_logs_datadog_demo"
450+
),
451+
},
452+
"lambda": {
453+
"arn": (
454+
"arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us"
455+
)
371456
},
372-
"lambda": {"arn": ("arn:aws:lambda:us-east-1:172597598159:function:post-coupon-prod-us")},
373457
"timestamp": 10000,
374458
}
375459

@@ -447,7 +531,9 @@ def test_generate_enhanced_lambda_metrics_out_of_memory(
447531

448532
logs_input = {
449533
"message": (
450-
"2020-06-09T15:02:26.150Z 7c9567b5-107b-4a6c-8798-0157ac21db52 FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory\n\n"
534+
"2020-06-09T15:02:26.150Z 7c9567b5-107b-4a6c-8798-0157ac21db52 "
535+
"FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - "
536+
"JavaScript heap out of memory\n\n"
451537
),
452538
"aws": {
453539
"awslogs": {

0 commit comments

Comments
 (0)