Skip to content

Commit b3ebc80

Browse files
swfuncswfunc
committed
Make integration test work for async metrics
1 parent 840f7cd commit b3ebc80

15 files changed

+151
-225
lines changed

‎scripts/run_integration_tests.sh‎

Lines changed: 65 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#!/bin/bash
22

3+
# Stop execution if any command has errors
34
set -e
45

56
# These values need to be in sync with serverless.yml, where there needs to be a function
@@ -16,37 +17,90 @@ integration_tests_dir="$repo_dir/tests/integration"
1617

1718
script_start_time=$(date --iso-8601=seconds)
1819

20+
mismatch_found=false
21+
1922
echo "Start time is $script_start_time"
2023

21-
echo "Building new layers that will be uploaded with our test functions"
24+
echo "Building layers that will be deployed with our test functions"
2225
# source $scripts_dir/build_layers.sh
2326

27+
if [ -n "$OVERWRITE" ]; then
28+
echo "Overwriting snapshots in this execution"
29+
fi
30+
2431
echo "Deploying functions"
2532
cd $integration_tests_dir
2633
serverless deploy
2734

2835
echo "Invoking functions"
2936
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
3037
for runtime in "${RUNTIMES[@]}"; do
31-
echo "Invoking $handler_name-$runtime"
32-
serverless invoke -f "$handler_name-$runtime"
38+
function_name="$handler_name-$runtime"
39+
function_snapshot_path="./snapshots/$function_name.return_value"
40+
41+
return_value=$(serverless invoke -f $function_name)
42+
43+
if [ -n "$OVERWRITE" ]; then
44+
# If $OVERWRITE is set to true, write the new logs over the current snapshot
45+
echo "Overwriting return value snapshot for $function_name"
46+
echo "$return_value" >$function_snapshot_path
47+
else
48+
# Compare new return value to snapshot
49+
set +e # Don't exit this script if there is a diff
50+
diff_output=$(echo "$return_value" | diff - $function_snapshot_path)
51+
if [ $? -eq 1 ]; then
52+
echo "FAILURE: Return value for $function_name does not match snapshot:"
53+
echo "$diff_output"
54+
mismatch_found=true
55+
else
56+
echo "SUCCESS: Return value for $function_name matches snapshot"
57+
fi
58+
set -e
59+
fi
3360
done
3461
done
3562

36-
echo "Sleeping for $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..."
63+
echo "Sleeping $LOGS_WAIT_SECONDS seconds to wait for logs to appear in CloudWatch..."
3764
sleep $LOGS_WAIT_SECONDS
3865

3966
echo "Fetching logs for invocations and comparing to snapshots"
4067
for handler_name in "${LAMBDA_HANDLERS[@]}"; do
4168
for runtime in "${RUNTIMES[@]}"; do
42-
logs=$(serverless logs -f "$handler_name-$runtime" --startTime $script_start_time)
43-
python compare_to_snapshots.py "$handler_name-$runtime" "$logs"
69+
function_name="$handler_name-$runtime"
70+
function_snapshot_path="./snapshots/$function_name.logs"
71+
72+
# Fetch logs with serverless cli
73+
logs=$(serverless logs -f $function_name --startTime $script_start_time)
74+
75+
# Filter serverless cli errors
76+
logs=$(echo "$logs" | sed '/Serverless: Recoverable error occurred/d')
77+
78+
# Replace invocation-specific data with XXXX to normalize between executions
79+
logs=$(echo "$logs" | sed -E 's/(RequestId|TraceId|SegmentId|Duration|Memory Used|"e"): [a-z0-9\.\-]+/\1: XXXX/g')
80+
81+
if [ -n "$OVERWRITE" ]; then
82+
# If $OVERWRITE is set to true, write the new logs over the current snapshot
83+
echo "Overwriting snapshot for $function_name"
84+
echo "$logs" >$function_snapshot_path
85+
else
86+
# Compare new logs to snapshots
87+
set +e # Don't exit this script if there is a diff
88+
diff_output=$(echo "$logs" | diff - $function_snapshot_path)
89+
if [ $? -eq 1 ]; then
90+
echo "FAILURE: Mismatch found between new $function_name logs and snapshot:"
91+
echo "$diff_output"
92+
mismatch_found=true
93+
else
94+
echo "SUCCESS: New logs for $function_name match snapshot"
95+
fi
96+
set -e
97+
fi
4498
done
4599
done
46100

47-
# Go back to the repo root
48-
cd $repo_dir
49-
50-
# Download the new CloudWatch logs for each Lambda
101+
if [ "$mismatch_found" = true ]; then
102+
echo "TEST FAILED: A mismatch between newly generated logs and a snapshot was found above. If this is expected, re-run this script with OVERWRITE=true to generate new snapshots"
103+
exit 1
104+
fi
51105

52-
# Filter out all logs that aren't metrics and traces and compare to the snapshots in this repo
106+
echo "TEST SUCCEEDED: No difference found between new logs and snapshots"

‎tests/integration/compare_to_snapshots.py‎

Lines changed: 0 additions & 186 deletions
This file was deleted.

‎tests/integration/serverless.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,3 +52,35 @@ functions:
5252
- { Ref: Python38LambdaLayer }
5353
environment:
5454
DD_FLUSH_TO_LOG: true
55+
56+
sync-metrics-python27:
57+
handler: submit_metrics.handle
58+
runtime: python2.7
59+
layers:
60+
- { Ref: Python27LambdaLayer }
61+
environment:
62+
DD_API_KEY: "abcdefghijk"
63+
64+
sync-metrics-python36:
65+
handler: submit_metrics.handle
66+
runtime: python3.6
67+
layers:
68+
- { Ref: Python36LambdaLayer }
69+
environment:
70+
DD_API_KEY: "abcdefghijk"
71+
72+
sync-metrics-python37:
73+
handler: submit_metrics.handle
74+
runtime: python3.7
75+
layers:
76+
- { Ref: Python37LambdaLayer }
77+
environment:
78+
DD_API_KEY: "abcdefghijk"
79+
80+
sync-metrics-python38:
81+
handler: submit_metrics.handle
82+
runtime: python3.8
83+
layers:
84+
- { Ref: Python38LambdaLayer }
85+
environment:
86+
DD_API_KEY: "abcdefghijk"

‎tests/integration/snapshots/async-metrics-python27.logs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
START RequestId: XXXX Version: $LATEST
2+
{"e": XXXX, "m": "aws.lambda.enhanced.invocations", "t": ["region:us-east-1", "account_id:601427279990", "functionname:integration-tester-dev-async-metrics-python27", "cold_start:true", "memorysize:1024", "runtime:python2.7", "dd_lambda_layer:datadog-python27_2.13.0"], "v": 1}
3+
{"e": XXXX, "m": "hello.dog", "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python27_2.13.0"], "v": 1}
4+
{"e": XXXX, "m": "tests.integration.count", "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python27_2.13.0"], "v": 21}
5+
END RequestId: XXXX
6+
REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms
7+
XRAY TraceId: XXXX SegmentId: XXXX Sampled: true

‎tests/integration/snapshots/async-metrics-python27.return_value‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"body": "hello, dog!",
3+
"statusCode": 200
4+
}

‎tests/integration/snapshots/async-metrics-python27.snapshot‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎tests/integration/snapshots/async-metrics-python36.logs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
START RequestId: XXXX Version: $LATEST
2+
{"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:601427279990", "functionname:integration-tester-dev-async-metrics-python36", "cold_start:true", "memorysize:1024", "runtime:python3.6", "dd_lambda_layer:datadog-python36_2.13.0"]}
3+
{"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python36_2.13.0"]}
4+
{"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python36_2.13.0"]}
5+
END RequestId: XXXX
6+
REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms
7+
XRAY TraceId: XXXX SegmentId: XXXX Sampled: true

‎tests/integration/snapshots/async-metrics-python36.return_value‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"statusCode": 200,
3+
"body": "hello, dog!"
4+
}

‎tests/integration/snapshots/async-metrics-python36.snapshot‎

Lines changed: 0 additions & 7 deletions
This file was deleted.

‎tests/integration/snapshots/async-metrics-python37.logs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
START RequestId: XXXX Version: $LATEST
2+
{"m": "aws.lambda.enhanced.invocations", "v": 1, "e": XXXX, "t": ["region:us-east-1", "account_id:601427279990", "functionname:integration-tester-dev-async-metrics-python37", "cold_start:true", "memorysize:1024", "runtime:python3.7", "dd_lambda_layer:datadog-python37_2.13.0"]}
3+
{"m": "hello.dog", "v": 1, "e": XXXX, "t": ["team:serverless", "role:hello", "dd_lambda_layer:datadog-python37_2.13.0"]}
4+
{"m": "tests.integration.count", "v": 21, "e": XXXX, "t": ["test:integration", "role:hello", "dd_lambda_layer:datadog-python37_2.13.0"]}
5+
END RequestId: XXXX
6+
REPORT RequestId: XXXX Duration: XXXX ms Billed Duration: XXXX ms Memory Size: 1024 MB Max Memory Used: XXXX MB Init Duration: XXXX ms
7+
XRAY TraceId: XXXX SegmentId: XXXX Sampled: true

0 commit comments

Comments
 (0)