Skip to content

Commit f0f206c

Browse files
lukaszgryglickilukaszgryglicki
authored
Merge pull request #4676 from communitybridge/unicron-add-logs-lookup-utils
Logs lookup utils for EasyCLA
2 parents 17552f5 + 40f46ed commit f0f206c

File tree

2 files changed

+96
-18
lines changed

2 files changed

+96
-18
lines changed

‎utils/search_aws_log_group.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
# STAGE=dev DEBUG=1 DTFROM='3 days ago' DTTO='2 days ago' ./utils/search_aws_log_group.sh 'cla-backend-dev-githubactivity' 'error'
3+
4+
if [ -z "$STAGE" ]
5+
then
6+
export STAGE=dev
7+
fi
8+
9+
if [ -z "${1}" ]
10+
then
11+
echo "$0: you must specify log group name, for example: 'cla-backend-dev-githubactivity', 'cla-backend-dev-apiv2', 'cla-backend-dev-api-v3-lambda', 'cla-backend-go-api-v4-lambda'"
12+
exit 1
13+
fi
14+
15+
log_group=$(echo "$1" | sed -E "s/\b(dev|prod)\b/${STAGE}/g")
16+
17+
if [ -z "${2}" ]
18+
then
19+
echo "$0: you must specify the search term, for example 'error'"
20+
exit 2
21+
fi
22+
23+
if [ -z "${DTFROM}" ]
24+
then
25+
export DTFROM="$(date -d '3 days ago' +%s)000"
26+
else
27+
export DTFROM="$(date -d "${DTFROM}" +%s)000"
28+
fi
29+
30+
if [ -z "${DTTO}" ]
31+
then
32+
export DTTO="$(date +%s)000"
33+
else
34+
export DTTO="$(date -d "${DTTO}" +%s)000"
35+
fi
36+
37+
if [ ! -z "${DEBUG}" ]
38+
then
39+
echo "aws --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${2}\""
40+
aws --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\""
41+
else
42+
aws --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\"" | jq -r '.events'
43+
fi
44+

‎utils/search_aws_logs.sh‎

Lines changed: 52 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,32 @@
11
#!/bin/bash
2-
# STAGE=dev DEBUG=1 DTFROM='3 days ago' DTTO='2 days ago' ./utils/search_aws_logs.sh 'cla-backend-dev-githubactivity' 'error'
32

4-
if [ -z "$STAGE" ]
3+
# Copyright The Linux Foundation and each contributor to LFX.
4+
# SPDX-License-Identifier: MIT
5+
6+
# REGION=us-east-1|us-east-2 STAGE=dev DEBUG=1 DTFROM='3 days ago' DTTO='2 days ago' ./utils/search_aws_logs.sh 'error'
7+
# To find distinct log groups: | jq -r 'map(.logGroupName) | unique | .[]'
8+
# in us-east-1 (mostly V1, V2 and V3):
9+
# To see specific log group: | jq 'map(select(.logGroupName == "/aws/lambda/cla-backend-dev-apiv1"))'
10+
# To filter out one log group: | jq 'map(select(.logGroupName != "/aws/lambda/cla-backend-dev-apiv2"))'
11+
# To filter out one log group: | jq 'map(select(.logGroupName != "cla-backend-dev-api-v3-lambda"))'
12+
# in us-east-2 (V4):
13+
# To filter out one log group: | jq 'map(select(.logGroupName != "cla-backend-go-api-v4-lambda"))'
14+
15+
if [ -z "${STAGE}" ]
516
then
6-
export STAGE=dev
17+
export STAGE=prod
718
fi
819

9-
if [ -z "${1}" ]
20+
if [ -z "${REGION}" ]
1021
then
11-
echo "$0: you must specify log group name, for example: 'cla-backend-dev-githubactivity', 'cla-backend-dev-apiv2', 'cla-backend-dev-api-v3-lambda', 'cla-backend-go-api-v4-lambda'"
12-
exit 1
22+
export REGION="us-east-1"
1323
fi
1424

15-
log_group=$(echo "$1" | sed -E "s/\b(dev|prod)\b/${STAGE}/g")
16-
17-
if [ -z "${2}" ]
25+
search="${1}"
26+
if [ -z "${1}" ]
1827
then
19-
echo "$0: you must specify the search term, for example 'error'"
20-
exit 2
28+
echo "$0: you should specify the search term, defaulting to 'error'"
29+
search="error"
2130
fi
2231

2332
if [ -z "${DTFROM}" ]
@@ -34,11 +43,36 @@ else
3443
export DTTO="$(date -d "${DTTO}" +%s)000"
3544
fi
3645

37-
if [ ! -z "${DEBUG}" ]
38-
then
39-
echo "aws --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"/aws/lambda/${log_group}\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${2}\""
40-
aws --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\""
41-
else
42-
aws --profile "lfproduct-${STAGE}" logs filter-log-events --log-group-name "/aws/lambda/${log_group}" --start-time "${DTFROM}" --end-time "${DTTO}" --filter-pattern "\"${2}\"" | jq -r '.events'
43-
fi
46+
mapfile -t log_groups_array < <(aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs describe-log-groups --log-group-name-prefix "/aws/lambda/cla-" --query "logGroups[].logGroupName" | jq -r '.[]')
47+
results=()
48+
for log_group in "${log_groups_array[@]}"
49+
do
50+
if [ ! -z "${DEBUG}" ]
51+
then
52+
echo "lookup log group '${log_group}': aws --region "${REGION}" --profile \"lfproduct-${STAGE}\" logs filter-log-events --log-group-name \"$log_group\" --start-time \"${DTFROM}\" --end-time \"${DTTO}\" --filter-pattern \"${search}\"" >&2
53+
fi
54+
logs=$(aws --region "${REGION}" --profile "lfproduct-${STAGE}" logs filter-log-events \
55+
--log-group-name "$log_group" \
56+
--start-time "${DTFROM}" \
57+
--end-time "${DTTO}" \
58+
--filter-pattern "\"${search}\"" | jq --arg logGroupName "$log_group" '
59+
.events[] |
60+
.logGroupName = $logGroupName |
61+
.dt = ( (.timestamp / 1000) | strftime("%Y-%m-%d %H:%M:%S") ) + "." + ( (.timestamp % 1000 | tostring) | if length == 1 then "00" + . elif length == 2 then "0" + . else . end )
62+
')
63+
if [ ! -z "$logs" ]
64+
then
65+
results+=("${logs}")
66+
fi
67+
done
68+
69+
jsons=""
70+
for log in "${results[@]}"; do
71+
if [ -n "$jsons" ]
72+
then
73+
jsons+=$'\n'
74+
fi
75+
jsons+=$(echo "$log" | jq -r '.')
76+
done
4477

78+
echo "$jsons" | jq -s 'sort_by(.dt) | reverse'

0 commit comments

Comments
 (0)