Skip to content

Commit 9803b96

Browse files
lukey-lukelukey-luke
committed
Merge remote-tracking branch 'origin/ls/task-dpc-5227-sweeper-lambda' into ls/task-dpc-5227-sweeper-lambda 
2 parents 8c043cc + fbaf578 commit 9803b96

File tree

5 files changed

+176
-2
lines changed

5 files changed

+176
-2
lines changed

‎scripts/setcloudwatchretention.sh‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
AWS_REGION="${AWS_REGION:=us-east-1}"
6+
RETENTION_DAYS="${RETENTION_DAYS:-180}"
7+
DRY_RUN="${DRY_RUN:-true}"
8+
9+
DATE_TAG=$(date +%Y%m%d-%H%M%S)
10+
PLAN_FILE="retention-plan-${DATE_TAG}.sh"
11+
REPORT_FILE="retention-report-${DATE_TAG}.csv"
12+
13+
PROCESSED=0
14+
IGNORED_CMS=()
15+
UPDATED=()
16+
SKIPPED=()
17+
IGNORED=()
18+
TF_MAINTAINED=()
19+
20+
# These log groups are excluded because their values are set via Terraform.
21+
# Defined explicitly because some Terraform managed log groups do not have retention set via Terraform
22+
EXCLUSION_LIST=(
23+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_job_data"
24+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-bfd-insights-get_stale_pending_jobs"
25+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_active_acos"
26+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_stale_cclf_imports"
27+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_num_benes_per_aco"
28+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_suppression_metrics"
29+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_num_days_to_make_first_request"
30+
"/aws/kinesisfirehose/bfd-insights-bcda-prod-get_acos_with_expired_credentials"
31+
"aws/kinesisfirehose/bfd-insights-bcda-prod-get_job_data"
32+
"/aws/lambda/insights_data_sampler_prod"
33+
"/aws/lambda/ab2d-prod-audit"
34+
"/aws/lambda/ab2d-sandbox-audit"
35+
"/aws/lambda/ab2d-test-audit"
36+
"/aws/lambda/ab2d-dev-audit"
37+
"/aws/events/ecs/dpc-prod-backend"
38+
"/aws/events/ecs/dpc-prod-frontend"
39+
)
40+
41+
if [[ "$DRY_RUN" == "true" ]]; then
42+
echo "#!/usr/bin/env bash" > "$PLAN_FILE"
43+
echo "set -euo pipefail" >> $PLAN_FILE
44+
echo "" >> "$PLAN_FILE"
45+
fi
46+
47+
echo "REGION: $AWS_REGION"
48+
echo "Target retention: "
49+
echo "Dry run: $DRY_RUN"
50+
echo "Plan file: $PLAN_FILE"
51+
echo "Report file: $REPORT_FILE"
52+
echo "--------------------------------------"
53+
54+
LOG_GROUPS_JSON=$(aws logs describe-log-groups --region "$AWS_REGION" --output json)
55+
while IFS=$'\t' read -r NAME RETENTION; do
56+
57+
((PROCESSED++))
58+
59+
LOWER_NAME=$(echo "$NAME" | tr '[:upper:]' '[:lower:]')
60+
echo "Evaluating: $LOWER_NAME"
61+
62+
EXCLUDE=false
63+
64+
for excluded_group_name in "${EXCLUSION_LIST[@]}"; do
65+
if [[ "$excluded_group_name" == "$LOWER_NAME" ]]; then
66+
EXCLUDE=true
67+
break # Exit the loop once a match is found
68+
fi
69+
done
70+
71+
if [[ "$EXCLUDE" == true ]] ; then
72+
TF_MAINTAINED+=("$NAME $RETENTION")
73+
continue
74+
fi
75+
76+
if [[ "$LOWER_NAME" == *dev* ]] || \
77+
[[ "$LOWER_NAME" == *test* ]] || \
78+
[[ "$LOWER_NAME" == *sandbox* ]]; then
79+
IGNORED+=("$NAME $RETENTION")
80+
echo "IGNORING (uncovered environment) ${LOWER_NAME}"
81+
continue
82+
fi
83+
84+
if echo "$LOWER_NAME" | grep -iq "cms-cloud"; then
85+
IGNORED_CMS+=("$NAME")
86+
echo "IGNORING (cms managed) ${LOWER_NAME}"
87+
continue
88+
fi
89+
90+
if [[ "$RETENTION" == "null" ]]; then
91+
RETENTION=0
92+
fi
93+
94+
if [[ "$RETENTION" -ge "$RETENTION_DAYS" ]]; then
95+
SKIPPED+=("$NAME $RETENTION")
96+
echo "SKIPPING (retention sufficient) ${LOWER_NAME} "
97+
continue
98+
fi
99+
100+
CMD=(
101+
aws logs put-retention-policy \
102+
--log-group-name "$LOWER_NAME" \
103+
--retention-in-days "$RETENTION"
104+
)
105+
106+
if [[ "$DRY_RUN" == "true" ]]; then
107+
printf '%q ' "${CMD[@]}" >> "$PLAN_FILE"
108+
echo >> "$PLAN_FILE"
109+
else
110+
read -p "Run retention update for: $LOWER_NAME (current: ${RETENTION:-unset})? [y/N] " CONFIRM
111+
if [[ "$CONFIRM" =~ ^[Yy]$ ]]; then
112+
if "${CMD[@]}"; then
113+
echo "UPDATED $LOWER_NAME"
114+
UPDATED+=("$LOWER_NAME")
115+
else
116+
echo "Error updating $LOWER_NAME"
117+
fi
118+
else
119+
echo "SKIPPING: $LOWER_NAME"
120+
SKIPPED+=("$LOWER_NAME")
121+
fi
122+
fi
123+
124+
125+
done < <(jq -r '.logGroups[] | "\(.logGroupName)\t\(.retentionInDays // "null")"' <<< "$LOG_GROUPS_JSON")
126+
127+
# Summary #
128+
129+
echo "----------------------"
130+
echo "ALL UPDATED: (${#UPDATED[@]})"
131+
printf " %s\n" "${UPDATED[@]:-}"
132+
133+
echo "----------------------"
134+
echo "MAINTAINED BY TF AND SKIPPED: (${#TF_MAINTAINED[@]})"
135+
printf " %s\n" "${TF_MAINTAINED[@]:-}"
136+
137+
echo "----------------------"
138+
echo "ALL OTHERS SKIPPED: (${#SKIPPED[@]})"
139+
printf " %s\n" "${SKIPPED[@]:-}"
140+
141+
echo "----------------------"
142+
echo "CMS MANAGED AND IGNORED: (${#IGNORED_CMS[@]})"
143+
printf " %s\n" "${IGNORED_CMS[@]:-}"
144+
145+
echo "----------------------"
146+
echo "ALL IGNORED: (${#IGNORED[@]})"
147+
printf " %s\n" "${IGNORED[@]:-}"
148+
149+
echo "total processed: $PROCESSED"

‎terraform/modules/service/main.tf‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,11 @@ resource "aws_ecs_task_definition" "this" {
3333
}
3434
]))
3535

36+
runtime_platform {
37+
operating_system_family = "LINUX"
38+
cpu_architecture = var.cpu_architecture
39+
}
40+
3641
dynamic "volume" {
3742
for_each = var.volumes != null ? toset(var.volumes) : toset([])
3843

‎terraform/modules/service/variables.tf‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,12 @@ variable "image" {
5555
type = string
5656
}
5757

58+
variable "cpu_architecture" {
59+
description = "The cpu architecture needed."
60+
type = string
61+
default = "X86_64"
62+
}
63+
5864
variable "load_balancers" {
5965
description = "Load balancer(s) for use by the AWS ECS service."
6066
type = list(object({

‎terraform/services/insights/views/bcda/prod-jobs-with-cms-id.view.sql‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,14 @@
11
-- Get all jobs with CMS ID added from acos table
22
CREATE OR REPLACE VIEW bcda_prod_jobs_with_cms_id AS
33
SELECT
4-
jobs.*,
4+
jobs.id,
5+
jobs.aco_id,
6+
jobs.request_url,
7+
jobs.status,
8+
jobs.created_at,
9+
jobs.updated_at,
10+
jobs.job_count,
11+
jobs.transaction_time,
512
acos.cms_id
613
FROM jobs
714
LEFT JOIN acos ON acos.uuid = jobs.aco_id;

‎terraform/services/insights/views/bcda/sandbox-jobs-with-cms-id.view.sql‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,14 @@
22
-- This should be run in SANDBOX only!
33
CREATE OR REPLACE VIEW bcda_sandbox_jobs_with_cms_id AS
44
SELECT
5-
jobs.*,
5+
jobs.id,
6+
jobs.aco_id,
7+
jobs.request_url,
8+
jobs.status,
9+
jobs.created_at,
10+
jobs.updated_at,
11+
jobs.job_count,
12+
jobs.transaction_time,
613
acos.cms_id
714
FROM jobs
815
LEFT JOIN acos ON acos.uuid = jobs.aco_id;

0 commit comments

Comments
 (0)