Skip to content

Commit b4a2b59

Browse files
nyerglernyergler
committed
Run schame cache in a dedicated ECS service
1 parent 4ae1152 commit b4a2b59

File tree

7 files changed

+261
-9
lines changed

7 files changed

+261
-9
lines changed

‎terraform/modules/ecs/README.md‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -207,10 +207,12 @@ module "polytomic-ecs" {
207207
| [aws_cloudwatch_event_rule.oom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource |
208208
| [aws_cloudwatch_event_target.sns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource |
209209
| [aws_ecs_service.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
210+
| [aws_ecs_service.schemacache](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
210211
| [aws_ecs_service.sync](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
211212
| [aws_ecs_service.web](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
212213
| [aws_ecs_service.worker](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_service) | resource |
213214
| [aws_ecs_task_definition.scheduler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
215+
| [aws_ecs_task_definition.schemacache](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
214216
| [aws_ecs_task_definition.stats_reporter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
215217
| [aws_ecs_task_definition.sync](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
216218
| [aws_ecs_task_definition.web](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/ecs_task_definition) | resource |
@@ -331,10 +333,12 @@ module "polytomic-ecs" {
331333
| <a name="input_polytomic_record_log_disabled"></a> [polytomic\_record\_log\_disabled](#input\_polytomic\_record\_log\_disabled) | Globally disable record logging for this deployment | `bool` | `false` | no |
332334
| <a name="input_polytomic_resource_scheduler_cpu"></a> [polytomic\_resource\_scheduler\_cpu](#input\_polytomic\_resource\_scheduler\_cpu) | CPU units for the scheduler container | `number` | `1024` | no |
333335
| <a name="input_polytomic_resource_scheduler_memory"></a> [polytomic\_resource\_scheduler\_memory](#input\_polytomic\_resource\_scheduler\_memory) | Memory units for the scheduler container | `number` | `2048` | no |
336+
| <a name="input_polytomic_resource_schemacache_cpu"></a> [polytomic\_resource\_schemacache\_cpu](#input\_polytomic\_resource\_schemacache\_cpu) | CPU units for the schemacache container | `number` | `2048` | no |
337+
| <a name="input_polytomic_resource_schemacache_memory"></a> [polytomic\_resource\_schemacache\_memory](#input\_polytomic\_resource\_schemacache\_memory) | Memory units for the schemacache container | `number` | `4096` | no |
334338
| <a name="input_polytomic_resource_sync_count"></a> [polytomic\_resource\_sync\_count](#input\_polytomic\_resource\_sync\_count) | Number of sync containers to run | `number` | `2` | no |
335339
| <a name="input_polytomic_resource_sync_cpu"></a> [polytomic\_resource\_sync\_cpu](#input\_polytomic\_resource\_sync\_cpu) | CPU units for the sync container | `number` | `4096` | no |
336340
| <a name="input_polytomic_resource_sync_memory"></a> [polytomic\_resource\_sync\_memory](#input\_polytomic\_resource\_sync\_memory) | Memory units for the sync container | `number` | `8192` | no |
337-
| <a name="input_polytomic_resource_sync_storage"></a> [polytomic\_resource\_sync\_storage](#input\_polytomic\_resource\_sync\_storage) | Ephemeral storage for the sync container | `number` | `21` | no |
341+
| <a name="input_polytomic_resource_sync_storage"></a> [polytomic\_resource\_sync\_storage](#input\_polytomic\_resource\_sync\_storage) | Ephemeral storage for the sync container | `number` | `100` | no |
338342
| <a name="input_polytomic_resource_web_cpu"></a> [polytomic\_resource\_web\_cpu](#input\_polytomic\_resource\_web\_cpu) | CPU units for the web container | `number` | `2048` | no |
339343
| <a name="input_polytomic_resource_web_memory"></a> [polytomic\_resource\_web\_memory](#input\_polytomic\_resource\_web\_memory) | Memory units for the web container | `number` | `4096` | no |
340344
| <a name="input_polytomic_resource_worker_cpu"></a> [polytomic\_resource\_worker\_cpu](#input\_polytomic\_resource\_worker\_cpu) | CPU units for the worker container | `number` | `2048` | no |

‎terraform/modules/ecs/ecs-tasks.tf‎

Lines changed: 69 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,49 @@ resource "aws_ecs_task_definition" "worker" {
8585
}
8686
}
8787

88+
resource "aws_ecs_task_definition" "schemacache" {
89+
family = "${var.prefix}-schemacache"
90+
91+
requires_compatibilities = ["FARGATE"]
92+
network_mode = "awsvpc"
93+
cpu = var.polytomic_resource_schemacache_cpu
94+
memory = var.polytomic_resource_schemacache_memory
95+
96+
task_role_arn = aws_iam_role.polytomic_ecs_task_role.arn
97+
execution_role_arn = aws_iam_role.polytomic_ecs_execution_role.arn
98+
tags = merge(
99+
var.tags,
100+
{
101+
Name = "${var.prefix}-schemacache"
102+
})
103+
104+
105+
runtime_platform {
106+
operating_system_family = "LINUX"
107+
cpu_architecture = "X86_64"
108+
}
109+
110+
container_definitions = templatefile(
111+
"${path.module}/task-definitions/schemacache.json.tftpl",
112+
merge(local.environment,
113+
{
114+
schemacache_log_group = module.ecs_log_groups["schemacache"].cloudwatch_log_group_name
115+
}
116+
)
117+
)
118+
119+
volume {
120+
name = "polytomic"
121+
122+
efs_volume_configuration {
123+
file_system_id = module.efs.id
124+
root_directory = "/"
125+
transit_encryption = "ENABLED"
126+
transit_encryption_port = 2999
127+
}
128+
}
129+
}
130+
88131
resource "aws_ecs_task_definition" "sync" {
89132
family = "${var.prefix}-sync"
90133

@@ -194,8 +237,6 @@ resource "aws_ecs_service" "web" {
194237

195238
propagate_tags = "TASK_DEFINITION"
196239

197-
198-
199240
network_configuration {
200241
subnets = var.vpc_id == "" ? module.vpc[0].private_subnets : var.private_subnet_ids
201242
assign_public_ip = false
@@ -210,6 +251,32 @@ resource "aws_ecs_service" "web" {
210251
}
211252
}
212253

254+
resource "aws_ecs_service" "schemacache" {
255+
name = "${var.prefix}-schemacache"
256+
cluster = var.ecs_cluster_name == "" ? module.ecs[0].cluster_arn : data.aws_ecs_cluster.cluster[0].arn
257+
task_definition = aws_ecs_task_definition.worker.arn
258+
desired_count = 1
259+
260+
enable_execute_command = true
261+
platform_version = "1.4.0"
262+
263+
launch_type = "FARGATE"
264+
tags = merge(
265+
var.tags,
266+
{
267+
Name = "${var.prefix}-schemacache"
268+
})
269+
270+
271+
propagate_tags = "TASK_DEFINITION"
272+
273+
network_configuration {
274+
subnets = var.vpc_id == "" ? module.vpc[0].private_subnets : var.private_subnet_ids
275+
assign_public_ip = false
276+
security_groups = concat(var.additional_ecs_security_groups, [module.fargate_sg.security_group_id])
277+
}
278+
}
279+
213280
resource "aws_ecs_service" "worker" {
214281
name = "${var.prefix}-worker"
215282
cluster = var.ecs_cluster_name == "" ? module.ecs[0].cluster_arn : data.aws_ecs_cluster.cluster[0].arn
@@ -261,8 +328,6 @@ resource "aws_ecs_service" "sync" {
261328
}
262329
}
263330

264-
265-
266331
resource "aws_ecs_service" "scheduler" {
267332
name = "${var.prefix}-scheduler"
268333
cluster = var.ecs_cluster_name == "" ? module.ecs[0].cluster_arn : data.aws_ecs_cluster.cluster[0].arn

‎terraform/modules/ecs/logs.tf‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ module "ecs_log_groups" {
1818
source = "terraform-aws-modules/cloudwatch/aws//modules/log-group"
1919
version = "~> 3.0"
2020

21-
for_each = toset(["sync", "scheduler", "stats-reporter", "web", "worker"])
21+
for_each = toset(["sync", "scheduler", "schemacache", "stats-reporter", "web", "worker"])
2222

2323
name = "${var.prefix}-${each.key}-logs"
2424
retention_in_days = var.log_retention_days
@@ -30,4 +30,4 @@ module "ecs_log_groups" {
3030
Name = "${var.prefix}-${each.key}-logs"
3131
}
3232
)
33-
}
33+
}

‎terraform/modules/ecs/main.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ locals {
9595
sync_memory = var.polytomic_resource_sync_memory
9696
worker_memory = var.polytomic_resource_worker_memory
9797
scheduler_memory = var.polytomic_resource_scheduler_memory
98+
schemacache_memory = var.polytomic_resource_schemacache_memory
9899
image = var.polytomic_image,
99100
region = var.region,
100101
polytomic_port = var.polytomic_port,

‎terraform/modules/ecs/task-definitions/schemacache.json.tftpl‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
[{
2+
%{ if !polytomic_logger }
3+
"logConfiguration": {
4+
"logDriver": "awslogs",
5+
"secretOptions": null,
6+
"options": {
7+
"awslogs-group": "${schemacache_log_group}",
8+
"awslogs-region": "${region}",
9+
"awslogs-stream-prefix": "service"
10+
}
11+
},
12+
%{ else }
13+
"logConfiguration": {
14+
"logDriver": "splunk",
15+
"options": {
16+
"splunk-token": "test",
17+
"splunk-url": "http://localhost:8000",
18+
"splunk-verify-connection": "false"
19+
}
20+
},
21+
"dependsOn": [
22+
{
23+
"containerName": "vector",
24+
"condition": "HEALTHY"
25+
}
26+
],
27+
%{ endif }
28+
"environment": [
29+
%{ for key, value in env ~}
30+
{
31+
"name": "${key}",
32+
"value": "${value}"
33+
},
34+
%{ endfor ~}
35+
%{~ if polytomic_dd_agent }
36+
{
37+
"name": "DD_AGENT_HOST",
38+
"value": "localhost"
39+
},
40+
{
41+
"name": "METRICS",
42+
"value": "true"
43+
},
44+
%{ endif ~}
45+
{
46+
"name": "ROLE",
47+
"value": "schemacache"
48+
}
49+
],
50+
"secrets": [
51+
%{ for key, _ in secrets ~}
52+
{
53+
"name": "${key}",
54+
"valueFrom": "${task_secret_arn}:${key}::"
55+
}%{ if key != keys(secrets)[length(keys(secrets)) - 1] },%{ endif }
56+
%{ endfor ~}
57+
],
58+
"mountPoints": [
59+
{
60+
"containerPath": "${mount_path}",
61+
"sourceVolume": "polytomic"
62+
}],
63+
"image": "${image}",
64+
"portMappings": [
65+
{
66+
"containerPort": 8126,
67+
"hostPort": 8126
68+
}
69+
],
70+
"name": "schemacache",
71+
"ulimits": [
72+
{
73+
"name": "nofile",
74+
"softLimit": 1024000,
75+
"hardLimit": 1024000
76+
}
77+
]
78+
}
79+
%{~ if polytomic_logger },
80+
{
81+
"image": "${polytomic_logger_image}",
82+
"logConfiguration": {
83+
"logDriver": "awslogs",
84+
"secretOptions": null,
85+
"options": {
86+
"awslogs-group": "${schemacache_log_group}",
87+
"awslogs-region": "${region}",
88+
"awslogs-stream-prefix": "service"
89+
}
90+
},
91+
"name": "vector",
92+
"essential": true,
93+
"environment": [
94+
%{ for key, value in env ~}
95+
{
96+
"name": "${key}",
97+
"value": "${value}"
98+
},
99+
%{ endfor ~}
100+
{
101+
"name": "ROLE",
102+
"value": "logger"
103+
}
104+
],
105+
"secrets": [
106+
%{ for key, _ in support_secrets ~}
107+
{
108+
"name": "${key}",
109+
"valueFrom": "${task_secret_arn}:${key}::"
110+
}%{ if key != keys(support_secrets)[length(keys(support_secrets)) - 1] },%{ endif }
111+
%{ endfor ~}
112+
],
113+
"healthCheck": {
114+
"command": [
115+
"CMD-SHELL",
116+
"wget --no-verbose --tries=1 --spider http://localhost:8000/services/collector/health > /dev/null || exit 1"
117+
],
118+
"interval":30,
119+
"retries":3,
120+
"startPeriod":100,
121+
"timeout":5
122+
}
123+
}
124+
%{ endif }
125+
%{~ if polytomic_dd_agent },
126+
{
127+
"name": "datadog-agent",
128+
"image": "${polytomic_dd_agent_image}",
129+
"essential": true,
130+
"environment": [
131+
%{ for key, value in env ~}
132+
{
133+
"name": "${key}",
134+
"value": "${value}"
135+
},
136+
%{ endfor ~}
137+
{
138+
"name": "DD_API_KEY",
139+
"value": "ENC[DD_API_KEY]"
140+
},
141+
{
142+
"name": "DD_SECRET_BACKEND_COMMAND",
143+
"value": "/bin/ptctl"
144+
},
145+
{
146+
"name": "DD_SECRET_BACKEND_ARGUMENTS",
147+
"value": "config get -f vector -"
148+
},
149+
{
150+
"name": "DD_DOCKER_ENV_AS_TAGS",
151+
"value": "{\\\"role\\\": \\\"polytomic.role\\\"}"
152+
},
153+
{
154+
"name": "DD_APM_ENABLED",
155+
"value": "true"
156+
},
157+
{
158+
"name": "ECS_FARGATE",
159+
"value": "true"
160+
}
161+
],
162+
"secrets": [
163+
%{ for key, _ in support_secrets ~}
164+
{
165+
"name": "${key}",
166+
"valueFrom": "${task_secret_arn}:${key}::"
167+
}%{ if key != keys(support_secrets)[length(keys(support_secrets)) - 1] },%{ endif }
168+
%{ endfor ~}
169+
]
170+
}
171+
%{ endif }
172+
]

‎terraform/modules/ecs/task-definitions/worker.json.tftpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
%{ endif ~}
4545
{
4646
"name": "ROLE",
47-
"value": "worker,schemacache"
47+
"value": "worker"
4848
}
4949
],
5050
"secrets": [

‎terraform/modules/ecs/vars.tf‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,16 @@ variable "polytomic_resource_scheduler_memory" {
157157
default = 2048 // 2 GB
158158
}
159159

160+
variable "polytomic_resource_schemacache_cpu" {
161+
description = "CPU units for the schemacache container"
162+
default = 2048
163+
}
164+
165+
variable "polytomic_resource_schemacache_memory" {
166+
description = "Memory units for the schemacache container"
167+
default = 4096 // 4 GB
168+
}
169+
160170
variable "polytomic_resource_sync_count" {
161171
description = "Number of sync containers to run"
162172
default = 2
@@ -174,7 +184,7 @@ variable "polytomic_resource_sync_memory" {
174184

175185
variable "polytomic_resource_sync_storage" {
176186
description = "Ephemeral storage for the sync container"
177-
default = 21
187+
default = 100 // 100 GB
178188
}
179189

180190
variable "polytomic_mssql_tx_isolation" {

0 commit comments

Comments
 (0)