Full featured graylog setup

 * Separate MongoDB
 * Separate ElasticSearch

Author: knorrus

modules/elasticsearch/ec2.tf

@@ -4,32 +4,32 @@ resource "aws_security_group" "elasticsearch_sg" {
 
   # Elasticsearch native transport protocol
   ingress {
-    from_port = 9300
-    to_port = 9300
+    from_port = "${var.elasticsearch_native_port}"
+    to_port = "${var.elasticsearch_native_port}"
     protocol = "tcp"
     cidr_blocks = ["${var.trusted_networks}"]
   }
 
   # Elasticsearch HTTP service
   ingress {
-    from_port = 9200
-    to_port = 9200
+    from_port = "${var.elasticsearch_http_port}"
+    to_port = "${var.elasticsearch_http_port}"
     protocol = "tcp"
     cidr_blocks = ["${var.trusted_networks}"]
   }
 
   # Elasticsearch native transport protocol
   egress {
-    from_port = 9300
-    to_port = 9300
+    from_port = "${var.elasticsearch_native_port}"
+    to_port = "${var.elasticsearch_native_port}"
     protocol = "tcp"
     cidr_blocks = ["${var.trusted_networks}"]
   }
 
   # Elasticsearch HTTP service
   egress {
-    from_port = 9200
-    to_port = 9200
+    from_port = "${var.elasticsearch_http_port}"
+    to_port = "${var.elasticsearch_http_port}"
     protocol = "tcp"
     cidr_blocks = ["${var.trusted_networks}"]
   }
@@ -40,27 +40,29 @@ resource "aws_security_group" "elasticsearch_sg" {
   }
 }
 
-data "template_file" "elasticsearch_master_cloudconfig" {
+data "template_file" "master_node_cloudconfig" {
   template = "${file("${path.module}/resources/userdata.tpl")}"
   vars {
-    configuration_script = "${base64encode(file("${path.module}/../resources/install-unix-tools.sh"))}"
-    cluster_name = "${var.ecs_cluster_name}"
     cluster_role = "elasticsearch-master"
-    host_name = "${lower(var.verbose_name)}-elasticsearch"
-    volume_device = "${var.data_volume_device}"
+    cluster_name = "${var.ecs_cluster_name}"
+    instance_group = "${var.ecs_instance_group}"
+    host_name = "${lower(var.env_name)}-elasticsearch-master"
     volume_path = "${var.data_volume_path}"
+    volume_device = "${var.data_volume_device}"
+    configuration_script = "${base64encode(file("${path.module}/../resources/install-unix-tools.sh"))}"
   }
 }
 
-data "template_file" "elasticsearch_data_cloudconfig" {
+data "template_file" "data_node_cloudconfig" {
   template = "${file("${path.module}/resources/userdata.tpl")}"
   vars {
-    configuration_script = "${base64encode(file("${path.module}/../resources/install-unix-tools.sh"))}"
-    cluster_name = "${var.ecs_cluster_name}"
     cluster_role = "elasticsearch-data"
-    host_name = "${lower(var.verbose_name)}-elasticsearch"
-    volume_device = "${var.data_volume_device}"
+    cluster_name = "${var.ecs_cluster_name}"
+    instance_group = "${var.ecs_instance_group}"
+    host_name = "${lower(var.env_name)}-elasticsearch-data"
     volume_path = "${var.data_volume_path}"
+    volume_device = "${var.data_volume_device}"
+    configuration_script = "${base64encode(file("${path.module}/../resources/install-unix-tools.sh"))}"
   }
 }
 
@@ -76,7 +78,7 @@ resource "aws_instance" "elasticsearch_master_instance" {
   source_dest_check = false
   disable_api_termination = "${var.enable_termination_protection}"
   instance_initiated_shutdown_behavior = "stop"
-  user_data = "${data.template_file.elasticsearch_master_cloudconfig.rendered}"
+  user_data = "${data.template_file.master_node_cloudconfig.rendered}"
   tags {
     Env = "${var.env_name}"
     Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Master-Zone${count.index}"
@@ -85,7 +87,6 @@ resource "aws_instance" "elasticsearch_master_instance" {
 
 resource "aws_instance" "elasticsearch_data_instance" {
   count = "${var.data_nodes_count}"
-  depends_on = ["aws_ebs_volume.elasticsearch_data_volume"]
   ami = "${var.instance_ami}"
   instance_type = "${var.data_instance_type}"
   subnet_id = "${element(var.vpc_subnets, count.index)}"
@@ -96,7 +97,7 @@ resource "aws_instance" "elasticsearch_data_instance" {
   source_dest_check = false
   disable_api_termination = "${var.enable_termination_protection}"
   instance_initiated_shutdown_behavior = "stop"
-  user_data = "${data.template_file.elasticsearch_data_cloudconfig.rendered}"
+  user_data = "${data.template_file.data_node_cloudconfig.rendered}"
   tags {
     Env = "${var.env_name}"
     Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Data-Zone${count.index}"
@@ -107,20 +108,19 @@ resource "aws_ebs_volume" "elasticsearch_data_volume" {
   count = "${var.data_nodes_count}"
   availability_zone = "${element(var.availability_zones, count.index)}"
   size = "${var.data_instance_storage_size}"
-
   tags {
     Env = "${var.env_name}"
-    Name = "${var.env_name}-${var.verbose_name}-Elasticseach-Volume-Zone${count.index}"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Data-Volume-Zone${count.index}"
   }
 }
 
 resource "aws_ebs_volume" "elasticsearch_master_volume" {
   count = "${var.master_nodes_count}"
   availability_zone = "${element(var.availability_zones, count.index)}"
-  size = 10
+  size = "${var.master_instance_storage_size}"
   tags {
     Env = "${var.env_name}"
-    Name = "${var.env_name}-${var.verbose_name}-Elasticseach-Volume-Zone${count.index}"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Master-Volume-Zone${count.index}"
   }
 }
 
@@ -140,23 +140,24 @@ resource "aws_volume_attachment" "elasticsearch_master_volume_attachement" {
   instance_id = "${element(aws_instance.elasticsearch_master_instance.*.id, count.index)}"
 }
 
+
 data "aws_route53_zone" "local" {
   zone_id = "${var.vpc_dns_zone_id}"
 }
 
-resource "aws_route53_record" "elasticsearch_master_node_dns_records" {
-  count = "${var.master_nodes_count}"
+resource "aws_route53_record" "elasticsearch_master_record" {
+  count = "${var.master_nodes_count > 0 ? 1 : 0}"
   zone_id = "${var.vpc_dns_zone_id}"
-  name = "${var.master_nodes_count == 1 ? format("elasticsearch.master.%s", data.aws_route53_zone.local.name) : format("elasticsearch.master%d.%s", count.index, data.aws_route53_zone.local.name)}"
+  name = "elasticsearch.master.${data.aws_route53_zone.local.name}",
   type = "A"
   ttl = "60"
-  records = ["${element(aws_instance.elasticsearch_master_instance.*.private_ip, 0)}"]
+  records = ["${aws_instance.elasticsearch_master_instance.*.private_ip}"]
 }
 
-resource "aws_route53_record" "elasticsearch_data_node_dns_records" {
-  count = "${var.data_nodes_count}"
+resource "aws_route53_record" "elasticsearch_data_record" {
+  count = "${var.data_nodes_count > 0 ? 1 : 0}"
   zone_id = "${var.vpc_dns_zone_id}"
-  name = "${var.data_nodes_count == 1 ? format("elasticsearch.%s", data.aws_route53_zone.local.name) : format("elasticsearch%d.%s", count.index, data.aws_route53_zone.local.name)}"
+  name = "elasticsearch.${data.aws_route53_zone.local.name}",
   type = "A"
   ttl = "60"
   records = ["${element(aws_instance.elasticsearch_data_instance.*.private_ip, 0)}"]

modules/elasticsearch/ecs.tf

@@ -1,65 +1,67 @@
-resource "aws_ecs_cluster" "elasticsearch_cluster" {
-  name = "${var.ecs_cluster_name}"
+data "aws_ecs_cluster" "ecs_cluster" {
+  cluster_name = "${var.ecs_cluster_name}"
 }
 
 resource "aws_ecs_task_definition" "elasticsearch_master_task" {
   family = "${lower(var.env_name)}-elasticsearch-master"
   container_definitions = "${data.template_file.elasticsearch_master_config.rendered}"
   volume {
-    name = "elasticseach-data"
-    host_path = "${var.data_volume_path}"
+    name = "elasticsearch-data"
+    host_path = "${var.data_volume_path}/elasticsearch-data"
   }
 }
 
 resource "aws_ecs_task_definition" "elasticsearch_data_task" {
   family = "${lower(var.env_name)}-elasticsearch-data"
   container_definitions = "${data.template_file.elasticsearch_data_config.rendered}"
   volume {
-    name = "elasticseach-data"
-    host_path = "${var.data_volume_path}"
+    name = "elasticsearch-data"
+    host_path = "${var.data_volume_path}/elasticsearch-data"
   }
 }
 
 resource "aws_ecs_service" "elasticsearch_master_service" {
-  depends_on = ["aws_iam_role_policy.docker_policy", "aws_ecs_task_definition.elasticsearch_master_task"]
-  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch-master"
-  cluster = "${aws_ecs_cluster.elasticsearch_cluster.id}"
+  name = "${lower(var.env_name)}-elasticsearch-master"
+  desired_count = "${var.master_tasks_count}"
+  cluster = "${data.aws_ecs_cluster.ecs_cluster.id}"
   task_definition = "${aws_ecs_task_definition.elasticsearch_master_task.arn}"
-  desired_count = "${var.master_nodes_count}"
+  /* Place only of dedicated instances */
   placement_constraints {
     type = "memberOf"
     expression = "attribute:cluster_role == elasticsearch-master"
   }
+  depends_on = ["aws_iam_role_policy.docker_policy", "aws_ecs_task_definition.elasticsearch_master_task"]
 }
 
 resource "aws_ecs_service" "elasticsearch_data_service" {
-  depends_on = ["aws_iam_role_policy.docker_policy", "aws_ecs_task_definition.elasticsearch_master_task"]
-  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch-data"
-  cluster = "${aws_ecs_cluster.elasticsearch_cluster.id}"
+  name = "${lower(var.env_name)}-elasticsearch-data"
+  desired_count = "${var.data_tasks_count}"
+  cluster = "${data.aws_ecs_cluster.ecs_cluster.id}"
   task_definition = "${aws_ecs_task_definition.elasticsearch_data_task.arn}"
-  desired_count = "${var.data_nodes_count}"
   placement_constraints {
     type = "memberOf"
-    expression = "attribute:cluster_role == elasticsearch-data"
+    expression = "attribute:group == ${var.ecs_instance_group}"
   }
+  depends_on = ["aws_iam_role_policy.docker_policy", "aws_ecs_task_definition.elasticsearch_master_task"]
 }
 
 data "template_file" "elasticsearch_master_config" {
   template = "${file("${path.module}/resources/elasticsearch.json")}"
   vars {
     elasticsearch_version = "${var.elasticsearch_version}"
     container_name = "elasticsearch-master"
-    container_memory = 512
-    native_transport_port = 9300
-    http_transport_port = 9200
+    container_memory = "${var.master_memory_limit}"
+    native_transport_port = "${var.elasticsearch_native_port}"
+    http_transport_port = "${var.elasticsearch_http_port}"
     cluster_name = "${var.elasticsearch_cluster_name}"
     node_name = "${lower(var.verbose_name)}-elasticsearch-master"
     is_master = "true"
     is_data = "false"
-    min_master_nodes = "${var.master_nodes_count == 0 ? (var.master_nodes_count / 2) + 1 : 0}"
-    master_nodes_addresses = ""
-    heap_size = 256
-    volume_name = "elasticseach-data"
+    num_shards = "${var.elasticsearch_num_shards}"
+    num_replicas = "${var.elasticsearch_num_replicas}"
+    heap_size = "${var.elasticsearch_memory_limit / 2}"
+    volume_name = "elasticsearch-data"
+    extra-options = "-Ddiscovery.zen.minimum_master_nodes=${(var.master_nodes_count / 2) + 1 }"
   }
 }
 
@@ -69,15 +71,16 @@ data "template_file" "elasticsearch_data_config" {
     elasticsearch_version = "${var.elasticsearch_version}"
     container_name = "elasticsearch-data"
     container_memory = "${var.elasticsearch_memory_limit}"
-    native_transport_port = 9300
-    http_transport_port = 9200
+    native_transport_port = "${var.elasticsearch_native_port}"
+    http_transport_port = "${var.elasticsearch_http_port}"
     cluster_name = "${var.elasticsearch_cluster_name}"
     node_name = "${lower(var.verbose_name)}-elasticsearch-data"
     is_master = "${var.is_data_nodes_master_eiligible == 1 ? "true" : "false"}"
     is_data = "true"
-    master_nodes_addresses = "${join(", ", concat(var.external_masters_addresses, aws_route53_record.elasticsearch_master_node_dns_records.*.name))}"
-    min_master_nodes = "${(var.master_nodes_count / 2) + 1 }"
+    num_shards = "${var.elasticsearch_num_shards}"
+    num_replicas = "${var.elasticsearch_num_replicas}"
     heap_size = "${var.elasticsearch_memory_limit / 2}"
-    volume_name = "elasticseach-data"
+    volume_name = "elasticsearch-data"
+    extra-options = "-Ddiscovery.zen.ping.unicast.hosts=${join(", ", concat(var.external_masters_addresses, formatlist("$s:%s", aws_route53_record.elasticsearch_master_record.*.name, var.elasticsearch_native_port)))} -Ddiscovery.zen.minimum_master_nodes=${(var.master_nodes_count / 2) + 1 }"
   }
 }

modules/elasticsearch/iam.tf

@@ -1,10 +1,10 @@
 resource "aws_iam_instance_profile" "elasticsearch" {
-  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch"
+  name = "${lower(var.env_name)}-elasticsearch"
   role = "${aws_iam_role.elasticsearch_role.name}"
 }
 
 resource "aws_iam_role" "elasticsearch_role" {
-  name = "${lower(var.env_name)}-${var.verbose_name}-elasticsearch"
+  name = "${lower(var.env_name)}-elasticsearch"
   assume_role_policy = "${data.aws_iam_policy_document.ec2_assume_policy.json}"
 }
 
@@ -50,7 +50,6 @@ data "aws_iam_policy_document" "docker_policy" {
       "ecr:BatchGetImage",
       "logs:CreateLogStream",
       "logs:PutLogEvents",
-
       "ec2:AuthorizeSecurityGroupIngress",
       "ec2:Describe*",
       "elasticloadbalancing:DeregisterInstancesFromLoadBalancer",

modules/elasticsearch/output.tf

@@ -1,19 +1,29 @@
-output "private_ips" {
-  value = ["${aws_instance.elasticsearch_data_instance.*.private_ip}"]
+output "elasticsearch_http_address" {
+  value = "${var.master_nodes_count > 0  ? format("http://%s:%s", aws_route53_record.elasticsearch_master_record.name, var.elasticsearch_http_port) : format("http://%s:%s", aws_route53_record.elasticsearch_data_record.name, var.elasticsearch_http_port)}"
+}
+
+output "elasticsearch_native_address" {
+  value = "${var.master_nodes_count > 0  ? format("%s:%s", aws_route53_record.elasticsearch_master_record.name, var.elasticsearch_native_port) : format("%s:%s", aws_route53_record.elasticsearch_data_record.name, var.elasticsearch_native_port)}"
 }
 
-output "elasticseacrch_master_addresses" {
-  value = "${formatlist("%s:%s", aws_route53_record.elasticsearch_master_node_dns_records.*.name)}"
+output "master_instance_ids" {
+  value = ["${aws_instance.elasticsearch_master_instance.*.id}"]
 }
 
-output "elasticseacrch_node_addresses" {
-  value = "${formatlist("%s:%s", aws_route53_record.elasticsearch_data_node_dns_records.*.name)}"
+
+output "master_instance_ips" {
+  value = ["${aws_instance.elasticsearch_master_instance.*.private_ip}"]
 }
 
-output "instance_ids" {
+output "data_instance_ids" {
   value = ["${aws_instance.elasticsearch_data_instance.*.id}"]
 }
 
+
+output "data_instance_ips" {
+  value = ["${aws_instance.elasticsearch_data_instance.*.private_ip}"]
+}
+
 output "elasticsearch_sg_id" {
   value = "${aws_security_group.elasticsearch_sg.id}"
 }