Added elasticsearch module

Author: knorrus

modules/elasticsearch/ec2.tf

@@ -1,107 +1,163 @@
-resource "aws_instance" "elasticsearch_instance" {
-  count = "${var.instances_count}"
-  depends_on = ["aws_ebs_volume.elasticsearch_volume"]
-  ami = "${var.ami_id}"
-  instance_type = "${var.instance_type}"
-  subnet_id = "${element(var.vpc_subnets, count.index)}"
-  key_name = "${var.instance_key_name}"
-  iam_instance_profile = "${aws_iam_instance_profile.elasticsearch.name}"
-  vpc_security_group_ids = ["${concat(var.security_groups, list(aws_security_group.elasticsearch.id))}"]
-  associate_public_ip_address = false
-  source_dest_check = false
-  disable_api_termination = "${var.enable_termination_protection}"
-  instance_initiated_shutdown_behavior = "stop"
-
-  tags {
-    Env = "${var.env_name}"
-    Name = "${var.env_name}: ${var.verbose_name} Elasticsearch ${count.index}"
-  }
-  user_data = <<USER_DATA_END
-#cloud-config
-write_files:
-- path: /usr/bin/install-unix-tools
-  encoding: b64
-  content: ${base64encode(file("${path.module}/../resources/install-unix-tools.sh"))}
-  owner: root:root
-  permissions: '0755'
-- path: /etc/dive-in-docker.conf
-  content: elasticsearch
-- path: /etc/ecs/ecs.config
-  content: |
-    ECS_CLUSTER=${var.ecs_cluster_name}
-    ECS_AVAILABLE_LOGGING_DRIVERS=["json-file","syslog","journald","gelf","awslogs"]
-- path: /etc/sysctl.d/01-elasticsearch.conf
-  content: |
-    :syslogtag, startswith, "${var.syslog_tag_prefix}" /var/log/${var.docker_log_file_name}
-runcmd:
-  - [ cloud-init-per, once, "install-unix-tools", "install-unix-tools", "-t", "1.0", "full"]
-  - [ cloud-init-per, once, "set-hostname", "aws-set-hostname", "${lower(var.verbose_name)}-elasticsearch-{count.index}", "-s"]
-  - [ cloud-init-per, once, "read-custom-syslog", "sysctl", "-p", "/etc/sysctl.d/01-elasticsearch.conf"]
-  - [ cloud-init-per, once, "docker-stop", "service", "docker", "stop"]
-  - [ cloud-init-per, once, "mount-ebs", "mount-ebs", "${var.data_volume_device}", "${var.data_volume_path}", "0777" ]
-  - [ cloud-init-per, once, "docker-start", "service", "docker", "start"]
-  - [ cloud-init-per, once, "start-ecs", "start", "ecs"]
-USER_DATA_END
-}
-
-resource "aws_ebs_volume" "elasticsearch_volume" {
-  count = "${length(var.instances_count)}"
-  availability_zone = "${element(var.availability_zones, count.index)}"
-  size = "${var.storage_size}"
-
-  tags {
-    Env = "${var.env_name}"
-    Name = "${var.env_name}: ${var.verbose_name} Elasticseach Volume ${count.index}"
-  }
-}
-
-resource "aws_volume_attachment" "elasticsearch_volume_attachement" {
-  count = "${length(var.instances_count)}"
-  device_name = "${var.data_volume_device}"
-  force_detach = true
-  volume_id = "${element(aws_ebs_volume.elasticsearch_volume.*.id, count.index)}"
-  instance_id = "${element(aws_instance.elasticsearch_instance.*.id, count.index)}"
-}
-
-
-resource "aws_security_group" "elasticsearch" {
-  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch"
+resource "aws_security_group" "elasticsearch_sg" {
+  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch-access"
   vpc_id = "${var.vpc_id}"
 
   # Elasticsearch native transport protocol
   ingress {
     from_port = 9300
     to_port = 9300
     protocol = "tcp"
-    cidr_blocks = ["${var.native_trusted_networks}"]
+    cidr_blocks = ["${var.trusted_networks}"]
   }
 
   # Elasticsearch HTTP service
   ingress {
     from_port = 9200
     to_port = 9200
     protocol = "tcp"
-    cidr_blocks = ["${var.http_trusted_networks}"]
+    cidr_blocks = ["${var.trusted_networks}"]
   }
 
   # Elasticsearch native transport protocol
   egress {
     from_port = 9300
     to_port = 9300
     protocol = "tcp"
-    cidr_blocks = ["${var.native_trusted_networks}"]
+    cidr_blocks = ["${var.trusted_networks}"]
   }
 
   # Elasticsearch HTTP service
   egress {
     from_port = 9200
     to_port = 9200
     protocol = "tcp"
-    cidr_blocks = ["${var.http_trusted_networks}"]
+    cidr_blocks = ["${var.trusted_networks}"]
   }
 
   tags {
     Env = "${var.env_name}"
-    Name = "${var.env_name}: ${var.verbose_name} Elasticsearch"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Access"
+  }
+}
+
+data "template_file" "elasticsearch_master_cloudconfig" {
+  template = "${file("${path.module}/resources/userdata.tpl")}"
+  vars {
+    configuration_script = "${base64encode(file("${path.module}/resources/install-unix-tools.sh"))}"
+    cluster_name = "${var.ecs_cluster_name}"
+    cluster_role = "elasticsearch-master"
+    host_name = "${lower(var.verbose_name)}-elasticsearch"
+    volume_device = "${var.data_volume_device}"
+    volume_path = "${var.data_volume_path}"
+  }
+}
+
+data "template_file" "elasticsearch_data_cloudconfig" {
+  template = "${file("${path.module}/resources/userdata.tpl")}"
+  vars {
+    configuration_script = "${base64encode(file("${path.module}/resources/install-unix-tools.sh"))}"
+    cluster_name = "${var.ecs_cluster_name}"
+    cluster_role = "elasticsearch-data"
+    host_name = "${lower(var.verbose_name)}-elasticsearch"
+    volume_device = "${var.data_volume_device}"
+    volume_path = "${var.data_volume_path}"
   }
 }
+
+resource "aws_instance" "elasticsearch_master_instance" {
+  count = "${var.master_nodes_count}"
+  ami = "${var.instance_ami}"
+  instance_type = "${var.master_instance_type}"
+  subnet_id = "${element(var.vpc_subnets, count.index)}"
+  key_name = "${var.instance_key_name}"
+  iam_instance_profile = "${aws_iam_instance_profile.elasticsearch.name}"
+  vpc_security_group_ids = ["${concat(var.vpc_security_groups, list(aws_security_group.elasticsearch_sg.id))}"]
+  associate_public_ip_address = false
+  source_dest_check = false
+  disable_api_termination = "${var.enable_termination_protection}"
+  instance_initiated_shutdown_behavior = "stop"
+  user_data = "${data.template_file.elasticsearch_master_cloudconfig.rendered}"
+  tags {
+    Env = "${var.env_name}"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Master-Zone${count.index}"
+  }
+}
+
+resource "aws_instance" "elasticsearch_data_instance" {
+  count = "${var.data_nodes_count}"
+  depends_on = ["aws_ebs_volume.elasticsearch_data_volume"]
+  ami = "${var.instance_ami}"
+  instance_type = "${var.data_instance_type}"
+  subnet_id = "${element(var.vpc_subnets, count.index)}"
+  key_name = "${var.instance_key_name}"
+  iam_instance_profile = "${aws_iam_instance_profile.elasticsearch.name}"
+  vpc_security_group_ids = ["${concat(var.vpc_security_groups, list(aws_security_group.elasticsearch_sg.id))}"]
+  associate_public_ip_address = false
+  source_dest_check = false
+  disable_api_termination = "${var.enable_termination_protection}"
+  instance_initiated_shutdown_behavior = "stop"
+  user_data = "${data.template_file.elasticsearch_data_cloudconfig.rendered}"
+  tags {
+    Env = "${var.env_name}"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticsearch-Data-Zone${count.index}"
+  }
+}
+
+resource "aws_ebs_volume" "elasticsearch_data_volume" {
+  count = "${var.data_nodes_count}"
+  availability_zone = "${element(var.availability_zones, count.index)}"
+  size = "${var.data_instance_storage_size}"
+
+  tags {
+    Env = "${var.env_name}"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticseach-Volume-Zone${count.index}"
+  }
+}
+
+resource "aws_ebs_volume" "elasticsearch_master_volume" {
+  count = "${var.master_nodes_count}"
+  availability_zone = "${element(var.availability_zones, count.index)}"
+  size = 10
+  tags {
+    Env = "${var.env_name}"
+    Name = "${var.env_name}-${var.verbose_name}-Elasticseach-Volume-Zone${count.index}"
+  }
+}
+
+resource "aws_volume_attachment" "elasticsearch_data_volume_attachement" {
+  count = "${var.data_nodes_count}"
+  device_name = "${var.data_volume_device}"
+  force_detach = true
+  volume_id = "${element(aws_ebs_volume.elasticsearch_data_volume.*.id, count.index)}"
+  instance_id = "${element(aws_instance.elasticsearch_data_instance.*.id, count.index)}"
+}
+
+resource "aws_volume_attachment" "elasticsearch_master_volume_attachement" {
+  count = "${var.master_nodes_count}"
+  device_name = "${var.data_volume_device}"
+  force_detach = true
+  volume_id = "${element(aws_ebs_volume.elasticsearch_master_volume.*.id, count.index)}"
+  instance_id = "${element(aws_instance.elasticsearch_master_instance.*.id, count.index)}"
+}
+
+data "aws_route53_zone" "local" {
+  zone_id = "${var.vpc_dns_zone_id}"
+}
+
+resource "aws_route53_record" "elasticsearch_master_node_dns_records" {
+  count = "${var.master_nodes_count}"
+  zone_id = "${var.vpc_dns_zone_id}"
+  name = "${var.master_nodes_count == 1 ? format("elasticsearch.master.%s", data.aws_route53_zone.local.name) : format("elasticsearch.master%d.%s", count.index, data.aws_route53_zone.local.name)}"
+  type = "A"
+  ttl = "60"
+  records = ["${element(aws_instance.elasticsearch_master_instance.*.private_ip, 0)}"]
+}
+
+resource "aws_route53_record" "elasticsearch_data_node_dns_records" {
+  count = "${var.data_nodes_count}"
+  zone_id = "${var.vpc_dns_zone_id}"
+  name = "${var.data_nodes_count == 1 ? format("elasticsearch.%s", data.aws_route53_zone.local.name) : format("elasticsearch%d.%s", count.index, data.aws_route53_zone.local.name)}"
+  type = "A"
+  ttl = "60"
+  records = ["${element(aws_instance.elasticsearch_data_instance.*.private_ip, 0)}"]
+}

modules/elasticsearch/ecs.tf

@@ -1,15 +1,83 @@
+resource "aws_ecs_cluster" "elasticsearch_cluster" {
+  name = "${var.ecs_cluster_name}"
+}
+
+resource "aws_ecs_task_definition" "elasticsearch_master_task" {
+  family = "${lower(var.env_name)}-elasticsearch-master"
+  container_definitions = "${data.template_file.elasticsearch_master_config.rendered}"
+  volume {
+    name = "elasticseach-data"
+    host_path = "${var.data_volume_path}"
+  }
+}
+
+resource "aws_ecs_task_definition" "elasticsearch_data_task" {
+  family = "${lower(var.env_name)}-elasticsearch-data"
+  container_definitions = "${data.template_file.elasticsearch_data_config.rendered}"
+  volume {
+    name = "elasticseach-data"
+    host_path = "${var.data_volume_path}"
+  }
+}
+
+resource "aws_ecs_service" "elasticsearch_master_service" {
+  depends_on = ["aws_iam_role_policy.docker_policy", "aws_ecs_task_definition.elasticsearch_master_task"]
+  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch-master"
+  cluster = "${aws_ecs_cluster.elasticsearch_cluster.id}"
+  task_definition = "${aws_ecs_task_definition.elasticsearch_master_task.arn}"
+  desired_count = "${var.master_nodes_count}"
+  placement_constraints {
+    type = "memberOf"
+    expression = "attribute:cluster_role == elasticsearch-master"
+  }
+}
+
+resource "aws_ecs_service" "elasticsearch_data_service" {
+  depends_on = ["aws_iam_role_policy.docker_policy", "aws_ecs_task_definition.elasticsearch_master_task"]
+  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch-data"
+  cluster = "${aws_ecs_cluster.elasticsearch_cluster.id}"
+  task_definition = "${aws_ecs_task_definition.elasticsearch_data_task.arn}"
+  desired_count = "${var.data_nodes_count}"
+  placement_constraints {
+    type = "memberOf"
+    expression = "attribute:cluster_role == elasticsearch-data"
+  }
+}
+
 data "template_file" "elasticsearch_master_config" {
   template = "${file("${path.module}/resources/elasticsearch.json")}"
   vars {
-    container_name = "elasticsearch-master"
     elasticsearch_version = "${var.elasticsearch_version}"
-    memory = "${var.container_memory_limit}"
-    node_name = "${var.verbose_name}-elasticsearch-master"
-    elasticsearch-cluster-name = "${var.elasticsearch_cluster_name}"
-    volume_name = "elasticseach-data"
+    container_name = "elasticsearch-master"
+    container_memory = 512
     native_transport_port = 9300
-    http_service_port = 9200
-    extra-options = ""
+    http_transport_port = 9200
+    cluster_name = "${var.elasticsearch_cluster_name}"
+    node_name = "${lower(var.verbose_name)}-elasticsearch-master"
+    is_master = "true"
+    is_data = "false"
+    min_master_nodes = "${var.master_nodes_count == 0 ? (var.master_nodes_count / 2) + 1 : 0}"
+    master_nodes_addresses = ""
+    heap_size = 256
+    volume_name = "elasticseach-data"
   }
 }
 
+data "template_file" "elasticsearch_data_config" {
+  template = "${file("${path.module}/resources/elasticsearch.json")}"
+  vars {
+    elasticsearch_version = "${var.elasticsearch_version}"
+    container_name = "elasticsearch-data"
+    container_memory = "${var.elasticsearch_memory_limit}"
+    native_transport_port = 9300
+    http_transport_port = 9200
+    cluster_name = "${var.elasticsearch_cluster_name}"
+    node_name = "${lower(var.verbose_name)}-elasticsearch-data"
+    is_master = "${var.is_data_nodes_master_eiligible == 1 ? "true" : "false"}"
+    is_data = "true"
+    master_nodes_addresses = "${join(", ", concat(var.external_masters_addresses, aws_route53_record.elasticsearch_master_node_dns_records.*.name))}"
+    min_master_nodes = "${(var.master_nodes_count / 2) + 1 }"
+    heap_size = "${var.elasticsearch_memory_limit / 2}"
+    volume_name = "elasticseach-data"
+  }
+}

modules/elasticsearch/iam.tf

@@ -1,11 +1,11 @@
 resource "aws_iam_instance_profile" "elasticsearch" {
-  name = "${lower(var.env_name)}-${var.verbose_name}-elasticsearch"
+  name = "${lower(var.env_name)}-${lower(var.verbose_name)}-elasticsearch"
   role = "${aws_iam_role.elasticsearch_role.name}"
 }
 
 resource "aws_iam_role" "elasticsearch_role" {
   name = "${lower(var.env_name)}-${var.verbose_name}-elasticsearch"
-  assume_role_policy = "${data.aws_iam_policy_document.ec2_assume_policy}"
+  assume_role_policy = "${data.aws_iam_policy_document.ec2_assume_policy.json}"
 }
 
 resource "aws_iam_role_policy" "docker_policy" {

modules/elasticsearch/output.tf

@@ -1,13 +1,21 @@
 output "private_ips" {
-  value = ["${aws_instance.elasticsearch_instance.*.private_ip}"]
+  value = ["${aws_instance.elasticsearch_data_instance.*.private_ip}"]
+}
+
+output "elasticseacrch_master_addresses" {
+  value = "${formatlist("%s:%s", aws_route53_record.elasticsearch_master_node_dns_records.*.name)}"
+}
+
+output "elasticseacrch_node_addresses" {
+  value = "${formatlist("%s:%s", aws_route53_record.elasticsearch_data_node_dns_records.*.name)}"
 }
 
 output "instance_ids" {
-  value = ["${aws_instance.elasticsearch_instance.*.id}"]
+  value = ["${aws_instance.elasticsearch_data_instance.*.id}"]
 }
 
 output "elasticsearch_sg_id" {
-  value = "${aws_security_group.elasticsearch.id}"
+  value = "${aws_security_group.elasticsearch_sg.id}"
 }
 
 output "instance_profile_id" {
@@ -16,4 +24,4 @@ output "instance_profile_id" {
 
 output "iam_role_id" {
   value = "${aws_iam_role.elasticsearch_role.id}"
-}
+}