Split external_dependencies module into module for each infra component

Author: burmek

.terraform.lock.hcl

@@ -2,6 +2,12 @@ enable_ec2_deployment = true
 
 enable_eks_deployment = false
 
-enable_external_dependencies = true
+#enable_external_dependencies = false
+
+enable_elasticache = true
+
+enable_rds = true
+
+enable_s3 = true
 
 s3_bucket_name = "cometml-use2"

.terraform/modules/modules.json

@@ -2,6 +2,12 @@ enable_ec2_deployment = false
 
 enable_eks_deployment = false
 
-enable_external_dependencies = true
+#enable_external_dependencies = true
+
+enable_elasticache = true
+
+enable_rds = true
+
+enable_s3 = true
 
 s3_bucket_name = "cometml-use2-tftest-dev"

ec2_external-dependencies.tfvars

@@ -70,6 +70,7 @@ module "eks_deployment" {
   cluster_version = var.eks_cluster_version
 }
 
+/*
 module "external_dependencies" {
   source = "./modules/external_dependencies"
   count  = var.enable_external_dependencies ? 1 : 0
@@ -80,7 +81,39 @@ module "external_dependencies" {
 
   comet_ml_s3_bucket  = var.s3_bucket_name
 
+  elasticache_rds_allowfrom_sg = module.ec2_deployment[0].allinone_sg_id
+}
+*/
+
+module "comet_elasticache" {
+  source = "./modules/comet_elasticache"
+  count  = var.enable_elasticache ? 1 : 0
+
+  vpc_id              = module.vpc.vpc_id
+  vpc_private_subnets = module.vpc.private_subnets
+
+  # need to get SGs from ec2_deployment or eks_deployment, depending on which is used
+  # index is used on the ec2_deployment becuase of the count usage in the toggle: "After the count apply the resource becomes a group, so later in the reference use 0-index of the group"
+  elasticache_rds_allowfrom_sg = module.ec2_deployment[0].allinone_sg_id
+}
+
+module "comet_rds" {
+  source = "./modules/comet_rds"
+  count  = var.enable_rds ? 1 : 0
+
+  availability_zones = local.azs
+  vpc_id              = module.vpc.vpc_id
+  vpc_private_subnets = module.vpc.private_subnets
+
   # need to get SGs from ec2_deployment or eks_deployment, depending on which is used
   # index is used on the ec2_deployment becuase of the count usage in the toggle: "After the count apply the resource becomes a group, so later in the reference use 0-index of the group"
-  elasticcache_rds_allowfrom_sg = module.ec2_deployment[0].allinone_sg_id
+  elasticache_rds_allowfrom_sg = module.ec2_deployment[0].allinone_sg_id
+  
+}
+
+module "comet_s3" {
+  source = "./modules/comet_s3"
+  count  = var.enable_s3 ? 1 : 0
+
+  comet_ml_s3_bucket  = var.s3_bucket_name
 }

external_dependencies_only.tfvars

@@ -0,0 +1,42 @@
+locals {
+  redis_port = 6379
+
+  tags = {
+    Terraform_Managed = "true"
+    Environment       = var.environment
+  }
+}
+
+resource "aws_elasticache_cluster" "comet-ml-ec-redis" {
+  cluster_id           = "cometml-ec-redis-${var.environment}"
+  engine               = var.elasticache_engine
+  node_type            = var.elasticache_redis_instance_type
+  num_cache_nodes      = var.elasticache_num_cache_nodes
+  parameter_group_name = var.elasticache_param_group_name
+  engine_version       = var.elasticache_engine_version
+  port                 = local.redis_port
+  subnet_group_name    = aws_elasticache_subnet_group.comet-ml-ec-subnet-group.name
+  # SG resource ref
+  security_group_ids = [aws_security_group.redis_inbound_sg.id]
+}
+
+resource "aws_elasticache_subnet_group" "comet-ml-ec-subnet-group" {
+  name = "cometml-ec-sng-${var.environment}"
+  subnet_ids = var.vpc_private_subnets
+}
+
+resource "aws_security_group" "redis_inbound_sg" {
+  name        = "cometml_redis_in_sg_${var.environment}"
+  description = "Redis Security Group"
+  vpc_id = var.vpc_id
+}
+
+resource "aws_vpc_security_group_ingress_rule" "redis_port_inbound_allow" {
+  security_group_id = aws_security_group.redis_inbound_sg.id
+
+  from_port   = local.redis_port
+  to_port     = local.redis_port
+  ip_protocol    = "tcp"
+  # security groups need to change depending on whether Cx is using eks or ec2 deployment; hard-coded index won't work
+  referenced_security_group_id = var.elasticache_rds_allowfrom_sg
+}

main.tf

@@ -0,0 +1,53 @@
+variable "environment" {
+  description = "Deployment environment, i.e. dev/stage/prod, etc"
+  type        = string
+  default     = "dev"
+}
+
+variable "elasticache_redis_instance_type" {
+  description = "Elasticache Redis instance type"
+  type        = string
+  default     = "cache.r4.xlarge"
+}
+
+variable "elasticache_engine" {
+  description = "Engine type for Elasticache cluster"
+  type        = string
+  default     = "redis"
+}
+
+variable "elasticache_engine_version" {
+  description = "Version number for Elasticache engine"
+  type        = string
+  default     = "5.0.6"
+}
+
+variable "elasticache_param_group_name" {
+  description = "Name for the Elasticache cluster parameter group"
+  type        = string
+  default     = "default.redis5.0"
+}
+
+variable "elasticache_num_cache_nodes" {
+  description = "Number of nodes in the Elasticache cluster"
+  type        = number
+  default     = 1
+}
+
+variable "vpc_id" {
+  description = "ID of the VPC that will contain the provisioned resources"
+  type        = string
+  default     = ""
+}
+
+variable "vpc_private_subnets" {
+  description = "IDs of private subnets within the VPC"
+  type        = list(string)
+  default     = []
+}
+
+variable "elasticache_rds_allowfrom_sg" {
+  description = "Security group(s) attached to the Comet instance(s); Specified in the ingress allow rules on the Elasticache and RDS security groups"
+  type        = string
+  default     = ""
+}