[engine-type] allow to use different db engine type

Author: oavdeev

modules/datastore/README.md

@@ -19,6 +19,8 @@ To read more, see [the Metaflow docs](https://docs.metaflow.org/metaflow-on-aws/
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
+| <a name="input_db_engine"></a> [db\_engine](#input\_db\_engine) | n/a | `string` | `"postgresql"` | no |
+| <a name="input_db_engine_version"></a> [db\_engine\_version](#input\_db\_engine\_version) | n/a | `string` | `"11"` | no |
 | <a name="input_db_instance_type"></a> [db\_instance\_type](#input\_db\_instance\_type) | RDS instance type to launch for PostgresQL database. | `string` | `"db.t2.small"` | no |
 | <a name="input_db_name"></a> [db\_name](#input\_db\_name) | Name of PostgresQL database for Metaflow service. | `string` | `"metaflow"` | no |
 | <a name="input_db_username"></a> [db\_username](#input\_db\_username) | PostgresQL username; defaults to 'metaflow' | `string` | `"metaflow"` | no |

modules/datastore/outputs.tf

@@ -29,7 +29,7 @@ output "datastore_s3_bucket_kms_key_arn" {
 }
 
 output "rds_master_instance_endpoint" {
-  value       = aws_db_instance.this.endpoint
+  value       = local.use_aurora ? aws_rds_cluster.this[0].endpoint : aws_db_instance.this[0].endpoint
   description = "The database connection endpoint in address:port format"
 }
 

modules/datastore/rds.tf

@@ -53,17 +53,57 @@ resource "random_password" "this" {
 
 resource "random_pet" "final_snapshot_id" {}
 
+locals {
+  use_aurora = length(regexall("^aurora-", var.db_engine)) > 0
+}
+
+resource "aws_rds_cluster" "this" {
+  count              = local.use_aurora ? 1 : 0
+  cluster_identifier = "${var.resource_prefix}${var.db_name}${var.resource_suffix}"
+  kms_key_id         = aws_kms_key.rds.arn
+  engine             = var.db_engine
+
+  database_name        = var.db_name
+  master_username      = var.db_username
+  master_password      = random_password.this.result
+  db_subnet_group_name = aws_db_subnet_group.this.id
+
+  engine_version    = var.db_engine_version
+  storage_encrypted = true
+
+  final_snapshot_identifier = "${var.resource_prefix}${var.db_name}-final-snapshot${var.resource_suffix}-${random_pet.final_snapshot_id.id}" # Snapshot upon delete
+  vpc_security_group_ids    = [aws_security_group.rds_security_group.id]
+
+  tags = merge(
+    var.standard_tags,
+    {
+      Name     = "${var.resource_prefix}${var.db_name}${var.resource_suffix}"
+      Metaflow = "true"
+    }
+  )
+}
+
+resource "aws_rds_cluster_instance" "cluster_instances" {
+  count              = local.use_aurora ? 1 : 0
+  identifier         = "${var.resource_prefix}${var.db_name}${var.resource_suffix}-${count.index}"
+  cluster_identifier = aws_rds_cluster.this[0].id
+  instance_class     = var.db_instance_type
+  engine             = aws_rds_cluster.this[0].engine
+  engine_version     = aws_rds_cluster.this[0].engine_version
+}
+
 /*
  Define rds db instance.
 */
 resource "aws_db_instance" "this" {
+  count                     = local.use_aurora ? 0 : 1
   publicly_accessible       = false
   allocated_storage         = 20    # Allocate 20GB
   storage_type              = "gp2" # general purpose SSD
   storage_encrypted         = true
   kms_key_id                = aws_kms_key.rds.arn
-  engine                    = "postgres"
-  engine_version            = "11"
+  engine                    = var.db_engine
+  engine_version            = var.db_engine_version
   instance_class            = var.db_instance_type                                         # Hardware configuration
   identifier                = "${var.resource_prefix}${var.db_name}${var.resource_suffix}" # used for dns hostname needs to be customer unique in region
   name                      = var.db_name                                                  # unique id for CLI commands (name of DB table which is why we're not adding the prefix as no conflicts will occur and the API expects this table name)

modules/datastore/variables.tf

@@ -4,6 +4,16 @@ variable "db_instance_type" {
   default     = "db.t2.small"
 }
 
+variable "db_engine" {
+  type    = string
+  default = "postgresql"
+}
+
+variable "db_engine_version" {
+  type    = string
+  default = "11"
+}
+
 variable "db_name" {
   type        = string
   description = "Name of PostgresQL database for Metaflow service."