[yugabyte] Working AWS implementation (#1191)

the-glu · web-flow · commit 51e00a4d8486 · 2025-06-17T15:50:39.000+01:00
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_dns.tf
@@ -1,6 +1,8 @@
 
 locals {
-  crdb_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_crdb[*] : i.tags.ExpectedDNS => i.public_ip }
+  crdb_hostnames             = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_crdb[*] : i.tags.ExpectedDNS => i.public_ip }
+  yugabyte_master_hostnames  = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_yugabyte_masters[*] : i.tags.ExpectedDNS => i.public_ip }
+  yugabyte_tserver_hostnames = var.aws_route53_zone_id == "" ? {} : { for i in aws_eip.ip_yugabyte_tservers[*] : i.tags.ExpectedDNS => i.public_ip }
 }
 
 
@@ -37,3 +39,25 @@ resource "aws_route53_record" "crdb_hostname" {
   ttl     = 300
   records = [each.value]
 }
+
+# Yugabyte master nodes DNS
+resource "aws_route53_record" "yugabyte_master_hostnames" {
+  for_each = local.yugabyte_master_hostnames
+
+  zone_id = var.aws_route53_zone_id
+  name    = each.key
+  type    = "A"
+  ttl     = 300
+  records = [each.value]
+}
+
+# Yugabyte tserver nodes DNS
+resource "aws_route53_record" "yugabyte_tserver_hostnames" {
+  for_each = local.yugabyte_tserver_hostnames
+
+  zone_id = var.aws_route53_zone_id
+  name    = each.key
+  type    = "A"
+  ttl     = 300
+  records = [each.value]
+}
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/network_lb.tf
@@ -73,7 +73,7 @@ resource "aws_eip" "gateway" {
 
 # Public Elastic IPs for the crdb instances
 resource "aws_eip" "ip_crdb" {
-  count = var.node_count
+  count = var.datastore_type == "cockroachdb" ? var.node_count : 0
   vpc   = true
 
   tags = {
@@ -82,3 +82,27 @@ resource "aws_eip" "ip_crdb" {
     ExpectedDNS = format("%s.%s", count.index, var.crdb_hostname_suffix)
   }
 }
+
+# Public Elastic IPs for the yubagybte master instances
+resource "aws_eip" "ip_yugabyte_masters" {
+  count = var.datastore_type == "yugabyte" ? var.node_count : 0
+  vpc   = true
+
+  tags = {
+    Name = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index)
+    # Preserve mapping between ips and hostnames
+    ExpectedDNS = format("%s.master.%s", count.index, var.crdb_hostname_suffix)
+  }
+}
+
+# Public Elastic IPs for the yubagybte tserver instances
+resource "aws_eip" "ip_yugabyte_tservers" {
+  count = var.datastore_type == "yugabyte" ? var.node_count : 0
+  vpc   = true
+
+  tags = {
+    Name = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index)
+    # Preserve mapping between ips and hostnames
+    ExpectedDNS = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix)
+  }
+}
diff --git a/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf b/deploy/infrastructure/dependencies/terraform-aws-kubernetes/output.tf
@@ -30,10 +30,42 @@ output "crdb_nodes" {
   ]
 }
 
+output "yugabyte_masters_nodes" {
+  value = [
+    for i in aws_eip.ip_yugabyte_masters : {
+      ip  = i.allocation_id
+      dns = i.tags.ExpectedDNS
+    }
+  ]
+  depends_on = [
+    aws_eip.ip_yugabyte_masters
+  ]
+}
+
+output "yugabyte_tservers_nodes" {
+  value = [
+    for i in aws_eip.ip_yugabyte_tservers : {
+      ip  = i.allocation_id
+      dns = i.tags.ExpectedDNS
+    }
+  ]
+  depends_on = [
+    aws_eip.ip_yugabyte_tservers
+  ]
+}
+
 output "crdb_addresses" {
   value = [for i in aws_eip.ip_crdb[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }]
 }
 
+output "yugabyte_masters_addresses" {
+  value = [for i in aws_eip.ip_yugabyte_masters[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }]
+}
+
+output "yugabyte_tservers_addresses" {
+  value = [for i in aws_eip.ip_yugabyte_tservers[*] : { expected_dns : i.tags.ExpectedDNS, address : i.public_ip }]
+}
+
 output "gateway_address" {
   value = {
     expected_dns : aws_eip.gateway[0].tags.ExpectedDNS,
@@ -56,4 +88,4 @@ output "workload_subnet" {
 
 output "iam_role_node_group_arn" {
   value = aws_iam_role.dss-cluster-node-group.arn
-}
+}
diff --git a/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf b/deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf
@@ -102,6 +102,19 @@ resource "local_file" "helm_chart_values" {
         totalMasters = length(var.yugabyte_external_nodes) + var.node_count
       }
 
+      storage = {
+        master = {
+          storageClass = var.kubernetes_storage_class
+        }
+        tserver = {
+          storageClass = var.kubernetes_storage_class
+        }
+      }
+
+      preflight = {
+        skipUlimit = true
+      }
+
       master = {
         extraEnv = [{
           name = "HOSTNAMENO"
diff --git a/deploy/infrastructure/modules/terraform-aws-dss/main.tf b/deploy/infrastructure/modules/terraform-aws-dss/main.tf
@@ -4,6 +4,7 @@ module "terraform-aws-kubernetes" {
   aws_region                   = var.aws_region
   app_hostname                 = var.app_hostname
   crdb_hostname_suffix         = var.crdb_hostname_suffix
+  datastore_type               = var.datastore_type
   aws_instance_type            = var.aws_instance_type
   aws_route53_zone_id          = var.aws_route53_zone_id
   aws_iam_permissions_boundary = var.aws_iam_permissions_boundary
@@ -15,27 +16,35 @@ module "terraform-aws-kubernetes" {
 
 module "terraform-commons-dss" {
   # See variables.tf for variables description.
-  image                          = var.image
-  image_pull_secret              = var.image_pull_secret
-  kubernetes_namespace           = var.kubernetes_namespace
-  kubernetes_storage_class       = var.aws_kubernetes_storage_class
-  app_hostname                   = var.app_hostname
-  crdb_image_tag                 = var.crdb_image_tag
-  crdb_cluster_name              = var.crdb_cluster_name
-  crdb_hostname_suffix           = var.crdb_hostname_suffix
-  should_init                    = var.should_init
-  authorization                  = var.authorization
-  crdb_locality                  = var.crdb_locality
-  crdb_external_nodes            = var.crdb_external_nodes
-  node_count                     = var.node_count
-  crdb_internal_nodes            = module.terraform-aws-kubernetes.crdb_nodes
-  ip_gateway                     = module.terraform-aws-kubernetes.ip_gateway
-  kubernetes_api_endpoint        = module.terraform-aws-kubernetes.kubernetes_api_endpoint
-  kubernetes_cloud_provider_name = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name
-  kubernetes_context_name        = module.terraform-aws-kubernetes.kubernetes_context_name
-  kubernetes_get_credentials_cmd = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd
-  workload_subnet                = module.terraform-aws-kubernetes.workload_subnet
-  gateway_cert_name              = module.terraform-aws-kubernetes.app_hostname_cert_arn
+  image                            = var.image
+  image_pull_secret                = var.image_pull_secret
+  kubernetes_namespace             = var.kubernetes_namespace
+  kubernetes_storage_class         = var.aws_kubernetes_storage_class
+  app_hostname                     = var.app_hostname
+  crdb_image_tag                   = var.crdb_image_tag
+  crdb_cluster_name                = var.crdb_cluster_name
+  crdb_hostname_suffix             = var.crdb_hostname_suffix
+  datastore_type                   = var.datastore_type
+  should_init                      = var.should_init
+  authorization                    = var.authorization
+  crdb_locality                    = var.crdb_locality
+  crdb_external_nodes              = var.crdb_external_nodes
+  node_count                       = var.node_count
+  yugabyte_cloud                   = var.yugabyte_cloud
+  yugabyte_region                  = var.yugabyte_region
+  yugabyte_zone                    = var.yugabyte_zone
+  yugabyte_light_resources         = var.yugabyte_light_resources
+  yugabyte_external_nodes          = var.yugabyte_external_nodes
+  crdb_internal_nodes              = module.terraform-aws-kubernetes.crdb_nodes
+  yugabyte_internal_masters_nodes  = module.terraform-aws-kubernetes.yugabyte_masters_nodes
+  yugabyte_internal_tservers_nodes = module.terraform-aws-kubernetes.yugabyte_tservers_nodes
+  ip_gateway                       = module.terraform-aws-kubernetes.ip_gateway
+  kubernetes_api_endpoint          = module.terraform-aws-kubernetes.kubernetes_api_endpoint
+  kubernetes_cloud_provider_name   = module.terraform-aws-kubernetes.kubernetes_cloud_provider_name
+  kubernetes_context_name          = module.terraform-aws-kubernetes.kubernetes_context_name
+  kubernetes_get_credentials_cmd   = module.terraform-aws-kubernetes.kubernetes_get_credentials_cmd
+  workload_subnet                  = module.terraform-aws-kubernetes.workload_subnet
+  gateway_cert_name                = module.terraform-aws-kubernetes.app_hostname_cert_arn
 
   source = "../../dependencies/terraform-commons-dss"
 }
diff --git a/deploy/infrastructure/modules/terraform-aws-dss/output.tf b/deploy/infrastructure/modules/terraform-aws-dss/output.tf
@@ -2,6 +2,14 @@ output "crdb_addresses" {
   value = module.terraform-aws-kubernetes.crdb_addresses
 }
 
+output "yugabyte_masters_addresses" {
+  value = module.terraform-aws-kubernetes.yugabyte_masters_addresses
+}
+
+output "yugabyte_tservers_addresses" {
+  value = module.terraform-aws-kubernetes.yugabyte_tservers_addresses
+}
+
 output "gateway_address" {
   value = module.terraform-aws-kubernetes.gateway_address
 }
diff --git a/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml b/deploy/services/helm-charts/dss/templates/yugabyte-loadbalancers.yaml
@@ -12,26 +12,26 @@ metadata:
     service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
     {{- include (printf "%s-lb-crdb-annotations" $cloudProvider)
       (dict
-        "name" (printf "%s-%s" "yugabyte-db-master-external-node" ( $i | toString) )
+        "name" (printf "%s-%s" "ybdb-master-ext" ( $i | toString) )
         "ip" $lb.ip
         "subnet" $lb.subnet
         "cloudProvider" $cloudProvider
       ) | nindent 4
     }}
   labels:
     app: yugabyte
-    name: yugabyte-db-master-external-node-{{$i}}
-  name: yugabyte-db-master-external-node-{{$i}}
+    name: ybdb-master-ext-{{$i}}
+  name: ybdb-master-ext-{{$i}}
 spec:
   {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}}
   ports:
-    - name: yugabyte-master-db-external-node-{{$i}}
+    - name: yugabyte-master-db-ext-{{$i}}
       port: 7100
       targetPort: 7100
-    - name: yugabyte-master-ui-external-node-{{$i}}
+    - name: yugabyte-master-ui-ext-{{$i}}
       port: 7000
       targetPort: 7000
-    - name: yugabyte-master-ui2-external-node-{{$i}}
+    - name: yugabyte-master-ui2-ext-{{$i}}
       port: 9000
       targetPort: 9000
   publishNotReadyAddresses: true
@@ -50,38 +50,38 @@ metadata:
     service.alpha.kubernetes.io/tolerate-unready-endpoints: "true"
     {{- include (printf "%s-lb-crdb-annotations" $cloudProvider)
       (dict
-        "name" (printf "%s-%s" " yugabyte-db-tserver-external-node" ( $i | toString) )
+        "name" (printf "%s-%s" " ybdb-tserver-ext" ( $i | toString) )
         "ip" $lb.ip
         "subnet" $lb.subnet
         "cloudProvider" $cloudProvider
       ) | nindent 4
     }}
   labels:
     app: yugabyte
-    name: yugabyte-db-tserver-external-node-{{$i}}
-  name: yugabyte-db-tserver-external-node-{{$i}}
+    name: yuga-tserver-ext-{{$i}}
+  name: yuga-tserver-ext-{{$i}}
 spec:
   {{- include (printf "%s-lb-spec" $cloudProvider) (dict "ip" $lb.ip) | nindent 2}}
   ports:
-    - name: yugabyte-tserver-db-external-node-{{$i}}
+    - name: yugabyte-tserver-db-ext-{{$i}}
       port: 9100
       targetPort: 9100
-    - name: yugabyte-tserver-ui-external-node-{{$i}}
+    - name: yugabyte-tserver-ui-ext-{{$i}}
       port: 9000
       targetPort: 9000
-    - name: yugabyte-tserver-ui2-external-node-{{$i}}
+    - name: yugabyte-tserver-ui2-ext-{{$i}}
       port: 7000
       targetPort: 7000
-    - name: yugabyte-tserver-ycql-external-node-{{$i}}
+    - name: yugabyte-tserver-ycql-ext-{{$i}}
       port: 9042
       targetPort: 9042
-    - name: yugabyte-tserver-ysql-external-node-{{$i}}
+    - name: yugabyte-tserver-ysql-ext-{{$i}}
       port: 5433
       targetPort: 5433
-    - name: yugabyte-tserver-metrics-external-node-{{$i}}
+    - name: yugabyte-tserver-metrics-ext-{{$i}}
       port: 13000
       targetPort: 13000
-    - name: yugabyte-tserver-metrics-2-external-node-{{$i}}
+    - name: yugabyte-tserver-metrics-2-ext-{{$i}}
       port: 12000
       targetPort: 12000
   publishNotReadyAddresses: true

Original file line number	Diff line number	Diff line change
`@@ -2,6 +2,14 @@ output "crdb_addresses" {`
`2`	`2`	`value = module.terraform-aws-kubernetes.crdb_addresses`
`3`	`3`	`}`
`4`	`4`
	`5`	`+output "yugabyte_masters_addresses" {`
	`6`	`+ value = module.terraform-aws-kubernetes.yugabyte_masters_addresses`
	`7`	`+}`
	`8`	`+`
	`9`	`+output "yugabyte_tservers_addresses" {`
	`10`	`+ value = module.terraform-aws-kubernetes.yugabyte_tservers_addresses`
	`11`	`+}`
	`12`	`+`
`5`	`13`	`output "gateway_address" {`
`6`	`14`	`value = module.terraform-aws-kubernetes.gateway_address`
`7`	`15`	`}`