Yugabyte in GCP

Author: the-glu

deploy/infrastructure/dependencies/terraform-aws-kubernetes/variables.gen.tf

@@ -65,6 +65,23 @@ variable "crdb_hostname_suffix" {
   EOT
 }
 
+variable "datastore_type" {
+  type        = string
+  description = <<-EOT
+  Type of datastore used
+
+  Supported technologies: cockroachdb, yugabyte
+  EOT
+
+  validation {
+    condition     = contains(["cockroachdb", "yugabyte"], var.datastore_type)
+    error_message = "Supported technologies: cockroachdb, yugabyte"
+  }
+
+  default = "cockroachdb"
+}
+
+
 variable "cluster_name" {
   type        = string
   description = <<-EOT

deploy/infrastructure/dependencies/terraform-commons-dss/helm.tf

@@ -6,7 +6,7 @@ locals {
 
 resource "local_file" "helm_chart_values" {
   filename = "${local.workspace_location}/helm_values.yml"
-  content = yamlencode({
+  content = var.datastore_type == "cockroachdb" ? yamlencode({
     cockroachdb = {
       image = {
         tag = var.crdb_image_tag
@@ -69,5 +69,132 @@ resource "local_file" "helm_chart_values" {
     global = {
       cloudProvider = var.kubernetes_cloud_provider_name
     }
-  })
+}) : yamlencode({
+    cockroachdb = {
+      enabled = false
+      image = {
+        tag = "dummy"
+      }
+      fullnameOverride = "dummy"
+      conf = {
+        cluster-name = "dummy"
+        locality = "dummy"
+      }
+      statefulset = {}
+    }
+    yugabyte = {
+      enabled = true
+
+      resource = var.yugabyte_light_resources ? {
+        master = {
+          requests = {
+            cpu = "0.1"
+            memory = "0.5G"
+          }
+        }
+        tserver = {
+          requests = {
+            cpu = "0.1"
+            memory = "0.5G"
+          }
+        }
+      } : {}
+      enableLoadBalancer = false
+
+      master = {
+        extraEnv = [{
+          name = "HOSTNAMENO"
+          valueFrom = {
+            fieldRef = {
+              fieldPath = "metadata.labels['apps.kubernetes.io/pod-index']"
+            }
+          }
+        }]
+        serverBroadcastAddress: "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}"
+        rpcBindAddress: "$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}"
+        advanced = {
+          preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.master.${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\"
+        }
+      }
+
+      tserver = {
+        extraEnv = [{
+          name = "HOSTNAMENO"
+          valueFrom =  {
+            fieldRef = {
+              fieldPath = "metadata.labels['apps.kubernetes.io/pod-index']"
+            }
+          }
+        }]
+        serverBroadcastAddress: "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}"
+        rpcBindAddress: "$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}"
+        advanced = {
+          preCommands: "sed -E \"/\\.svc\\.cluster\\.local/ s/^([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+)([[:space:]]+)/\\1 $(echo \"$${HOSTNAMENO}.tserver.${var.crdb_hostname_suffix}\" | sed 's/[\\/&]/\\\\&/g')\\2/\" /etc/hosts > /tmp/newhosts && /bin/cp /tmp/newhosts /etc/hosts && \\"
+        }
+      }
+
+      gflags = {
+        master = {
+          placement_cloud: var.yugabyte_cloud
+          placement_region: var.yugabyte_region
+          placement_zone: var.yugabyte_zone
+          use_private_ip: "zone"
+        }
+        tserver = {
+          placement_cloud: var.yugabyte_cloud
+          placement_region: var.yugabyte_region
+          placement_zone: var.yugabyte_zone
+          use_private_ip: "zone"
+        }
+      }
+
+      masterAddresses = join(",", ["0.master.${var.crdb_hostname_suffix},1.master.${var.crdb_hostname_suffix},2.master.${var.crdb_hostname_suffix}", join(",", var.yugabyte_external_nodes)])
+    }
+
+    loadBalancers = {
+      cockroachdbNodes = []
+
+      yugabyteMasterNodes = [
+        for ip in var.yugabyte_internal_masters_nodes[*].ip :
+        {
+          ip     = ip
+          subnet = var.workload_subnet
+        }
+      ]
+
+      yugabyteTserverNodes = [
+        for ip in var.yugabyte_internal_tservers_nodes[*].ip :
+        {
+          ip     = ip
+          subnet = var.workload_subnet
+        }
+      ]
+
+      dssGateway = {
+        ip        = var.ip_gateway
+        subnet    = var.workload_subnet
+        certName  = var.gateway_cert_name
+        sslPolicy = var.ssl_policy
+      }
+    }
+
+    dss = {
+      image = var.image
+
+      conf = {
+        pubKeys = [
+          "/test-certs/auth2.pem"
+        ]
+        jwksEndpoint = var.authorization.jwks != null ? var.authorization.jwks.endpoint : ""
+        jwksKeyIds   = var.authorization.jwks != null ? [var.authorization.jwks.key_id] : []
+        hostname     = var.app_hostname
+        enableScd    = var.enable_scd
+      }
+    }
+
+    global = {
+      cloudProvider = var.kubernetes_cloud_provider_name
+    }
+})
+
 }

deploy/infrastructure/dependencies/terraform-commons-dss/scripts.tf

@@ -1,5 +1,6 @@
 
 resource "local_file" "make_certs" {
+  count = var.datastore_type == "cockroachdb" ? 1 : 0
   content = templatefile("${path.module}/templates/make-certs.sh.tmp", {
     cluster_context = var.kubernetes_context_name
     namespace       = var.kubernetes_namespace
@@ -10,13 +11,24 @@ resource "local_file" "make_certs" {
 }
 
 resource "local_file" "apply_certs" {
+  count = var.datastore_type == "cockroachdb" ? 1 : 0
   content = templatefile("${path.module}/templates/apply-certs.sh.tmp", {
     cluster_context = var.kubernetes_context_name
     namespace       = var.kubernetes_namespace
   })
   filename = "${local.workspace_location}/apply-certs.sh"
 }
 
+resource "local_file" "dss_certs" {
+  count = var.datastore_type == "yugabyte" ? 1 : 0
+  content = templatefile("${path.module}/templates/dss-certs.sh.tmp", {
+    cluster_context      = var.kubernetes_context_name
+    namespace            = var.kubernetes_namespace
+    crdb_hostname_suffix = var.crdb_hostname_suffix
+  })
+  filename = "${local.workspace_location}/dss-certs.sh"
+}
+
 resource "local_file" "get_credentials" {
   content = templatefile("${path.module}/templates/get-credentials.sh.tmp", {
     get_credentials_cmd = var.kubernetes_get_credentials_cmd

deploy/infrastructure/dependencies/terraform-commons-dss/templates/dss-certs.sh.tmp

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+# This file was automatically generated by terraform-commons-dss.
+# Do not edit it directly.
+
+set -eo pipefail
+
+OS=$(uname)
+if [[ "$OS" == "Darwin" ]]; then
+	# OSX uses BSD readlink
+	BASEDIR="$(dirname "$0")"
+else
+	BASEDIR=$(readlink -e "$(dirname "$0")")
+fi
+cd "$BASEDIR/../../../deploy/operations/certificates-management/" || exit 1
+
+./dss-certs.py --name ${cluster_context} --organization default_orga --cluster-context ${cluster_context} --nodes-public-address "<ID>.<TYPE>.${crdb_hostname_suffix}" --namespace ${namespace} "$@"

deploy/infrastructure/dependencies/terraform-commons-dss/variables.gen.tf

@@ -22,6 +22,23 @@ variable "crdb_hostname_suffix" {
   EOT
 }
 
+variable "datastore_type" {
+  type        = string
+  description = <<-EOT
+  Type of datastore used
+
+  Supported technologies: cockroachdb, yugabyte
+  EOT
+
+  validation {
+    condition     = contains(["cockroachdb", "yugabyte"], var.datastore_type)
+    error_message = "Supported technologies: cockroachdb, yugabyte"
+  }
+
+  default = "cockroachdb"
+}
+
+
 variable "image" {
   type        = string
   description = <<-EOT
@@ -225,3 +242,62 @@ variable "kubernetes_namespace" {
   }
 }
 
+variable "yugabyte_cloud" {
+  type        = string
+  description = <<-EOT
+  Cloud of yugabyte instances, used for partionning.
+
+  Should be set to dss unless you're doing advanced partitionning.
+  EOT
+
+  default = "dss"
+}
+
+
+variable "yugabyte_region" {
+  type        = string
+  description = <<-EOT
+  Region of yugabyte instances, used for partionning.
+
+  Should be different from others USS in a cluster.
+  EOT
+
+  default = "uss-1"
+}
+
+
+variable "yugabyte_zone" {
+  type        = string
+  description = <<-EOT
+  Zone of yugabyte instances, used for partionning.
+
+  Should be set to zone unless you're doing advanced partitionning.
+  EOT
+
+  default = "zone"
+}
+
+
+variable "yugabyte_light_resources" {
+  type        = bool
+  description = <<-EOT
+  Enable light resources reservation for yugabyte instances.
+
+  Useful for a dev cluster when you don't want to overload your kubernetes cluster.
+  EOT
+
+  default = false
+}
+
+
+variable "yugabyte_external_nodes" {
+  type        = list(string)
+  description = <<-EOT
+  Fully-qualified domain name of existing yugabyte master nodes outside of the cluster if you are joining an existing pool.
+  Example: ["0.master.db.dss.example.com", "1.master.db.dss.example.com", "2.master.db.dss.example.com"]
+  EOT
+  default     = []
+}
+
+
+

deploy/infrastructure/dependencies/terraform-commons-dss/variables_internal.tf

@@ -29,6 +29,22 @@ variable "crdb_internal_nodes" {
   description = "List of the IP addresses and related dns for the Cockroach DB nodes"
 }
 
+variable "yugabyte_internal_masters_nodes" {
+  type = list(object({
+    dns = string
+    ip  = string
+  }))
+  description = "List of the IP addresses and related dns for the Yugabyte DB master nodes"
+}
+
+variable "yugabyte_internal_tservers_nodes" {
+  type = list(object({
+    dns = string
+    ip  = string
+  }))
+  description = "List of the IP addresses and related dns for the Yugabyte DB tserver nodes"
+}
+
 variable "ip_gateway" {
   type        = string
   description = "IP of the gateway used by the DSS service"

deploy/infrastructure/dependencies/terraform-google-kubernetes/cluster.tf

@@ -52,14 +52,33 @@ resource "google_compute_global_address" "ip_gateway" {
 
 # Static IP addresses for CRDB instances
 resource "google_compute_address" "ip_crdb" {
-  count  = var.node_count
+  count  = var.datastore_type == "cockroachdb" ? var.node_count : 0
   name   = format("%s-ip-crdb%v", var.cluster_name, count.index)
   region = local.region
 
   # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames.
   description = format("%s.%s", count.index, var.crdb_hostname_suffix)
 }
 
+# Static IP addresses for yugabyte instances
+resource "google_compute_address" "ip_yugabyte_masters" {
+  count  = var.datastore_type == "yugabyte" ? var.node_count : 0
+  name   = format("%s-ip-yugabyte-master%v", var.cluster_name, count.index)
+  region = local.region
+
+  # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames.
+  description = format("%s.master.%s", count.index, var.crdb_hostname_suffix)
+}
+
+resource "google_compute_address" "ip_yugabyte_tservers" {
+  count  = var.datastore_type == "yugabyte" ? var.node_count : 0
+  name   = format("%s-ip-yugabyte-tserver%v", var.cluster_name, count.index)
+  region = local.region
+
+  # Current google terraform provider doesn't allow tags or labels. Description is used to preserve mapping between ips and hostnames.
+  description = format("%s.tserver.%s", count.index, var.crdb_hostname_suffix)
+}
+
 locals {
   kubectl_cluster_context_name = format("gke_%s_%s_%s", google_container_cluster.kubernetes_cluster.project, google_container_cluster.kubernetes_cluster.location, google_container_cluster.kubernetes_cluster.name)
 }

deploy/infrastructure/dependencies/terraform-google-kubernetes/dns.tf

@@ -16,11 +16,31 @@ resource "google_dns_record_set" "gateway" {
 }
 
 resource "google_dns_record_set" "crdb" {
-  count = var.google_dns_managed_zone_name == "" ? 0 : var.node_count
+  count = var.google_dns_managed_zone_name == "" || var.datastore_type != "cockroachdb" ? 0 : var.node_count
   name  = "${google_compute_address.ip_crdb[count.index].description}." # description contains the expected hostname
   type  = "A"
   ttl   = 300
 
   managed_zone = data.google_dns_managed_zone.default[0].name
   rrdatas      = [google_compute_address.ip_crdb[count.index].address]
-}
+}
+
+resource "google_dns_record_set" "yugabyte_masters" {
+  count = var.google_dns_managed_zone_name == "" || var.datastore_type != "yugabyte" ? 0 : var.node_count
+  name  = "${google_compute_address.ip_yugabyte_masters[count.index].description}." # description contains the expected hostname
+  type  = "A"
+  ttl   = 300
+
+  managed_zone = data.google_dns_managed_zone.default[0].name
+  rrdatas      = [google_compute_address.ip_yugabyte_masters[count.index].address]
+}
+
+resource "google_dns_record_set" "yugabyte_tserver" {
+  count = var.google_dns_managed_zone_name == "" || var.datastore_type != "yugabyte" ? 0 : var.node_count
+  name  = "${google_compute_address.ip_yugabyte_tservers[count.index].description}." # description contains the expected hostname
+  type  = "A"
+  ttl   = 300
+
+  managed_zone = data.google_dns_managed_zone.default[0].name
+  rrdatas      = [google_compute_address.ip_yugabyte_tservers[count.index].address]
+}