Add lgalloc support

Author: bobbyiliev

README.md

@@ -123,6 +123,7 @@ export AWS_PROFILE=your-profile-name
 | <a name="input_install_aws_load_balancer_controller"></a> [install\_aws\_load\_balancer\_controller](#input\_install\_aws\_load\_balancer\_controller) | Whether to install the AWS Load Balancer Controller | `bool` | `true` | no |
 | <a name="input_install_materialize_operator"></a> [install\_materialize\_operator](#input\_install\_materialize\_operator) | Whether to install the Materialize operator | `bool` | `true` | no |
 | <a name="input_install_metrics_server"></a> [install\_metrics\_server](#input\_install\_metrics\_server) | Whether to install the metrics-server for the Materialize Console | `bool` | `true` | no |
+| <a name="input_install_openebs"></a> [install\_openebs](#input\_install\_openebs) | Whether to install OpenEBS for lgalloc support | `bool` | `true` | no |
 | <a name="input_kubernetes_namespace"></a> [kubernetes\_namespace](#input\_kubernetes\_namespace) | The Kubernetes namespace for the Materialize resources | `string` | `"materialize-environment"` | no |
 | <a name="input_log_group_name_prefix"></a> [log\_group\_name\_prefix](#input\_log\_group\_name\_prefix) | Prefix for the CloudWatch log group name (will be combined with environment name) | `string` | `"materialize"` | no |
 | <a name="input_materialize_instances"></a> [materialize\_instances](#input\_materialize\_instances) | Configuration for Materialize instances. Due to limitations in Terraform, `materialize_instances` cannot be defined on the first `terraform apply`. | <pre>list(object({<br/>    name                             = string<br/>    namespace                        = optional(string)<br/>    database_name                    = string<br/>    environmentd_version             = optional(string, "v0.130.4")<br/>    cpu_request                      = optional(string, "1")<br/>    memory_request                   = optional(string, "1Gi")<br/>    memory_limit                     = optional(string, "1Gi")<br/>    create_database                  = optional(bool, true)<br/>    create_nlb                       = optional(bool, true)<br/>    internal_nlb                     = optional(bool, true)<br/>    enable_cross_zone_load_balancing = optional(bool, true)<br/>    in_place_rollout                 = optional(bool, false)<br/>    request_rollout                  = optional(string)<br/>    force_rollout                    = optional(string)<br/>    balancer_memory_request          = optional(string, "256Mi")<br/>    balancer_memory_limit            = optional(string, "256Mi")<br/>    balancer_cpu_request             = optional(string, "100m")<br/>  }))</pre> | `[]` | no |
@@ -145,6 +146,10 @@ export AWS_PROFILE=your-profile-name
 | <a name="input_public_subnet_cidrs"></a> [public\_subnet\_cidrs](#input\_public\_subnet\_cidrs) | CIDR blocks for public subnets | `list(string)` | <pre>[<br/>  "10.0.101.0/24",<br/>  "10.0.102.0/24",<br/>  "10.0.103.0/24"<br/>]</pre> | no |
 | <a name="input_service_account_name"></a> [service\_account\_name](#input\_service\_account\_name) | Name of the service account | `string` | `"12345678-1234-1234-1234-123456789012"` | no |
 | <a name="input_single_nat_gateway"></a> [single\_nat\_gateway](#input\_single\_nat\_gateway) | Use a single NAT Gateway for all private subnets | `bool` | `false` | no |
+| <a name="input_storage_class_create"></a> [storage\_class\_create](#input\_storage\_class\_create) | Whether to create the storage class | `bool` | `true` | no |
+| <a name="input_storage_class_name"></a> [storage\_class\_name](#input\_storage\_class\_name) | Name of the storage class | `string` | `"openebs-lvm-instance-store-ext4"` | no |
+| <a name="input_storage_class_parameters"></a> [storage\_class\_parameters](#input\_storage\_class\_parameters) | Parameters for the storage class | <pre>object({<br/>    storage  = string<br/>    fsType   = string<br/>    volgroup = string<br/>  })</pre> | <pre>{<br/>  "fsType": "ext4",<br/>  "storage": "lvm",<br/>  "volgroup": "instance-store-vg"<br/>}</pre> | no |
+| <a name="input_storage_class_provisioner"></a> [storage\_class\_provisioner](#input\_storage\_class\_provisioner) | Storage class provisioner | `string` | `"local.csi.openebs.io"` | no |
 | <a name="input_tags"></a> [tags](#input\_tags) | Default tags to apply to all resources | `map(string)` | <pre>{<br/>  "Environment": "dev",<br/>  "Project": "materialize",<br/>  "Terraform": "true"<br/>}</pre> | no |
 | <a name="input_use_local_chart"></a> [use\_local\_chart](#input\_use\_local\_chart) | Whether to use a local chart instead of one from a repository | `bool` | `false` | no |
 | <a name="input_vpc_cidr"></a> [vpc\_cidr](#input\_vpc\_cidr) | CIDR block for VPC | `string` | `"10.0.0.0/16"` | no |

examples/simple/main.tf

@@ -51,11 +51,12 @@ module "materialize_infrastructure" {
 
   # EKS Configuration
   cluster_version                          = "1.32"
-  node_group_instance_types                = ["r8g.2xlarge"]
+  node_group_instance_types                = ["r7gd.2xlarge"]
   node_group_desired_size                  = 1
   node_group_min_size                      = 1
   node_group_max_size                      = 2
   node_group_capacity_type                 = "ON_DEMAND"
+  node_group_ami_type                      = "BOTTLEROCKET_ARM_64"
   enable_cluster_creator_admin_permissions = true
 
   # Storage Configuration

main.tf

@@ -34,6 +34,7 @@ module "eks" {
   cluster_enabled_log_types                = var.cluster_enabled_log_types
   node_group_capacity_type                 = var.node_group_capacity_type
   enable_cluster_creator_admin_permissions = var.enable_cluster_creator_admin_permissions
+  install_openebs                          = var.install_openebs
 
   tags = local.common_tags
 
@@ -185,6 +186,14 @@ locals {
         }
       }
     }
+    storage = {
+      storageClass = {
+        create      = var.storage_class_create
+        name        = var.storage_class_name
+        provisioner = var.storage_class_provisioner
+        parameters  = var.storage_class_parameters
+      }
+    }
   }
 
   merged_helm_values = merge(local.default_helm_values, var.helm_values)

modules/eks/main.tf

@@ -1,5 +1,47 @@
 locals {
   name_prefix = "${var.namespace}-${var.environment}"
+
+  disk_setup_user_data = base64encode(<<-EOT
+    #!/bin/bash
+    set -xeuo pipefail
+
+    yum update -y
+    yum install -y lvm2 util-linux jq e2fsprogs
+
+    BOTTLEROCKET_ROOT="/.bottlerocket/rootfs"
+    DRIVE_PATHS=()
+
+    mapfile -t SSD_NVME_DEVICE_LIST < <(lsblk --json --output-all | jq -r '.blockdevices[] | select(.model // empty | contains("Amazon EC2 NVMe Instance Storage")) | .path')
+
+    echo "Found NVMe devices: $${SSD_NVME_DEVICE_LIST[@]}"
+
+    if [ $${#SSD_NVME_DEVICE_LIST[@]} -eq 0 ]; then
+      echo "No NVMe instance storage devices found. Exiting."
+      exit 0
+    fi
+
+    for device in "$${SSD_NVME_DEVICE_LIST[@]}"; do
+      DRIVE_PATHS+=("$${BOTTLEROCKET_ROOT}$${device}")
+    done
+
+    echo "Configuring drives: $${DRIVE_PATHS[@]}"
+
+    for device_path in "$${DRIVE_PATHS[@]}"; do
+      echo "Creating PV on $${device_path}"
+      pvcreate "$${device_path}"
+    done
+
+    echo "Creating VG instance-store-vg"
+    vgcreate instance-store-vg "$${DRIVE_PATHS[@]}"
+
+    echo "PV Status:"
+    pvs
+    echo "VG Status:"
+    vgs
+
+    echo "Completed LVM setup successfully"
+  EOT
+  )
 }
 
 module "eks" {
@@ -36,6 +78,15 @@ module "eks" {
         "materialize.cloud/disk" = "true"
         "workload"               = "materialize-instance"
       }
+
+      bootstrap_extra_args = <<-TOML
+        [settings.bootstrap-containers.disk-setup]
+        source = "public.ecr.aws/amazonlinux/amazonlinux:2"
+        mode = "once"
+        essential = true
+        user-data = "${local.disk_setup_user_data}"
+      TOML
+
     }
   }
 
@@ -75,3 +126,29 @@ module "eks" {
 
   tags = var.tags
 }
+
+# Install OpenEBS for lgalloc support
+resource "kubernetes_namespace" "openebs" {
+  count = var.install_openebs ? 1 : 0
+
+  metadata {
+    name = var.openebs_namespace
+  }
+}
+
+resource "helm_release" "openebs" {
+  count = var.install_openebs ? 1 : 0
+
+  name       = "openebs"
+  namespace  = kubernetes_namespace.openebs[0].metadata[0].name
+  repository = "https://openebs.github.io/openebs"
+  chart      = "openebs"
+  version    = var.openebs_version
+
+  set {
+    name  = "engines.replicated.mayastor.enabled"
+    value = "false"
+  }
+
+  depends_on = [kubernetes_namespace.openebs]
+}

modules/eks/variables.tf

@@ -72,3 +72,22 @@ variable "enable_cluster_creator_admin_permissions" {
   type        = bool
   default     = true
 }
+
+# OpenEBS configuration
+variable "install_openebs" {
+  description = "Whether to install OpenEBS for NVMe storage"
+  type        = bool
+  default     = true
+}
+
+variable "openebs_namespace" {
+  description = "Namespace for OpenEBS components"
+  type        = string
+  default     = "openebs"
+}
+
+variable "openebs_version" {
+  description = "Version of OpenEBS Helm chart to install"
+  type        = string
+  default     = "4.2.0"
+}

modules/eks/versions.tf

@@ -6,5 +6,13 @@ terraform {
       source  = "hashicorp/aws"
       version = "~> 5.0"
     }
+    kubernetes = {
+      source  = "hashicorp/kubernetes"
+      version = "~> 2.0"
+    }
+    helm = {
+      source  = "hashicorp/helm"
+      version = "~> 2.0"
+    }
   }
 }

variables.tf

@@ -363,3 +363,42 @@ variable "install_metrics_server" {
   type        = bool
   default     = true
 }
+
+variable "install_openebs" {
+  description = "Whether to install OpenEBS for lgalloc support"
+  type        = bool
+  default     = true
+}
+
+variable "storage_class_create" {
+  description = "Whether to create the storage class"
+  type        = bool
+  default     = true
+}
+
+variable "storage_class_name" {
+  description = "Name of the storage class"
+  type        = string
+  default     = "openebs-lvm-instance-store-ext4"
+}
+
+variable "storage_class_provisioner" {
+  description = "Storage class provisioner"
+  type        = string
+  default     = "local.csi.openebs.io"
+}
+
+variable "storage_class_parameters" {
+  description = "Parameters for the storage class"
+  type = object({
+    storage  = string
+    fsType   = string
+    volgroup = string
+  })
+
+  default = {
+    storage  = "lvm"
+    fsType   = "ext4"
+    volgroup = "instance-store-vg"
+  }
+}