custom rdma subnet, updated defaults

Author: MarcinZablocki

bastion.tf

@@ -172,6 +172,8 @@ resource "null_resource" "cluster" {
       compute = var.node_count > 0 ? zipmap(local.cluster_instances_names, local.cluster_instances_ips) : zipmap([],[])
       public_subnet = data.oci_core_subnet.public_subnet.cidr_block, 
       private_subnet = data.oci_core_subnet.private_subnet.cidr_block, 
+      rdma_network = cidrhost(var.rdma_subnet, 0),
+      rdma_netmask = cidrnetmask(var.rdma_subnet),
       nfs = var.node_count > 0 ? local.cluster_instances_names[0] : "",
       home_nfs = var.home_nfs,
       scratch_nfs = var.use_scratch_nfs && var.node_count > 0,

inventory.tpl

@@ -13,8 +13,8 @@ bastion
 compute
 [all:vars]
 ansible_connection=ssh
-rdma_network=192.168.168.0
-rdma_netmask=255.255.252.0
+rdma_network=${rdma_network}
+rdma_netmask=${rdma_netmask}
 public_subnet=${public_subnet} 
 private_subnet=${private_subnet}
 nvme_path=/mnt/localdisk/
@@ -47,4 +47,4 @@ autoscaling_monitoring=${autoscaling_monitoring}
 autoscaling_mysql_service=${autoscaling_mysql_service}
 monitoring_mysql_ip=${monitoring_mysql_ip}
 admin_password = ${admin_password}
-admin_username = ${admin_username}
+admin_username = ${admin_username}

schema.yaml

@@ -116,8 +116,9 @@ variableGroups:
       - ${private_subnet_id}
       - ${vcn_subnet}
       - ${public_subnet}
-      - ${additional_subnet}
       - ${private_subnet}
+      - ${rdma_subnet}
+      - ${additional_subnet}
   - title: "Software"
     variables:
       - ${slurm}
@@ -146,6 +147,7 @@ variables:
     title: "target compartment"
     type: oci:identity:compartment:id
     default: ${compartment_ocid}
+    required: true
   ad:
     type: oci:identity:availabilitydomain:name
     visible: complexExpression
@@ -162,13 +164,11 @@ variables:
     type: boolean
     title: "use custom cluster name"
     description: "Use custom name for the cluster"
-    required: true
     default: false
   ldap:
     type: boolean
     title: "Configure LDAP authentication from bastion"
     description: "When selected nodes will be configured to use LDAP authentication. User and group management can be performed using cluster commands."
-    required: true
     default: true
   cluster_name:
     title: "Name of the cluster"
@@ -191,6 +191,7 @@ variables:
     dependsOn:
       compartmentId: ${targetCompartment}
     required: true
+    default: VM.Standard2.4
   bastion_ocpus:
     type: integer
     description: Number of OCPU's for flex shape
@@ -212,7 +213,7 @@ variables:
           - eq: 
             - ${bastion_shape}
             - "VM.Standard.A1.Flex"
-    required: false
+    required: true
   bastion_custom_memory: 
     title: Use custom memory size
     type: boolean
@@ -257,16 +258,13 @@ variables:
               - "VM.Standard.A1.Flex"
         - and: 
             - ${bastion_custom_memory}
-    required: false
-
+    required: true
 
-    
   use_standard_image:
     type: boolean
     title: "use standard bastion image"
     description: >
       "Use standard bastion image, otherwise provide custom image OCID"
-    required: true
     default: true
     visible: 
       and: 
@@ -287,6 +285,7 @@ variables:
     description: "Custom image ID for Bastion"
     type: string
     default: "opc"
+    required: true
     visible:
       not:
         - ${use_standard_image}
@@ -301,6 +300,7 @@ variables:
         - ${unsupported_bastion}
         - not: 
             - ${use_standard_image}
+    default: "image.ocid"
 
   bastion_image_compartment:
     title: "bastion image compartment"
@@ -339,7 +339,6 @@ variables:
       and:
         - ${use_advanced}
     default: false
-    required: true
   bastion_block_volume_size:
     required: true
     type: integer
@@ -387,6 +386,7 @@ variables:
       and:
         - ${use_advanced}
         - ${use_cluster_nfs}
+    reguired: true
   cluster_network:
     title: Use cluster network
     type: boolean
@@ -410,6 +410,7 @@ variables:
   instance_pool_shape:
     title: "Shape of the Compute Nodes"
     required: true
+    default: "VM.Standard2.4"
     type: oci:core:instanceshape:name
     dependsOn:
       compartmentId: ${targetCompartment}
@@ -440,7 +441,7 @@ variables:
           - eq: 
             - ${instance_pool_shape}
             - "VM.Standard.A1.Flex"
-    required: false
+    required: true
 
   instance_pool_custom_memory: 
     title: Use custom memory size
@@ -486,7 +487,7 @@ variables:
               - "VM.Standard.A1.Flex"
         - and: 
             - ${instance_pool_custom_memory}
-    required: false
+    required: true
 
   node_count:
     required: true
@@ -497,9 +498,8 @@ variables:
     description: "Number of Compute Instances (Permanent Cluster when autoscaling)"
 
   hyperthreading:
-    required: true
     type: boolean
-    title: "Keep Hyperthreading enabled"
+    title: "Hyperthreading enabled"
     default: true
     description: "When unchecked SMT will be disabled" 
 
@@ -515,7 +515,6 @@ variables:
     type: boolean
     title: "use marketplace image"
     description: "Use marketplace image, otherwise provide custom image OCID"
-    required: true
     default: true
 
   marketplace_listing:
@@ -628,13 +627,15 @@ variables:
         - ${cluster_network}
         - ${use_advanced}
         - ${use_scratch_nfs}
+    required: true
   scratch_nfs_type_pool:
     type: enum
     title: "Scratch storage configuration"
     enum:
       - "none"
       - "block"
     default: "none"
+    required: true
     visible:
       and: 
         - not:
@@ -702,6 +703,7 @@ variables:
     title: "NFS scratch space mount point"
     description: "Path to NFS share"
     default: "/nfs/scratch"
+    required: true
     type: string
     visible:
       and:
@@ -712,7 +714,6 @@ variables:
     title: "Use Existing VCN"
     description: "Use existing VCN or create new one"
     default: false
-    required: true
   vcn_compartment:
     title: "VCN compartment"
     type: oci:identity:compartment:id
@@ -735,13 +736,15 @@ variables:
       vcnId: ${vcn_id}
       hidePrivateSubnet: true
     visible: ${use_existing_vcn}
+    required: true
   private_subnet_id:
     type: oci:core:subnet:id
     dependsOn:
       compartmentId: ${vcn_compartment}
       vcnId: ${vcn_id}
       hidePublicSubnet: true
     visible: ${use_existing_vcn}
+    required: true
   vcn_subnet:
     type: string
     title: "VCN IP range"
@@ -769,6 +772,12 @@ variables:
     visible:
       not:
         - ${use_existing_vcn}
+  rdma_subnet:
+    type: string
+    title: "RDMA subnet IP range"
+    default: "192.168.168.0/22"
+    description: "Must be the same size as private subnet"
+    required: true
   private_subnet:
     type: string
     title: "Private subnet IP range"
@@ -784,9 +793,9 @@ variables:
     default: "0.0.0.0/0"
     description: "Allowed SSH network in CIDR notation"
     required: true
-    visible:
-      not:
-        - ${use_existing_vcn}
+#   visible:
+#     not:
+#       - ${use_existing_vcn}
   slurm:
     type: boolean
     title: "Install SLURM"
@@ -797,6 +806,7 @@ variables:
     type: String
     title: "Queue Name"
     default: "compute"
+    required: true
     description: "Add the permanent cluster to a specific queue, workq is the default queue" 
     visible:
         - ${slurm}
@@ -876,6 +886,7 @@ variables:
     description: "Compartment to add the FSS Mount Target and File System"
     type: oci:identity:compartment:id
     default: ${targetCompartment}
+    required: true
     visible: 
       and:
         - ${add_nfs}
@@ -892,6 +903,7 @@ variables:
     default: ${ad}
     description: "FSS Availability Domain"
     title: "FSS Availability Domain"
+    required: true
 
   nfs_target_path:
     type: string
@@ -901,10 +913,12 @@ variables:
     visible: 
       and:
         - ${add_nfs}
+    required: true
   nfs_source_IP:
     type: string
     title: "NFS server IP"
     default: "0.0.0.0"
+    required: true
     description: "IP address of the NFS server"
     visible: 
       and:
@@ -917,6 +931,7 @@ variables:
     default: "/app"
     description: "Value of the path on the NFS server"
     visible: ${add_nfs}
+    required: true
   nfs_options:
     type: string
     title: "Options"
@@ -931,16 +946,17 @@ variables:
     type: string
     title: "MySQL Shape Name"
     default: "MySQL.VM.Standard.E3.1.16GB"
+    required: true
     description: "MySQL Shape Name"
     visible: 
       and:
         - ${autoscaling_mysql_service} 
 
-
   admin_username:
     type: string
     title: "MySQL Monitoring username"
     default: "admin"
+    required: true
     description: ""
     visible: 
       and:
@@ -953,6 +969,7 @@ variables:
     visible: 
       and:
         - ${autoscaling_mysql_service}
+    required: true
 
   autoscaling_monitoring:
     type: boolean

variables.tf

@@ -4,10 +4,10 @@ variable "targetCompartment" {}
 variable "ad" {}
 variable "ssh_key" {}
 variable "cluster_network" { default = true } 
-variable "use_custom_name" {}
+variable "use_custom_name" { default = false }
 variable "cluster_name" { default = "" }
 variable "bastion_ad" {}
-variable "bastion_shape" {}
+variable "bastion_shape" { default = "VM.Standard2.4" }
 variable "use_standard_image" { default= true }
 variable "custom_bastion_image" { 
   type = string
@@ -16,9 +16,9 @@ variable "custom_bastion_image" {
 variable "bastion_boot_volume_size" {}
 variable "cluster_network_shape" { default = "BM.HPC2.36" }
 variable "instance_pool_shape" { default = "VM.Standard2.4" }
-variable "node_count" {}
-variable "boot_volume_size" {}
-variable "use_marketplace_image" {}
+variable "node_count" { default = 2 }
+variable "boot_volume_size" { default = 50 }
+variable "use_marketplace_image" { default = true}
 variable "image" { default = "ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa" }
 variable "image_ocid" { default = "ocid1.image.oc1..aaaaaaaa5yxem7wzie34hi5km4qm2t754tsfxrjuefyjivebrxjad4jcj5oa" }
 variable "unsupported_bastion_image" { default = "" } 
@@ -28,13 +28,14 @@ variable "cluster_nfs_path" { default = "/nfs/cluster" }
 variable "scratch_nfs_path" { default = "/nfs/scratch" } 
 variable "vcn_compartment" { default = ""}
 variable "vcn_id" { default = ""}
-variable "use_existing_vcn" {}
+variable "use_existing_vcn" { default = false}
 variable "public_subnet_id" { default = ""}
 variable "private_subnet_id" { default = ""}
-variable "vcn_subnet" { default = "" }
-variable "public_subnet" { default = "" }
-variable "additional_subnet" { default = "" }
-variable "private_subnet" { default = "" }
+variable "vcn_subnet" { default = "172.16.0.0/21" }
+variable "public_subnet" { default = "172.16.0.0/24" }
+variable "additional_subnet" { default = "172.16.1.0/24" }
+variable "rdma_subnet" { default = "192.168.168.0/22" }
+variable "private_subnet" { default = "172.16.4.0/22" }
 variable "ssh_cidr" { default = "0.0.0.0/0" }
 variable "slurm" { default = false }
 variable "ldap" { default = true } 
@@ -155,4 +156,4 @@ variable "admin_username" {
 variable "admin_password" { 
   type = string
   default = "Monitor2021!"
-}
+}