Add option to mount NVMe's as a Logical volume

Author: arnaudfroidmont

autoscaling/tf_init/bastion_update.tf

@@ -43,6 +43,8 @@ resource "local_file" "inventory" {
     nfs_source_path = var.nfs_source_path,
     nfs_options = var.nfs_options,
     localdisk = var.localdisk,
+    log_vol = var.log_vol,
+    redundancy = var.redundancy,
     cluster_nfs_path = var.cluster_nfs_path,
     scratch_nfs_path = var.scratch_nfs_path,
     cluster_network = var.cluster_network,

autoscaling/tf_init/inventory.tpl

@@ -55,6 +55,8 @@ nfs_source_IP=${nfs_source_IP}
 nfs_source_path=${nfs_source_path}
 nfs_options=${nfs_options}
 localdisk=${localdisk}
+redundancy=${redundancy}
+log_vol=${log_vol}
 ldap=${ldap}
 queue=${queue}
 instance_type=${instance_type}

bastion.tf

@@ -207,6 +207,8 @@ resource "null_resource" "cluster" {
       nfs_source_path = var.nfs_source_path,
       nfs_options = var.nfs_options,
       localdisk = var.localdisk,
+      log_vol = var.log_vol,
+      redundancy = var.redundancy,
       cluster_network = var.cluster_network,
       slurm = var.slurm,
       rack_aware = var.rack_aware,
@@ -362,6 +364,8 @@ resource "null_resource" "cluster" {
       nfs_source_path = var.nfs_source_path,
       nfs_options = var.nfs_options,
       localdisk = var.localdisk,
+      log_vol = var.log_vol,
+      redundancy = var.redundancy,
       monitoring = var.monitoring,
       hyperthreading = var.hyperthreading,
       unsupported = var.unsupported,

conf/variables.tpl

@@ -126,5 +126,7 @@ variable "bastion_username" { default = "${bastion_username}" }
 variable "compute_username" { default = "${compute_username}" }
 
 variable "localdisk" { default = "${localdisk}" }
+variable "log_vol" { default = "${log_vol}" }
+variable "redundancy" { default = "${redundancy}" }
 
 variable "instance_pool_ocpus_denseIO_flex" { default = "##OCPU##"}

inventory.tpl

@@ -47,6 +47,8 @@ nfs_source_IP=${nfs_source_IP}
 nfs_source_path=${nfs_source_path}
 nfs_options=${nfs_options}
 localdisk=${localdisk}
+redundancy=${redundancy}
+log_vol=${log_vol}
 instance_pool_ocpus=${instance_pool_ocpus}
 queue=${queue}
 monitoring=${monitoring}

playbooks/roles/localdisk/tasks/common.yml

@@ -1,32 +1,94 @@
 ---
+- name: check path
+  set_fact:
+    nvme_path_edited: "{% if nvme_path[-1] == '/' %}{{nvme_path[:-1]}}{% else%}{{nvme_path}}{% endif %}"
+
+- name: Get the number of NVMe's
+  set_fact:
+    nvme_count: "{{ hostvars[inventory_hostname]['ansible_devices'] | select('match','nvme[0-9]n1') | list | length}}"
+
+- name: Create a LVM?
+  set_fact:
+    one_lv: "{{( log_vol | bool ) and ( ( nvme_count | int ) > 1 )}}"
+
 - name: Create a new primary partition
   parted:
-    device: /dev/nvme0n1
+    device: "/dev/{{item}}"
     number: 1
     state: present
     label: gpt
-  when: "'nvme0n1' in hostvars[inventory_hostname].ansible_devices"
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_devices'] | select('match','nvme[0-9]n1') | list }}"
+
 - name: create a filesystem
   filesystem:
-    dev: /dev/nvme0n1p1
+    dev: "/dev/{{item}}p1"
     fstype: xfs
-    opts: -L localscratch
-  when: "'nvme0n1' in hostvars[inventory_hostname].ansible_devices"
+    opts: "-L locscratch{{item | replace('nvme','') | replace('n1','')}}"
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_devices'] | select('match','nvme[0-9]n1') | list }}"
+  when: not ( one_lv | bool )
+
 - name: Mount local volume
   mount:
-    path: "{{ nvme_path }}"
-    src: LABEL=localscratch
+    path: "{% if item | replace('nvme','') | replace('n1','') == '0' %}{{ nvme_path_edited}}{% else%}{{ nvme_path_edited}}{{item | replace('nvme','') | replace('n1','')}}{% endif %}"
+    src: "LABEL=locscratch{{item | replace('nvme','') | replace('n1','')}}"
     fstype: xfs
     opts: defaults,noatime
     state: mounted
-  when: "'nvme0n1' in hostvars[inventory_hostname].ansible_devices"
-- name: "set permissions on {{ nvme_path }}"
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_devices'] | select('match','nvme[0-9]n1') | list }}"
+  when: not ( one_lv | bool )
+
+- name: "set permissions on {{ nvme_path_edited }}"
   become: true
   file:
-    path: "{{ nvme_path }}"
+    path: "{% if item | replace('nvme','') | replace('n1','') == '0' %}{{ nvme_path_edited}}{% else%}{{ nvme_path_edited}}{{item | replace('nvme','') | replace('n1','')}}{% endif %}"
     state: directory
     owner: "{{ ansible_user }}"
     mode: 0775
     group: "{{privilege_group_name}}"
     recurse: no
-  when: "'nvme0n1' in hostvars[inventory_hostname].ansible_devices"
+  with_items:
+    - "{{ hostvars[inventory_hostname]['ansible_devices'] | select('match','nvme[0-9]n1') | list }}"
+  when: not ( one_lv | bool )
+
+- name: Create volume group
+  lvg:
+    vg: "vg_nvmes"
+    pvs: "{{['/dev/']|product(hostvars[inventory_hostname]['ansible_devices'] | select('match','nvme[0-9]n1') | list)|map('join', '') | join(',')}}"
+  when: one_lv | bool
+
+- name: Create Logical volume
+  lvol:
+    vg: "vg_nvmes"
+    lv: "lv_nvmes"
+    size: 100%FREE
+    opts: "{% if redundancy | bool %}--type raid10{% else%}-i4{% endif %}"
+  when: one_lv | bool
+
+- name: Create file system
+  filesystem:
+    fstype: xfs
+    dev: "/dev/vg_nvmes/lv_nvmes"
+  when: one_lv | bool
+
+- name: Mount local volume
+  mount:
+    path: "{{ nvme_path_edited}}"
+    src: "/dev/vg_nvmes/lv_nvmes"
+    fstype: xfs
+    opts: defaults,noatime
+    state: mounted
+  when: one_lv | bool
+
+- name: "set permissions on {{ nvme_path_edited }}"
+  become: true
+  file:
+    path: "{{ nvme_path_edited}}"
+    state: directory
+    owner: "{{ ansible_user }}"
+    mode: 0775
+    group: "{{privilege_group_name}}"
+    recurse: no
+  when: one_lv | bool

schema.yaml

@@ -138,6 +138,8 @@ variableGroups:
       - ${cluster_block_volume_size}
       - ${cluster_block_volume_performance}
       - ${localdisk}
+      - ${log_vol}
+      - ${redundancy}
   - title: "Network options"
     variables:
       - ${use_existing_vcn}
@@ -1142,6 +1144,27 @@ variables:
     description: "For nodes using a NVMe, mount the localdisk"
     visible: ${use_advanced}
 
+  log_vol:
+    type: boolean
+    title: "One Logical Volume"
+    default: true
+    description: "Mount all NVMe's as one logical volume"
+    visible: 
+      and: 
+        - ${use_advanced}
+        - ${localdisk}
+    
+  redundancy:
+    type: boolean
+    title: "Redundancy"
+    default: true
+    description: "Use RAID for redundancy"
+    visible: 
+      and: 
+        - ${use_advanced}
+        - ${localdisk}
+        - ${log_vol}
+
   nfs_target_path:
     type: string
     title: "NFS Path"

slurm_ha.tf

@@ -218,6 +218,8 @@ resource "null_resource" "cluster_backup" {
       nfs_source_path = var.nfs_source_path,
       nfs_options = var.nfs_options,
       localdisk = var.localdisk,
+      log_vol = var.log_vol,
+      redundancy = var.redundancy,
       cluster_network = var.cluster_network,
       slurm = var.slurm,
       slurm_nfs_path = var.slurm_nfs ? var.nfs_source_path : var.cluster_nfs_path,
@@ -373,6 +375,8 @@ resource "null_resource" "cluster_backup" {
       nfs_source_path = var.nfs_source_path,
       nfs_options = var.nfs_options,
       localdisk = var.localdisk,
+      log_vol = var.log_vol,
+      redundancy = var.redundancy,
       monitoring = var.monitoring,
       hyperthreading = var.hyperthreading,
       unsupported = var.unsupported,

variables.tf

@@ -237,7 +237,8 @@ variable cluster_nfs_export {default = ""}
 variable "private_deployment" { default = false }
 
 variable "localdisk" { default = true }
-
+variable "log_vol" { default = false }
+variable "redundancy" { default = true }
 
 variable "use_marketplace_image_login" { default = true}
 variable "use_old_marketplace_image_login" { default = false}